Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

server/packetstation/api/feed.py

@@ -0,0 +1,308 @@
+"""The VISUAL feed for Clerical: the From VISUAL panel (status, runs, ignored work orders, unmapped resources,
+projects without names), Run now, pulling in a work order released before go-live, and what VISUAL changed.
+
+Clerical (clerical or clerical_supervisor) or an admin. Pulling in a work order changes what the feed tracks, so an
+admin outside Clerical gives an IT reason for it. Screens read PostgreSQL only: nothing here calls VISUAL. Run now
+inserts a job request and NOTIFYs the worker, which does the reading.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Literal
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel, Field
+
+from .. import audit
+from ..domain import packets
+from ..domain.pipeline import jsonable
+from ..text import LIMITS, clean_opt, reason_ok
+from .deps import Db, Principal, current_principal, get_db, require_password_session
+from .errors import ApiError
+from .packets import WRITE
+
+router = APIRouter(prefix="/api/feed", tags=["feed"])
+
+CLERICAL = frozenset({"clerical", "clerical_supervisor"})
+JOBS = ("visual-feed", "drawing-lookup")
+IGNORED_MAX = 200
+UNMAPPED_MAX = 30
+STALE_DEFAULT = 90
+PULL_IN_REASON_MIN = 3
+PULL_IN_REASON_MAX = 300
+
+
+class RunBody(BaseModel):
+    job: Literal["visual-feed", "drawing-lookup"] = "visual-feed"
+    note: str | None = Field(default=None, max_length=1000)
+
+
+class PullInBody(BaseModel):
+    reason: str | None = Field(default=None, max_length=2000)
+    it_reason: str | None = Field(default=None, max_length=1000)
+
+
+# ---------------------------------------------------------------- access
+
+
+def require_clerical(p: Principal = Depends(current_principal)) -> Principal:  # noqa: B008
+    if CLERICAL & set(p.roles) or p.is_admin:
+        return p
+    raise ApiError(403, "forbidden", "Only Clerical can see the VISUAL feed.")
+
+
+def _it_reason(p: Principal, it_reason: str | None) -> str | None:
+    """None for Clerical. An admin outside Clerical needs a password session and an IT reason."""
+    if CLERICAL & set(p.roles):
+        return None
+    require_password_session(p)
+    why = reason_ok(it_reason)
+    if why is None:
+        raise ApiError(400, "it_reason_required", "Give an IT reason of at least 8 characters.")
+    return why
+
+
+# ---------------------------------------------------------------- shapes
+
+
+def _run_out(r: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "id": r["id"],
+        "job": r["job"],
+        "trigger": r["trigger"],
+        "requested_by": [str(u) for u in r.get("requested_by") or []],
+        "started_at": r["started_at"],
+        "finished_at": r["finished_at"],
+        "status": r["status"],
+        "counts": r["counts"] or {},
+        "errors": r["errors"] or [],
+        "build_version": r.get("build_version"),
+        "visual_target": r.get("visual_target"),
+        "host": r.get("host"),
+    }
+
+
+def _runs(conn: Any, job: str, limit: int) -> list[dict[str, Any]]:
+    return [
+        _run_out(r)
+        for r in packets.rows(conn, "SELECT * FROM job_runs WHERE job = %s ORDER BY id DESC LIMIT %s", (job, limit))
+    ]
+
+
+def _ignored_out(r: dict[str, Any]) -> dict[str, Any]:
+    keys = ("id", "wo_base", "lot", "split", "sub", "rel", "project", "unit_number", "res", "dwg", "last_seen_at")
+    out = {k: r[k] for k in keys}
+    out["pulled_in_at"] = r["pulled_in_at"]
+    out["pulled_in_by"] = str(r["pulled_in_by"]) if r["pulled_in_by"] else None
+    out["pulled_in_reason"] = r["pulled_in_reason"]
+    return out
+
+
+def _stale_seconds() -> int:
+    try:
+        from ..worker import STALE_SECONDS
+    except ImportError:  # the worker package is agent A's; the panel still answers without it
+        return STALE_DEFAULT
+    return int(STALE_SECONDS)
+
+
+def _feed_settings(conn: Any) -> Any:
+    from ..feed import config
+
+    return config.load(conn)
+
+
+def _request_run(conn: Any, p: Principal, job: str, note: str | None) -> int:
+    """Insert a job request and NOTIFY the worker in the same transaction. Returns the pending request count."""
+    conn.execute(
+        "INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)",
+        (job, p.user_id, clean_opt(note, LIMITS["job_note"])),
+    )
+    conn.execute("SELECT ps_notify('ps_job_request', %s)", (json.dumps({"job": job}),))
+    return int(
+        conn.execute("SELECT count(*) FROM job_requests WHERE job = %s AND claimed_at IS NULL", (job,)).fetchone()[0]
+    )
+
+
+# ---------------------------------------------------------------- endpoints
+
+
+@router.get("/overview")
+def overview(
+    p: Principal = Depends(require_clerical),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Everything the From VISUAL panel shows, from PostgreSQL only."""
+    conn = d.conn
+    fs = _feed_settings(conn)
+    sched = packets.one(conn, "SELECT next_run_at, paused FROM job_schedule WHERE job = 'visual-feed'")
+    beat = packets.one(
+        conn,
+        "SELECT at, build, running, now() - at > make_interval(secs => %s) AS stale FROM worker_heartbeat WHERE id = 1",
+        (_stale_seconds(),),
+    )
+    runs = _runs(conn, "visual-feed", 10)
+    last_run = runs[0] if runs else None
+    finished = next((r for r in runs if r["status"] in ("ok", "degraded")), None)
+    unmapped_raw = (finished or {}).get("counts", {}).get("unmapped") or {}
+    unmapped: list[dict[str, Any]] = []
+    if isinstance(unmapped_raw, dict):
+        for k, v in unmapped_raw.items():
+            if isinstance(v, int) or (isinstance(v, str) and v.isdigit()):
+                unmapped.append({"resource": str(k), "rows": int(v)})
+        unmapped.sort(key=lambda x: (-int(x["rows"]), str(x["resource"])))
+        unmapped = unmapped[:UNMAPPED_MAX]
+    ignored = packets.rows(
+        conn,
+        "SELECT * FROM visual_ignored_wos WHERE pulled_in_at IS NULL ORDER BY rel DESC NULLS LAST, wo_base, lot, id"
+        " LIMIT %s",
+        (IGNORED_MAX,),
+    )
+    nameless = packets.rows(
+        conn,
+        "SELECT p.id, p.number,"
+        " (SELECT count(*) FROM units u WHERE u.project_id = p.id AND u.removed_at IS NULL) AS units,"
+        " (SELECT count(*) FROM packets k JOIN units u ON u.id = k.unit_id"
+        "   WHERE u.project_id = p.id AND k.removed_at IS NULL) AS packets"
+        " FROM projects p WHERE p.source = 'visual' AND p.needs_details AND p.name IS NULL AND p.removed_at IS NULL"
+        " ORDER BY p.number LIMIT 200",
+    )
+    return {
+        "status": {
+            "paused": bool(fs.paused),
+            "next_run_at": (sched or {}).get("next_run_at"),
+            "worker": None
+            if beat is None
+            else {
+                "at": beat["at"],
+                "build": beat["build"],
+                "running": list(beat["running"] or []),
+                "stale": bool(beat["stale"]),
+            },
+        },
+        "last_run": last_run,
+        "runs": runs,
+        "drawing_runs": _runs(conn, "drawing-lookup", 10),
+        "pending_requests": int(
+            conn.execute(
+                "SELECT count(*) FROM job_requests WHERE job = 'visual-feed' AND claimed_at IS NULL"
+            ).fetchone()[0]
+        ),
+        "ignored": [_ignored_out(r) for r in ignored],
+        "unmapped": unmapped,
+        "projects_without_names": [
+            {"id": r["id"], "number": r["number"], "units": int(r["units"]), "packets": int(r["packets"])}
+            for r in nameless
+        ],
+        "golive_date": fs.golive_date,
+        "departments": list(fs.departments),
+    }
+
+
+@router.post("/run", dependencies=WRITE)
+def run_now(
+    body: RunBody | None = None,
+    p: Principal = Depends(require_clerical),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Run now: ask the worker for a feed (or drawing) run. Requests made while one is pending share that run."""
+    body = body or RunBody()
+    conn = d.conn
+    with conn.transaction():
+        pending = _request_run(conn, p, body.job, body.note)
+        audit.append(
+            conn,
+            action="feed.run_requested",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="job",
+            subject_id=body.job,
+            detail={"job": body.job, "pending": pending},
+            ip=p.ip,
+        )
+    return {"requested": True, "job": body.job, "pending": pending}
+
+
+@router.post("/ignored/{row_id}/pull-in", dependencies=WRITE)
+def pull_in(
+    row_id: int,
+    body: PullInBody,
+    p: Principal = Depends(require_clerical),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Track a work order released before the go-live date after all. The next feed run makes its packet."""
+    it_reason = _it_reason(p, body.it_reason)
+    reason = clean_opt(body.reason, PULL_IN_REASON_MAX)
+    if reason is None or len(reason) < PULL_IN_REASON_MIN:
+        raise ApiError(400, "reason_required", "Say why this work order is pulled in (at least 3 characters).")
+    conn = d.conn
+    with conn.transaction():
+        row = packets.one(conn, "SELECT * FROM visual_ignored_wos WHERE id = %s FOR UPDATE", (row_id,))
+        if row is None:
+            raise ApiError(404, "not_found", "No such work order on the ignored list.")
+        if row["pulled_in_at"] is not None:
+            raise ApiError(
+                409,
+                "already_pulled_in",
+                "This work order was already pulled in.",
+                extra={"current": jsonable(_ignored_out(row))},
+            )
+        row = packets.one(
+            conn,
+            "UPDATE visual_ignored_wos SET pulled_in_at = now(), pulled_in_by = %s, pulled_in_reason = %s"
+            " WHERE id = %s RETURNING *",
+            (p.user_id, reason, row_id),
+        )
+        assert row is not None
+        key = "/".join(str(row[k]) for k in ("wo_base", "lot", "split", "sub"))
+        audit.append(
+            conn,
+            action="feed.pulled_in",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="work_order",
+            subject_id=key,
+            detail={"id": row_id, "key": key, "rel": str(row["rel"]) if row["rel"] else None, "reason": reason},
+            it_reason=it_reason,
+            ip=p.ip,
+        )
+        _request_run(conn, p, "visual-feed", f"pull in {key}")
+    return _ignored_out(row)
+
+
+@router.get("/batches")
+def batches(
+    limit: int = Query(20, ge=1, le=200),
+    p: Principal = Depends(require_clerical),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> list[dict[str, Any]]:
+    return packets.rows(d.conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT %s", (limit,))
+
+
+@router.get("/changes")
+def changes(
+    batch_id: int | None = Query(None),
+    limit: int = Query(200, ge=1, le=2000),
+    p: Principal = Depends(require_clerical),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """What VISUAL changed in one batch (the latest when none is given): work order rows and material lines."""
+    conn = d.conn
+    if batch_id is None:
+        batch = packets.one(conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT 1")
+    else:
+        batch = packets.one(conn, "SELECT * FROM visual_batches WHERE id = %s", (batch_id,))
+        if batch is None:
+            raise ApiError(404, "not_found", "No such feed batch.")
+    if batch is None:
+        return {"batch": None, "rows": [], "materials": []}
+    rows = packets.rows(
+        conn, "SELECT * FROM visual_row_changes WHERE batch_id = %s ORDER BY id LIMIT %s", (batch["id"], limit)
+    )
+    mats = packets.rows(
+        conn, "SELECT * FROM visual_material_changes WHERE batch_id = %s ORDER BY id LIMIT %s", (batch["id"], limit)
+    )
+    return {"batch": batch, "rows": rows, "materials": mats}

server/packetstation/api/feed_settings.py

@@ -0,0 +1,204 @@
+"""Admin: the feed settings and the VISUAL resource map. Every route needs the admin role and a password session.
+
+A settings save NOTIFYs `ps_settings` so the worker reschedules at once. Resource map changes apply on the next
+feed run. Every change is audited with before and after.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, Body, Depends, Response
+from psycopg.rows import dict_row
+from pydantic import BaseModel, Field
+
+from .. import audit
+from ..domain.projects import Who
+from ..feed import config
+from .deps import Db, Principal, admin_principal, get_db
+from .errors import ApiError
+from .jobs import recent_runs
+from .packets import WRITE
+
+router = APIRouter(prefix="/api/admin/feed", tags=["admin"])
+
+FEED_JOB = "visual-feed"
+SPECIAL_TARGETS = ("fold", "ignore")
+RESOURCE_MAX = 100
+
+
+def _who(p: Principal) -> Who:
+    return Who(user_id=p.user_id, session_id=p.session_id, role_snapshot=p.role_snapshot, ip=p.ip)
+
+
+def _shop_codes(conn: Any) -> set[str]:
+    rows = conn.execute("SELECT code FROM departments WHERE kind = 'shop' AND active").fetchall()
+    return {r[0] for r in rows}
+
+
+def _settings_out(conn: Any) -> dict[str, Any]:
+    fs, version, updated_at, updated_by = config.load_versioned(conn)
+    nxt = conn.execute("SELECT next_run_at FROM job_schedule WHERE job = %s", (FEED_JOB,)).fetchone()
+    return {
+        **fs.model_dump(mode="json"),
+        "version": version,
+        "updated_at": updated_at,
+        "updated_by": str(updated_by) if updated_by else None,
+        "next_run_at": nxt[0] if nxt else None,
+        "runs": recent_runs(conn, FEED_JOB, 10),
+    }
+
+
+@router.get("/settings")
+def get_settings(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> dict[str, Any]:  # noqa: B008
+    return _settings_out(d.conn)
+
+
+@router.patch("/settings", dependencies=WRITE)
+def patch_settings(
+    patch: dict[str, Any] = Body(...),  # noqa: B008
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    conn = d.conn
+    if "departments" in patch and isinstance(patch["departments"], list):
+        shop = _shop_codes(conn)
+        bad = sorted({str(x).strip() for x in patch["departments"]} - shop)
+        if bad:
+            raise ApiError(
+                422,
+                "bad_settings",
+                f"Not an active shop department: {', '.join(bad)}.",
+                extra={"errors": [f"departments: {b} is not an active shop department" for b in bad]},
+            )
+    try:
+        config.save(conn, who=_who(p), patch=patch)
+    except config.SettingsError as e:
+        raise ApiError(422, "bad_settings", str(e), extra={"errors": e.errors}) from None
+    return _settings_out(conn)
+
+
+# ---------------------------------------------------------------- resource map
+
+
+class MapBody(BaseModel):
+    target: str = Field(max_length=20)
+    confirmed: bool = True
+
+
+def _resource(resource: str) -> str:
+    r = resource.strip()
+    if not r or len(r) > RESOURCE_MAX:
+        raise ApiError(400, "bad_resource", "Give the VISUAL resource name.")
+    return r
+
+
+def _map_row(conn: Any, resource: str) -> dict[str, Any] | None:
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(
+            "SELECT resource, target, confirmed, updated_at, updated_by FROM visual_resource_map WHERE resource = %s",
+            (resource,),
+        ).fetchone()
+    return dict(r) if r else None
+
+
+def _row_out(r: dict[str, Any], rows_last_run: int | None) -> dict[str, Any]:
+    return {
+        "resource": r["resource"],
+        "target": r.get("target"),
+        "confirmed": bool(r.get("confirmed")),
+        "rows_last_run": rows_last_run,
+        "updated_at": r.get("updated_at"),
+    }
+
+
+def _last_counts(conn: Any) -> dict[str, int]:
+    rows = conn.execute(
+        "SELECT res, count(*) FROM visual_rows"
+        " WHERE last_batch = (SELECT max(id) FROM visual_batches) AND res <> '' GROUP BY res"
+    ).fetchall()
+    return {r[0]: int(r[1]) for r in rows}
+
+
+@router.get("/resource-map")
+def resource_map(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> list[dict[str, Any]]:  # noqa: B008
+    conn = d.conn
+    counts = _last_counts(conn)
+    with conn.cursor(row_factory=dict_row) as cur:
+        mapped = cur.execute("SELECT resource, target, confirmed, updated_at FROM visual_resource_map").fetchall()
+    out = {r["resource"]: _row_out(r, counts.get(r["resource"])) for r in mapped}
+    for res, n in counts.items():
+        if res not in out:
+            out[res] = {"resource": res, "target": None, "confirmed": False, "rows_last_run": n, "updated_at": None}
+    return sorted(out.values(), key=lambda r: (r["target"] is not None, r["resource"].lower()))
+
+
+def _public(r: dict[str, Any] | None) -> dict[str, Any] | None:
+    if r is None:
+        return None
+    return {"target": r["target"], "confirmed": r["confirmed"]}
+
+
+@router.put("/resource-map/{resource}", dependencies=WRITE)
+def map_resource(
+    resource: str,
+    body: MapBody,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    conn = d.conn
+    res = _resource(resource)
+    target = body.target.strip()
+    if target.lower() in SPECIAL_TARGETS:
+        target = target.lower()
+    elif target not in _shop_codes(conn):
+        raise ApiError(422, "bad_target", "The target is Fold, Ignore or an active shop department.")
+    with conn.transaction():
+        before = _map_row(conn, res)
+        conn.execute(
+            "INSERT INTO visual_resource_map (resource, target, confirmed, updated_at, updated_by)"
+            " VALUES (%s, %s, %s, now(), %s) ON CONFLICT (resource) DO UPDATE SET target = EXCLUDED.target,"
+            " confirmed = EXCLUDED.confirmed, updated_at = now(), updated_by = EXCLUDED.updated_by",
+            (res, target, body.confirmed, p.user_id),
+        )
+        after = _map_row(conn, res)
+        audit.append(
+            conn,
+            action="feed.resource_mapped",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="visual_resource",
+            subject_id=res,
+            detail={"resource": res, "before": _public(before), "after": _public(after)},
+            ip=p.ip,
+        )
+    assert after is not None
+    return _row_out(after, _last_counts(conn).get(res))
+
+
+@router.delete("/resource-map/{resource}", status_code=204, dependencies=WRITE)
+def unmap_resource(
+    resource: str,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> Response:
+    conn = d.conn
+    res = _resource(resource)
+    with conn.transaction():
+        before = _map_row(conn, res)
+        if before is None:
+            raise ApiError(404, "not_found", "That resource is not mapped.")
+        conn.execute("DELETE FROM visual_resource_map WHERE resource = %s", (res,))
+        audit.append(
+            conn,
+            action="feed.resource_unmapped",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="visual_resource",
+            subject_id=res,
+            detail={"resource": res, "before": _public(before), "after": None},
+            ip=p.ip,
+        )
+    return Response(status_code=204)

server/packetstation/api/jobs.py

@@ -0,0 +1,170 @@
+"""The worker's jobs: schedule, heartbeat, recent runs, and "run now".
+
+Admin (password session) or Clerical (clerical or clerical_supervisor). "Run now" inserts a job_requests row and
+NOTIFYs `ps_job_request` in the same transaction; the worker claims every pending request in one run.
+`run_out`, `worker_status` and `request_run` are shared with api/feed.py.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from fastapi import APIRouter, Depends, Query
+from psycopg.rows import dict_row
+from pydantic import BaseModel, Field
+
+from .. import audit
+from ..worker.heartbeat import STALE_SECONDS
+from .deps import Db, Principal, current_principal, get_db, require_password_session
+from .errors import ApiError
+from .packets import WRITE
+
+router = APIRouter(prefix="/api/jobs", tags=["jobs"])
+
+CLERICAL = frozenset({"clerical", "clerical_supervisor"})
+JOB_CHANNEL = "ps_job_request"
+_JOB_NAME = re.compile(r"^[a-z0-9][a-z0-9-]{0,39}$")
+
+
+def jobs_principal(p: Principal = Depends(current_principal)) -> Principal:  # noqa: B008
+    """Clerical in any session, or an admin with a password session."""
+    if CLERICAL & set(p.roles):
+        return p
+    if p.is_admin:
+        return require_password_session(p)
+    raise ApiError(403, "forbidden", "Only Clerical or an admin can see and run the worker's jobs.")
+
+
+class RunBody(BaseModel):
+    note: str | None = Field(None, max_length=500)
+
+
+def run_out(r: dict[str, Any] | None) -> dict[str, Any] | None:
+    """One job_runs row as JSON."""
+    if r is None:
+        return None
+    return {
+        "id": r["id"],
+        "job": r["job"],
+        "trigger": r["trigger"],
+        "requested_by": [str(u) for u in (r.get("requested_by") or [])],
+        "started_at": r["started_at"],
+        "finished_at": r["finished_at"],
+        "status": r["status"],
+        "counts": r.get("counts") or {},
+        "errors": r.get("errors") or [],
+        "build_version": r.get("build_version"),
+        "visual_target": r.get("visual_target"),
+        "host": r.get("host"),
+    }
+
+
+def recent_runs(conn: Any, job: str, limit: int = 10) -> list[dict[str, Any]]:
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute("SELECT * FROM job_runs WHERE job = %s ORDER BY id DESC LIMIT %s", (job, limit)).fetchall()
+    return [o for o in (run_out(r) for r in rows) if o is not None]
+
+
+def worker_status(conn: Any) -> dict[str, Any]:
+    """{at, build, host, running, stale}: stale when the worker has not beaten for STALE_SECONDS or never did."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(
+            "SELECT at, build, host, running, now() - at > make_interval(secs => %s) AS stale"
+            " FROM worker_heartbeat WHERE id = 1",
+            (STALE_SECONDS,),
+        ).fetchone()
+    if r is None:
+        return {"at": None, "build": None, "host": None, "running": [], "stale": True}
+    return {
+        "at": r["at"],
+        "build": r["build"],
+        "host": r["host"],
+        "running": list(r["running"] or []),
+        "stale": bool(r["stale"]),
+    }
+
+
+def known_job(conn: Any, job: str) -> bool:
+    if not _JOB_NAME.match(job):
+        return False
+    return conn.execute("SELECT 1 FROM job_schedule WHERE job = %s", (job,)).fetchone() is not None
+
+
+def request_run(
+    conn: Any, p: Principal, job: str, *, note: str | None = None, action: str = "job.run_requested"
+) -> dict[str, Any]:
+    """Insert a request and NOTIFY the worker in one transaction. Returns {requested, pending}."""
+    with conn.transaction():
+        conn.execute(
+            "INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)",
+            (job, p.user_id, (note or "").strip() or None),
+        )
+        conn.execute("SELECT ps_notify(%s, %s)", (JOB_CHANNEL, json.dumps({"job": job}, separators=(",", ":"))))
+        pending = conn.execute(
+            "SELECT count(*) FROM job_requests WHERE job = %s AND claimed_at IS NULL", (job,)
+        ).fetchone()[0]
+        audit.append(
+            conn,
+            action=action,
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="job",
+            subject_id=job,
+            detail={"job": job, "pending": int(pending)},
+            ip=p.ip,
+        )
+    return {"requested": True, "pending": int(pending)}
+
+
+@router.get("")
+def list_jobs(p: Principal = Depends(jobs_principal), d: Db = Depends(get_db)) -> dict[str, Any]:  # noqa: B008
+    conn = d.conn
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute(
+            "SELECT s.job, s.next_run_at, s.interval_seconds, s.paused, s.updated_at,"
+            " (SELECT count(*) FROM job_requests q WHERE q.job = s.job AND q.claimed_at IS NULL) AS pending,"
+            " CASE WHEN r.id IS NULL THEN NULL ELSE to_jsonb(r) END AS last_run"
+            " FROM job_schedule s"
+            " LEFT JOIN LATERAL (SELECT * FROM job_runs j WHERE j.job = s.job ORDER BY j.id DESC LIMIT 1) r ON true"
+            " ORDER BY s.job"
+        ).fetchall()
+    jobs = [
+        {
+            "job": r["job"],
+            "next_run_at": r["next_run_at"],
+            "interval_seconds": r["interval_seconds"],
+            "paused": r["paused"],
+            "updated_at": r["updated_at"],
+            "pending": int(r["pending"]),
+            "last_run": run_out(r["last_run"]),
+        }
+        for r in rows
+    ]
+    return {"jobs": jobs, "heartbeat": worker_status(conn)}
+
+
+@router.get("/{job}/runs")
+def job_runs(
+    job: str,
+    p: Principal = Depends(jobs_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+    limit: int = Query(10, ge=1, le=100),
+) -> list[dict[str, Any]]:
+    if not known_job(d.conn, job):
+        raise ApiError(404, "not_found", "No such job.")
+    return recent_runs(d.conn, job, limit)
+
+
+@router.post("/{job}/run", dependencies=WRITE)
+def run_now(
+    job: str,
+    body: RunBody | None = None,
+    p: Principal = Depends(jobs_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    if not known_job(d.conn, job):
+        raise ApiError(404, "not_found", "No such job.")
+    return request_run(d.conn, p, job, note=body.note if body else None)

server/packetstation/api/packet_children.py

@@ -1392,6 +1392,191 @@ def issue_co(
     return _read(d, pp, packet_id)
 
 
+FEED_ORIGINS = ("visual", "drawing")
+MSG_CO_NOT_STAGED = "The new drawing has not been fetched from M-Files yet."
+
+
+def _vis_text(leg: Any, *names: str) -> str:
+    for n in names:
+        v = leg.get(n) if isinstance(leg, dict) else None
+        if v is not None and v != "":
+            return str(v)
+    return ""
+
+
+def _leg_row(leg: Any) -> dict[str, Any]:
+    """One work order row from a change order's `vis.legs` entry. Accepts a flat leg ({wo|wo_base|base, lot, split,
+    sub, seq, ...}) or one with a nested key ({key: {base, lot, split, sub}, seq, ...})."""
+    if not isinstance(leg, dict):
+        raise ApiError(422, "bad_change_order", "The change order's work order list cannot be read.")
+    key = leg.get("key") if isinstance(leg.get("key"), dict) else leg
+    base = _vis_text(key, "base", "wo_base", "wo")
+    if not base:
+        raise ApiError(422, "bad_change_order", "The change order's work order list cannot be read.")
+    seq_raw = leg.get("seq")
+    return {
+        "wo_base": base,
+        "lot": _vis_text(key, "lot"),
+        "split": _vis_text(key, "split"),
+        "sub": _vis_text(key, "sub"),
+        "seq": int(seq_raw) if seq_raw not in (None, "") else None,
+        "qty": _num(leg.get("qty")),
+        "target_hrs": _num(leg.get("tgt", leg.get("target_hrs"))),
+        "op": clean_opt(leg.get("op"), LIMITS["op"]),
+        "dwg": clean_opt(leg.get("dwg"), LIMITS["drawing"]),
+        "rev": clean_opt(leg.get("rev"), LIMITS["rev"]),
+        "partspecs": clean_opt(leg.get("partspecs"), 4000),
+        "visual_status": clean_opt(leg.get("st", leg.get("visual_status")), 10),
+    }
+
+
+def _apply_legs(conn: Any, packet_id: int, legs: list[Any], title: str | None) -> dict[str, int]:
+    """Replace the packet's work orders with the change order's legs. Rows whose key and seq match are updated in
+    place (their ids and gone marks stay), new ones are added, the rest are deleted."""
+    want = [_leg_row(lg) for lg in legs]
+    have = packets.rows(conn, "SELECT * FROM packet_work_orders WHERE packet_id = %s", (packet_id,))
+
+    def k(r: dict[str, Any]) -> tuple[Any, ...]:
+        return (r["wo_base"], r["lot"], r["split"], r["sub"], r["seq"])
+
+    by_key = {k(r): r for r in have}
+    seen: set[tuple[Any, ...]] = set()
+    counts = {"added": 0, "updated": 0, "removed": 0}
+    for sort, r in enumerate(want):
+        key = k(r)
+        if key in seen:
+            continue
+        seen.add(key)
+        cur = by_key.get(key)
+        if cur is not None:
+            conn.execute(
+                "UPDATE packet_work_orders SET qty = %s, target_hrs = %s, op = %s, dwg = %s, rev = %s, partspecs = %s,"
+                " visual_status = %s, sort = %s, row_version = row_version + 1 WHERE id = %s",
+                (
+                    r["qty"],
+                    r["target_hrs"],
+                    r["op"],
+                    r["dwg"],
+                    r["rev"],
+                    r["partspecs"],
+                    r["visual_status"],
+                    sort,
+                    cur["id"],
+                ),
+            )
+            counts["updated"] += 1
+        else:
+            conn.execute(
+                "INSERT INTO packet_work_orders (packet_id, wo_base, lot, split, sub, seq, qty, target_hrs, op,"
+                " description, visual_status, dwg, rev, partspecs, sort)"
+                " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
+                (
+                    packet_id,
+                    r["wo_base"],
+                    r["lot"],
+                    r["split"],
+                    r["sub"],
+                    r["seq"],
+                    r["qty"],
+                    r["target_hrs"],
+                    r["op"],
+                    title,
+                    r["visual_status"],
+                    r["dwg"],
+                    r["rev"],
+                    r["partspecs"],
+                    sort,
+                ),
+            )
+            counts["added"] += 1
+    gone = [r["id"] for r in have if k(r) not in seen]
+    if gone:
+        conn.execute("DELETE FROM packet_work_orders WHERE id = ANY(%s)", (gone,))
+        counts["removed"] = len(gone)
+    return counts
+
+
+def _plural(n: int, word: str) -> str:
+    return f"{n} {word}{'' if n == 1 else 's'}"
+
+
+def _apply_feed_co(w: Write, co: dict[str, Any]) -> str | None:
+    """Apply a VISUAL or M-Files change order on approval: work orders, BOM, header drawing and rev, the fingerprint,
+    and the staged drawing (old pages voided, new pages carry the change order). Returns a note for the log."""
+    from . import packet_visual as pv
+
+    conn = w.conn
+    pid = w.id
+    vis = co["vis"] if isinstance(co["vis"], dict) else {}
+    staged = co["staged"] if isinstance(co["staged"], dict) else None
+    row = w.packet
+    visual = dict(row["visual"] or {})
+    notes: list[str] = []
+    sets: dict[str, Any] = {}
+    if isinstance(vis.get("legs"), list) and vis["legs"]:
+        c = _apply_legs(conn, pid, vis["legs"], row["title"])
+        notes.append(f"work orders: {c['added']} added, {c['updated']} kept, {c['removed']} removed")
+    if isinstance(vis.get("bom"), list) and not visual.get("manual_bom"):
+        c = pv.merge_visual_bom(conn, pid, vis["bom"])
+        notes.append(f"BOM: {c['added']} added, {c['updated']} kept, {c['removed'] + c['na']} dropped")
+    if co["origin"] == "visual":
+        if vis.get("to_dwg") not in (None, ""):
+            sets["drawing"] = clean_opt(vis["to_dwg"], LIMITS["drawing"])
+        if vis.get("to_rev") not in (None, ""):
+            sets["rev"] = clean_opt(vis["to_rev"], LIMITS["rev"])
+        if co["target_fp"]:
+            sets["visual_fp_applied"] = co["target_fp"]
+        for col, key in (("visual_want", "want"), ("visual_rel", "rel")):
+            if key in vis:
+                sets[col] = vis[key] or None
+        for key in ("fp_parts", "folded", "partspecs"):
+            if key in vis:
+                visual[key] = vis[key]
+    flags = list(row["flags"] or [])
+    if staged and staged.get("asset_id"):
+        asset = pv.pdf_asset(conn, staged["asset_id"])
+        dwg = sets.get("drawing") or staged.get("dwg") or row["drawing"]
+        rev = str(staged["rev"]) if staged.get("rev") not in (None, "") else (sets.get("rev") or row["rev"])
+        first, count, old = pv.replace_drawing_pages(
+            conn, pid, asset, dwg=dwg, rev=rev, co_id=int(co["id"]), void_by=w.user_id
+        )
+        for i in range(max(len(old), count)):
+            o = old[i] if i < len(old) else None
+            new = i < count
+            conn.execute(
+                "INSERT INTO co_items (co_id, old_page, old_dwg, old_rev, new_page, new_dwg, new_rev, added)"
+                " VALUES (%s, %s, %s, %s, %s, %s, %s, %s)",
+                (
+                    co["id"],
+                    o["n"] if o else None,
+                    o["dwg"] if o else None,
+                    o["rev"] if o else None,
+                    first + i if new else None,
+                    dwg if new else None,
+                    rev if new else None,
+                    o is None,
+                ),
+            )
+        sets["drawing_state"] = "attached"
+        flags = pv.flags_without(flags, (*packets.DRAWING_FLAGS, "hand_drawing"))
+        visual["rev_confirmed"] = True
+        visual["vault"] = {
+            "asset_id": str(asset["id"]),
+            "name": staged.get("name"),
+            "rev": rev,
+            "at": pv.now_iso(),
+            "co_no": co["co_no"],
+        }
+        visual.pop("manual_dwg", None)
+        notes.append(f"drawing {dwg} rev {rev}, {_plural(count, 'page')}; {_plural(len(old), 'old page')} voided")
+    sets["flags"] = flags
+    sets["visual"] = Jsonb(visual)
+    cols = ", ".join(f"{c} = %s" + ("::drawing_state" if c == "drawing_state" else "") for c in sets)
+    conn.execute(f"UPDATE packets SET {cols} WHERE id = %s", (*sets.values(), pid))  # noqa: S608 (fixed names)
+    w.changed(*(c for c in sets if c not in ("flags", "visual")), "change_orders")
+    return "; ".join(notes) or None
+
+
 @router.post("/{packet_id}/change-orders/{no}/approve", dependencies=WRITE)
 def approve_co(
     packet_id: int,
@@ -1401,9 +1586,19 @@ def approve_co(
     p: Principal = Depends(current_principal),  # noqa: B008
     d: Db = Depends(get_db),  # noqa: B008
 ) -> dict[str, Any]:
+    """Approve a pending change order. A VISUAL or M-Files one is applied to the packet here; a minor one (target
+    hours or want date only) needs no department verification."""
     pp = _pp(d, p)
     conn = d.conn
-    content = {"action": str(A.CO_APPROVE), "packet_id": packet_id, "co_no": no}
+    pre = packets.one(
+        conn,
+        "SELECT origin::text AS origin, staged, target_fp FROM change_orders WHERE packet_id = %s AND co_no = %s",
+        (packet_id, no),
+    )
+    content: dict[str, Any] = {"action": str(A.CO_APPROVE), "packet_id": packet_id, "co_no": no}
+    if pre is not None and pre["origin"] in FEED_ORIGINS:
+        content["target_fp"] = pre["target_fp"]
+        content["staged_sha256"] = pre["staged"].get("sha256") if isinstance(pre["staged"], dict) else None
     with packet_write(
         d,
         p,
@@ -1417,6 +1612,12 @@ def approve_co(
     ) as w:
         co = _co(conn, packet_id, no)
         _co_state(co, "pending", "approved")
+        note: str | None = None
+        if co["origin"] in FEED_ORIGINS:
+            vis = co["vis"] if isinstance(co["vis"], dict) else {}
+            if vis.get("dwg_change") is True and not co["staged"]:
+                raise ApiError(409, "co_not_staged", MSG_CO_NOT_STAGED, extra={"waiting": co["waiting"]})
+            note = _apply_feed_co(w, co)
         update_row(
             conn,
             "change_orders",
@@ -1433,7 +1634,7 @@ def approve_co(
             for code in co_touched(conn, w.packet):
                 conn.execute("INSERT INTO co_required_depts (co_id, dept_code) VALUES (%s, %s)", (co["id"], code))
         _co_log_detail(w, no)
-        w.log(f"Change order {no} approved")
+        w.log(f"Change order {no} approved", note)
     return _read(d, pp, packet_id)
 
 

server/packetstation/api/packet_visual.py

@@ -0,0 +1,580 @@
+"""VISUAL packets by hand: approve the M-Files rev, upload the drawing or the BOM on a flagged draft, and go back to
+VISUAL's BOM.
+
+Every write goes through domain.pipeline.packet_write with its policy action and a signature. The hand uploads are
+for a VISUAL draft that is flagged (or already has a hand upload): Clerical checks it and sends it on; auto-release
+never takes it because its flags are not empty. Every answer is the full read model.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+from collections.abc import Mapping
+from datetime import UTC, datetime
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request
+from psycopg.types.json import Jsonb
+from pydantic import BaseModel, Field
+from starlette.concurrency import run_in_threadpool
+from starlette.datastructures import UploadFile
+
+from .. import policy
+from ..domain import assets as asset_store
+from ..domain import handbom, packets
+from ..domain.pipeline import SignBody, Write, next_counter, packet_write
+from ..text import LIMITS, clean_opt, reason_ok
+from .deps import Db, Principal, current_principal, get_db
+from .errors import ApiError
+from .packets import WRITE
+
+router = APIRouter(prefix="/api/packets", tags=["packet-visual"])
+
+A = policy.Action
+
+# A hand BOM file is a small sheet. The multipart overhead on top is allowed before the body is refused unread.
+BOM_MAX_BYTES = 5 * 1024 * 1024
+FORM_SLACK = 64 * 1024
+
+MSG_NOT_FLAGGED = (
+    "A drawing or BOM is uploaded by hand only on a flagged VISUAL draft. This one is not flagged: send it on as it is."
+)
+MSG_NOT_VISUAL = "This is not a packet made from VISUAL."
+MSG_DRAFT_ONLY = "Only a draft takes a hand upload. Once it leaves Clerical, changes come in as change orders."
+
+
+# ---------------------------------------------------------------- bodies
+
+
+class RevApproveBody(BaseModel):
+    reason: str | None = Field(default=None, max_length=2000)
+    sign: SignBody | None = None
+    it_reason: str | None = Field(default=None, max_length=1000)
+
+
+class HandDrawingBody(BaseModel):
+    asset_id: uuid.UUID
+    reason: str | None = Field(default=None, max_length=2000)
+    sign: SignBody | None = None
+    it_reason: str | None = Field(default=None, max_length=1000)
+
+
+class HandBomOffBody(BaseModel):
+    sign: SignBody | None = None
+    it_reason: str | None = Field(default=None, max_length=1000)
+
+
+# ---------------------------------------------------------------- shared helpers
+
+
+def _pp(d: Db, p: Principal) -> policy.Principal:
+    return packets.policy_principal_for(d.conn, p)
+
+
+def _read(d: Db, pp: policy.Principal, packet_id: int) -> dict[str, Any]:
+    return packets.read_packet(d.conn, packet_id, pp)
+
+
+def now_iso() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+def who_of(w: Write) -> dict[str, Any]:
+    return {"by": str(w.user_id), "by_name": w.principal.user.get("display_name")}
+
+
+def hand_flagged(row: Mapping[str, Any]) -> bool:
+    """The hand-upload guard: a VISUAL draft that is flagged, still needs its drawing, or already has a hand upload."""
+    if row["source_kind"] != "visual" or row["status"] != "draft":
+        return False
+    visual = row["visual"] or {}
+    return bool(
+        list(row["flags"] or [])
+        or row["drawing_state"] in packets.DRAWING_STATES_FLAGGED
+        or visual.get("manual_dwg")
+        or visual.get("manual_bom")
+    )
+
+
+def _hand_guard(w: Write) -> None:
+    row = w.packet
+    if row["source_kind"] != "visual":
+        raise ApiError(409, "not_visual", MSG_NOT_VISUAL)
+    if row["status"] != "draft":
+        raise ApiError(403, "wrong_status", MSG_DRAFT_ONLY)
+    if not hand_flagged(row):
+        raise ApiError(409, "not_flagged", MSG_NOT_FLAGGED)
+
+
+def pdf_asset(conn: Any, asset_id: Any) -> dict[str, Any]:
+    asset = asset_store.get(conn, asset_id)
+    if asset is None:
+        raise ApiError(404, "not_found", "No such file.")
+    if asset["mime"] != asset_store.PDF or not asset["pages"]:
+        raise ApiError(422, "not_pdf", "Attach a PDF drawing.")
+    return asset
+
+
+def flags_without(flags: Any, drop: tuple[str, ...], add: tuple[str, ...] = ()) -> list[str]:
+    out = [f for f in (flags or []) if f not in drop]
+    for f in add:
+        if f not in out:
+            out.append(f)
+    return out
+
+
+def replace_drawing_pages(
+    conn: Any,
+    packet_id: int,
+    asset: Mapping[str, Any],
+    *,
+    dwg: str | None,
+    rev: str | None,
+    co_id: int | None = None,
+    void_by: Any = None,
+) -> tuple[int, int, list[dict[str, Any]]]:
+    """Put an asset's pages on the packet as its drawing.
+
+    Without `co_id` (a draft) the current drawing pages are deleted (the assets stay). With `co_id` (a change order
+    approval) they are voided and point at the first new page. Returns (first new page, page count, old pages).
+    """
+    old = packets.rows(
+        conn,
+        "SELECT * FROM packet_pages WHERE packet_id = %s AND type = 'drawing' AND void_at IS NULL ORDER BY n",
+        (packet_id,),
+    )
+    count = int(asset["pages"] or 0)
+    if count < 1:
+        raise ApiError(422, "not_pdf", "That drawing has no pages.")
+    first = next_counter(conn, packet_id, "page", count)
+    if co_id is None:
+        conn.execute(
+            "DELETE FROM packet_pages WHERE packet_id = %s AND type = 'drawing' AND void_at IS NULL", (packet_id,)
+        )
+    elif old:
+        conn.execute(
+            "UPDATE packet_pages SET void_co_id = %s, void_at = now(), void_by = %s, void_new_n = %s,"
+            " row_version = row_version + 1 WHERE id = ANY(%s)",
+            (co_id, void_by, first, [pg["id"] for pg in old]),
+        )
+    for i in range(count):
+        conn.execute(
+            "INSERT INTO packet_pages (packet_id, n, type, asset_id, pn, dwg, rev, co_id)"
+            " VALUES (%s, %s, 'drawing', %s, %s, %s, %s, %s)",
+            (packet_id, first + i, asset["id"], i + 1, dwg, rev, co_id),
+        )
+    return first, count, old
+
+
+def set_visual(conn: Any, packet_id: int, visual: Mapping[str, Any]) -> None:
+    conn.execute("UPDATE packets SET visual = %s WHERE id = %s", (Jsonb(dict(visual)), packet_id))
+
+
+def _sha(asset: Mapping[str, Any]) -> str:
+    return bytes(asset["sha256"]).hex()
+
+
+# ---------------------------------------------------------------- approve the M-Files rev
+
+
+@router.post("/{packet_id}/rev-approve", dependencies=WRITE)
+def rev_approve(
+    packet_id: int,
+    body: RevApproveBody,
+    request: Request,
+    p: Principal = Depends(current_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """A Clerical Supervisor approves the rev M-Files has in place of the rev VISUAL calls, for this draft only."""
+    conn = d.conn
+    reason = reason_ok(body.reason)
+    pp = _pp(d, p)
+    row = packets.load_row(conn, packet_id)
+    staged = packets.staged_lookup(conn, row["drawing"]) if row else None
+    content = {
+        "action": str(A.REV_APPROVE),
+        "packet_id": packet_id,
+        "drawing": row["drawing"] if row else None,
+        "visual_rev": row["rev"] if row else None,
+        "rev": staged["rev"] if staged else None,
+        "sha256": bytes(staged["sha256"]).hex() if staged and staged["sha256"] else None,
+        "reason": reason,
+    }
+    with packet_write(
+        d,
+        p,
+        packet_id,
+        A.REV_APPROVE,
+        request=request,
+        extra={"reason": body.reason},
+        it_reason=body.it_reason,
+        sign=body.sign,
+        sign_content=content,
+        policy_principal=pp,
+    ) as w:
+        if w.packet["source_kind"] != "visual":
+            raise ApiError(409, "not_visual", MSG_NOT_VISUAL)
+        lk = packets.staged_lookup(conn, w.packet["drawing"])
+        if lk is None:
+            raise ApiError(
+                409, "nothing_staged", "No M-Files copy of this drawing is ready yet. Try after the next run."
+            )
+        asset = asset_store.get(conn, lk["asset_id"])
+        if asset is None or not asset["pages"]:
+            raise ApiError(
+                409, "nothing_staged", "The M-Files copy of this drawing is missing. Try after the next run."
+            )
+        rev = str(lk["rev"])
+        vis_rev = w.packet["rev"] or ""
+        replace_drawing_pages(conn, packet_id, asset, dwg=w.packet["drawing"], rev=rev)
+        visual = dict(w.packet["visual"] or {})
+        visual["rev_ok"] = {"rev": rev, "vis_rev": vis_rev, **who_of(w), "at": now_iso(), "reason": reason}
+        visual["rev_confirmed"] = True
+        visual["vault"] = {
+            "asset_id": str(asset["id"]),
+            "name": lk["name"],
+            "rev": rev,
+            "note": lk["note"],
+            "at": now_iso(),
+        }
+        visual.pop("manual_dwg", None)
+        set_visual(conn, packet_id, visual)
+        conn.execute(
+            "UPDATE packets SET drawing_state = 'attached', flags = %s WHERE id = %s",
+            (flags_without(w.packet["flags"], (*packets.DRAWING_FLAGS, "hand_drawing")), packet_id),
+        )
+        w.changed("pages", "drawing_state", "flags", "visual")
+        w.detail.update(asset_id=str(asset["id"]), rev=rev, vis_rev=vis_rev)
+        w.log("Different rev approved by hand", f"rev {rev} in place of VISUAL rev {vis_rev or '-'} · {reason}")
+    return _read(d, pp, packet_id)
+
+
+# ---------------------------------------------------------------- hand drawing
+
+
+@router.post("/{packet_id}/hand/drawing", dependencies=WRITE)
+def hand_drawing(
+    packet_id: int,
+    body: HandDrawingBody,
+    request: Request,
+    p: Principal = Depends(current_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Clerical uploads the drawing by hand on a flagged VISUAL draft. It replaces the drawing pages."""
+    conn = d.conn
+    reason = clean_opt(body.reason, LIMITS["reason"])
+    if reason is None:
+        raise ApiError(400, "reason_required", "Say why the drawing is uploaded by hand.")
+    asset = pdf_asset(conn, body.asset_id)
+    pp = _pp(d, p)
+    content = {
+        "action": "hand_drawing",
+        "packet_id": packet_id,
+        "asset_id": str(asset["id"]),
+        "sha256": _sha(asset),
+        "pages": asset["pages"],
+        "reason": reason,
+    }
+    with packet_write(
+        d,
+        p,
+        packet_id,
+        A.EDIT_DOCUMENT_SET,
+        request=request,
+        it_reason=body.it_reason,
+        sign=body.sign,
+        sign_content=content,
+        require_sign=True,
+        policy_principal=pp,
+    ) as w:
+        _hand_guard(w)
+        vis_rev = w.packet["rev"] or ""
+        first, count, _ = replace_drawing_pages(conn, packet_id, asset, dwg=w.packet["drawing"], rev=w.packet["rev"])
+        name = asset["original_name"] or "drawing.pdf"
+        visual = dict(w.packet["visual"] or {})
+        visual["manual_dwg"] = {
+            "asset_id": str(asset["id"]),
+            "name": name,
+            "reason": reason,
+            **who_of(w),
+            "at": now_iso(),
+            "vis_rev": vis_rev,
+        }
+        visual["rev_confirmed"] = False
+        set_visual(conn, packet_id, visual)
+        conn.execute(
+            "UPDATE packets SET drawing_state = 'hand', flags = %s WHERE id = %s",
+            (flags_without(w.packet["flags"], packets.DRAWING_FLAGS, ("hand_drawing",)), packet_id),
+        )
+        w.changed("pages", "drawing_state", "flags", "visual")
+        w.detail.update(asset_id=str(asset["id"]), pages=count, first_page=first)
+        w.log("Drawing uploaded by hand", f"{name}, {count} page{'s' if count != 1 else ''} · {reason}")
+    return _read(d, pp, packet_id)
+
+
+# ---------------------------------------------------------------- hand BOM
+
+
+def _insert_hand_bom(conn: Any, packet_id: int, lines: list[dict[str, Any]]) -> None:
+    conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,))
+    for i, b in enumerate(lines):
+        conn.execute(
+            "INSERT INTO bom_lines (packet_id, item, qty, part_id, description, dwg_ref, cl_item, length, material,"
+            " src, sort) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'hand', %s)",
+            (
+                packet_id,
+                b["item"],
+                b["qty"],
+                b["part_id"],
+                b["description"],
+                b["dwg_ref"],
+                b["cl_item"],
+                b["length"],
+                b["material"],
+                i,
+            ),
+        )
+
+
+def _hand_bom_write(
+    d: Db,
+    p: Principal,
+    request: Request,
+    packet_id: int,
+    *,
+    data: bytes,
+    name: str | None,
+    reason: str,
+    sign: SignBody,
+    it_reason: str | None,
+) -> dict[str, Any]:
+    conn = d.conn
+    try:
+        lines = handbom.parse(data, name)
+    except handbom.BadFile as e:
+        raise ApiError(422, "bad_file", str(e)) from None
+    pp = _pp(d, p)
+    sha = hashlib.sha256(data).hexdigest()
+    fname = clean_opt(name, LIMITS["file_name"]) or "bom"
+    content = {
+        "action": "hand_bom",
+        "packet_id": packet_id,
+        "file": fname,
+        "sha256": sha,
+        "lines": len(lines),
+        "reason": reason,
+    }
+    with packet_write(
+        d,
+        p,
+        packet_id,
+        A.EDIT_BOM,
+        request=request,
+        it_reason=it_reason,
+        sign=sign,
+        sign_content=content,
+        require_sign=True,
+        policy_principal=pp,
+    ) as w:
+        _hand_guard(w)
+        _insert_hand_bom(conn, packet_id, lines)
+        visual = dict(w.packet["visual"] or {})
+        visual["manual_bom"] = {
+            "name": fname,
+            "reason": reason,
+            **who_of(w),
+            "at": now_iso(),
+            "lines": len(lines),
+            "sha256": sha,
+        }
+        set_visual(conn, packet_id, visual)
+        conn.execute(
+            "UPDATE packets SET flags = %s WHERE id = %s",
+            (flags_without(w.packet["flags"], (), ("hand_bom",)), packet_id),
+        )
+        w.changed("bom", "flags", "visual")
+        w.detail.update(file=fname, sha256=sha, lines=len(lines))
+        n = len(lines)
+        w.log("BOM uploaded by hand", f"{fname} · {n} line{'s' if n != 1 else ''} · {reason}")
+    return _read(d, pp, packet_id)
+
+
+@router.post("/{packet_id}/hand/bom")
+async def hand_bom(
+    packet_id: int,
+    request: Request,
+    p: Principal = Depends(current_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Clerical uploads the BOM by hand (multipart: file, reason, sign_factor, sign_secret, it_reason)."""
+    length = request.headers.get("content-length")
+    if length is not None and length.isdigit() and int(length) > BOM_MAX_BYTES + FORM_SLACK:
+        raise ApiError(413, "too_large", f"The BOM file is over the {BOM_MAX_BYTES // (1024 * 1024)} MB limit.")
+    form = await request.form(max_files=1, max_fields=8)
+    try:
+        file = form.get("file")
+        if not isinstance(file, UploadFile):
+            raise ApiError(422, "invalid_request", "Send the BOM file in the 'file' field.")
+
+        def field(name: str, cap: int) -> str | None:
+            v = form.get(name)
+            return v[:cap] if isinstance(v, str) else None
+
+        reason = clean_opt(field("reason", 2000), LIMITS["reason"])
+        if reason is None:
+            raise ApiError(400, "reason_required", "Say why the BOM is uploaded by hand.")
+        factor, secret = field("sign_factor", 20), field("sign_secret", 1024)
+        if not factor or secret is None:
+            raise ApiError(
+                400, "signature_required", "This step needs your signature.", extra={"meaning": "Edit the BOM"}
+            )
+        if factor not in ("pin", "password"):
+            raise ApiError(422, "invalid_request", "sign_factor is pin or password.")
+        sign = SignBody(factor=factor, secret=secret)  # type: ignore[arg-type]
+        data = await file.read(BOM_MAX_BYTES + 1)
+        if len(data) > BOM_MAX_BYTES:
+            raise ApiError(413, "too_large", f"The BOM file is over the {BOM_MAX_BYTES // (1024 * 1024)} MB limit.")
+        name = file.filename
+        it_reason = field("it_reason", 1000)
+    finally:
+        await form.close()
+    return await run_in_threadpool(
+        lambda: _hand_bom_write(
+            d, p, request, packet_id, data=data, name=name, reason=reason, sign=sign, it_reason=it_reason
+        )
+    )
+
+
+@router.delete("/{packet_id}/hand/bom", dependencies=WRITE)
+def hand_bom_off(
+    packet_id: int,
+    request: Request,
+    body: HandBomOffBody | None = None,
+    p: Principal = Depends(current_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Drop the hand BOM and use VISUAL's BOM again, rebuilt from the cached materials."""
+    body = body or HandBomOffBody()
+    conn = d.conn
+    pp = _pp(d, p)
+    with packet_write(
+        d,
+        p,
+        packet_id,
+        A.EDIT_BOM,
+        request=request,
+        it_reason=body.it_reason,
+        sign=body.sign,
+        sign_content={"action": "hand_bom_off", "packet_id": packet_id},
+        require_sign=body.sign is not None,
+        policy_principal=pp,
+    ) as w:
+        if w.packet["source_kind"] != "visual":
+            raise ApiError(409, "not_visual", MSG_NOT_VISUAL)
+        if w.packet["status"] != "draft":
+            raise ApiError(403, "wrong_status", MSG_DRAFT_ONLY)
+        visual = dict(w.packet["visual"] or {})
+        old = visual.pop("manual_bom", None)
+        if not old:
+            raise ApiError(409, "no_hand_bom", "This packet has no BOM uploaded by hand.")
+        from ..feed import materials as feed_materials  # agent B's module, imported where it is used
+
+        lines = feed_materials.bom_for_packet(conn, packet_id)
+        conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,))
+        feed_materials.insert_bom(conn, packet_id, lines)
+        set_visual(conn, packet_id, visual)
+        conn.execute(
+            "UPDATE packets SET flags = %s WHERE id = %s",
+            (flags_without(w.packet["flags"], ("hand_bom",)), packet_id),
+        )
+        w.changed("bom", "flags", "visual")
+        n = len(lines)
+        name = old.get("name") if isinstance(old, dict) else None
+        note = f"{n} line{'s' if n != 1 else ''} from VISUAL" if n else "VISUAL has no materials for it yet"
+        w.log("Hand-uploaded BOM taken off", " · ".join(x for x in (name, note) if x))
+    return _read(d, pp, packet_id)
+
+
+# ---------------------------------------------------------------- VISUAL BOM lines (change order approval)
+
+
+def merge_visual_bom(conn: Any, packet_id: int, raw_lines: list[Any]) -> dict[str, int]:
+    """Bring a packet's BOM to VISUAL's lines (change order `vis.bom`, dicts or BomLine objects) without losing work.
+
+    When nothing has been recorded on the lines (no pulled mark, heat, MTR or short) the BOM is replaced through
+    feed.materials.insert_bom, exactly as the feed writes it. Otherwise lines are matched on vkey and updated in
+    place, so pulled marks, heats and shorts stay. New lines are added with the same encoding (item numbers from
+    feed.materials.db_items, the label in cl_item, cons Y/N, sub Y or NULL). Lines VISUAL dropped are deleted, or
+    marked n/a when they were pulled or have a short.
+    """
+    from ..feed.materials import BomLine, db_items, insert_bom
+
+    lines = [ln if isinstance(ln, BomLine) else BomLine.from_dict(ln) for ln in raw_lines]
+    counts = {"added": 0, "updated": 0, "removed": 0, "na": 0}
+    existing = packets.rows(conn, "SELECT * FROM bom_lines WHERE packet_id = %s ORDER BY item", (packet_id,))
+    worked = (
+        any(e["pulled"] or e["heat"] or e["mtr_asset_id"] for e in existing)
+        or conn.execute("SELECT 1 FROM bom_shorts WHERE packet_id = %s LIMIT 1", (packet_id,)).fetchone()
+    )
+    if not worked:
+        # Nothing recorded on the lines yet: write them exactly as the feed does.
+        conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,))
+        insert_bom(conn, packet_id, lines)
+        counts["added"] = len(lines)
+        counts["removed"] = len(existing)
+        return counts
+    by_vkey = {e["vkey"]: e for e in existing if e["vkey"]}
+    used = {int(e["item"]) for e in existing}
+    wanted = db_items(lines)
+    kept: set[int] = set()
+    for sort, (b, want_item) in enumerate(zip(lines, wanted, strict=True)):
+        values = (
+            b.qty,
+            b.part_id[:60] or None,
+            b.desc[:240] or None,
+            b.loc[:120] or None,
+            b.whs or None,
+            b.comm or None,
+            "Y" if b.cons else "N",
+            "Y" if b.sub else None,
+            b.na,
+            b.grp,
+            b.issued,
+            sort,
+        )
+        match = by_vkey.get(b.vkey) if b.vkey else None
+        if match is not None and int(match["id"]) not in kept:
+            kept.add(int(match["id"]))
+            conn.execute(
+                "UPDATE bom_lines SET qty = %s, part_id = %s, description = %s, loc = %s, whs = %s, comm = %s,"
+                " cons = %s, sub = %s, na = %s, grp = %s, issued = coalesce(%s, issued), sort = %s, cl_item = %s,"
+                " row_version = row_version + 1 WHERE id = %s",
+                (*values, b.item, match["id"]),
+            )
+            counts["updated"] += 1
+            continue
+        item = want_item
+        while item in used:
+            item += 1
+        used.add(item)
+        new_id = conn.execute(
+            "INSERT INTO bom_lines (packet_id, item, cl_item, qty, part_id, description, loc, whs, comm, cons, sub, na,"
+            " grp, issued, sort, vkey, src) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,"
+            " 'visual') RETURNING id",
+            (packet_id, item, b.item, *values, b.vkey),
+        ).fetchone()[0]
+        kept.add(int(new_id))
+        counts["added"] += 1
+    for e in existing:
+        if int(e["id"]) in kept:
+            continue
+        busy = e["pulled"] or conn.execute("SELECT 1 FROM bom_shorts WHERE bom_line_id = %s", (e["id"],)).fetchone()
+        if busy:
+            if not e["na"]:
+                conn.execute("UPDATE bom_lines SET na = true, row_version = row_version + 1 WHERE id = %s", (e["id"],))
+                counts["na"] += 1
+        else:
+            conn.execute("DELETE FROM bom_lines WHERE id = %s", (e["id"],))
+            counts["removed"] += 1
+    return counts

server/packetstation/api/packets.py

@@ -299,6 +299,7 @@ def list_packets(
     project_id: int | None = Query(None),
     q: str = Query("", max_length=200),
     flag: str = Query("", max_length=20),
+    co: str = Query("", max_length=20),
     limit: int = Query(500, ge=1, le=2000),
     removed: int = Query(0, ge=0, le=1),
 ) -> dict[str, Any]:
@@ -309,6 +310,7 @@ def list_packets(
         "project_id": project_id,
         "q": clean_text(q, 200),
         "flag": flag,
+        "co": co.strip().lower() or None,
         "limit": limit,
         "removed": bool(removed) and p.is_admin,
     }

server/packetstation/cli.py

@@ -1,4 +1,4 @@
-"""`packetstation` command line: app, worker, migrate, doctor, version, healthcheck."""
+"""`packetstation` command line: app, worker, job, fixtures, migrate, doctor, version, healthcheck."""
 
 from __future__ import annotations
 
@@ -55,7 +55,52 @@ def cmd_worker(_: argparse.Namespace) -> int:
     _setup_logging(s.log_level)
     guard.startup_checks(s)
     beat = Path(os.environ.get("WORKER_BEAT_FILE") or s.worker_beat_file)
-    return worker.run(beat, s.ps_build)
+    return worker.run(beat, s.ps_build, settings=s)
+
+
+def cmd_job_run(a: argparse.Namespace) -> int:
+    """Run one job once in the foreground (trigger `cli`) and print its job_runs row as JSON."""
+    from . import guard
+    from .worker.jobs import load_jobs
+    from .worker.runs import run_job
+    from .worker.scheduler import Scheduler
+
+    s = _settings()
+    _setup_logging(s.log_level)
+    guard.startup_checks(s)
+    jobs = load_jobs()
+    if a.name not in jobs:
+        print(f"packetstation job run: no job {a.name!r}; jobs: {', '.join(sorted(jobs))}", file=sys.stderr)
+        return 2
+    sched = Scheduler(s, jobs=jobs)
+    try:
+        row = run_job(sched, a.name, "cli")
+    except Exception as e:
+        from .worker.redaction import redact
+
+        print(f"packetstation job run: {redact(f'{type(e).__name__}: {e}')}", file=sys.stderr)
+        return 1
+    print(json.dumps(row, indent=2, default=str))
+    return 1 if row.get("status") == "failed" else 0
+
+
+def cmd_fixtures_use(a: argparse.Namespace) -> int:
+    """Point the fixture sources at state N: writes DIR/state. Never on STACK=prod."""
+    if os.environ.get("STACK", "").strip().lower() == "prod":
+        print("packetstation fixtures: refused on STACK=prod", file=sys.stderr)
+        return 2
+    if a.state < 1:
+        print("packetstation fixtures use: the state is a number from 1", file=sys.stderr)
+        return 2
+    d = Path(a.dir or os.environ.get("FIXTURE_DIR") or "/fixtures")
+    if not (d / f"feed.{a.state}.json").is_file():
+        print(f"packetstation fixtures use: {d / f'feed.{a.state}.json'} not found", file=sys.stderr)
+        return 1
+    tmp = d / "state.tmp"
+    tmp.write_text(str(a.state) + "\n", encoding="utf-8")
+    tmp.replace(d / "state")
+    print(f"fixture state {a.state} in {d}")
+    return 0
 
 
 def cmd_migrate(a: argparse.Namespace) -> int:
@@ -148,6 +193,17 @@ def build_parser() -> argparse.ArgumentParser:
     sub = ap.add_subparsers(dest="cmd", required=True)
     sub.add_parser("app", help="Run the web app (uvicorn on 0.0.0.0:8000)").set_defaults(func=cmd_app)
     sub.add_parser("worker", help="Run the background worker").set_defaults(func=cmd_worker)
+    j = sub.add_parser("job", help="Worker jobs")
+    jsub = j.add_subparsers(dest="job_cmd", required=True)
+    jr = jsub.add_parser("run", help="Run one job once now and print its run as JSON (exit 1 when it failed)")
+    jr.add_argument("name", help="visual-feed, drawing-lookup or housekeeping")
+    jr.set_defaults(func=cmd_job_run)
+    f = sub.add_parser("fixtures", help="Synthetic feed fixtures (CI and tests)")
+    fsub = f.add_subparsers(dest="fixtures_cmd", required=True)
+    fu = fsub.add_parser("use", help="Point the fixture sources at state N (writes DIR/state)")
+    fu.add_argument("state", type=int)
+    fu.add_argument("--dir", help="The fixture directory (default FIXTURE_DIR or /fixtures)")
+    fu.set_defaults(func=cmd_fixtures_use)
     m = sub.add_parser("migrate", help="Apply pending SQL migrations as the migrate role")
     m.add_argument(
         "--bootstrap-roles",

server/packetstation/domain/assets.py

@@ -18,6 +18,7 @@ from pathlib import Path
 from typing import Any, BinaryIO
 
 from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
 
 from .. import audit
 from ..api.errors import ApiError
@@ -69,7 +70,35 @@ def path_of(settings: Settings, key: str) -> Path:
 
 
 def pdf_pages(path: Path) -> int:
-    """Page count through pypdf; 422 bad_pdf when it does not parse or has no pages."""
+    """Page count through pypdfium2, or pypdf when pypdfium2 is missing or cannot read the file.
+
+    422 bad_pdf when neither parses it or it has no pages.
+    """
+    n = _pages_pdfium(path)
+    if n is None:
+        n = _pages_pypdf(path)
+    if n < 1:
+        raise ApiError(422, "bad_pdf", "That PDF has no pages.")
+    return n
+
+
+def _pages_pdfium(path: Path) -> int | None:
+    try:
+        import pypdfium2
+    except ImportError:
+        return None
+    try:
+        doc = pypdfium2.PdfDocument(str(path))
+    except Exception as e:  # any pdfium failure falls back to pypdf
+        log.info("pypdfium2 could not read the PDF (%s); trying pypdf", type(e).__name__)
+        return None
+    try:
+        return len(doc)
+    finally:
+        doc.close()
+
+
+def _pages_pypdf(path: Path) -> int:
     from pypdf import PdfReader
     from pypdf.errors import PyPdfError
 
@@ -78,8 +107,6 @@ def pdf_pages(path: Path) -> int:
     except (PyPdfError, ValueError, KeyError, TypeError, OSError, IndexError, AttributeError) as e:
         log.info("upload is not a readable PDF: %s", type(e).__name__)
         raise ApiError(422, "bad_pdf", "That PDF could not be read.") from None
-    if n < 1:
-        raise ApiError(422, "bad_pdf", "That PDF has no pages.")
     return n
 
 
@@ -115,6 +142,40 @@ def _fsync_dir(path: Path) -> None:
         os.close(fd)
 
 
+def _existing(conn: Any, sha: bytes, size: int) -> dict[str, Any] | None:
+    """The stored asset with the same bytes (same sha256 and size), oldest first."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(
+            "SELECT * FROM assets WHERE sha256 = %s AND size = %s ORDER BY uploaded_at LIMIT 1", (sha, size)
+        ).fetchone()
+    return dict(r) if r else None
+
+
+def _place(settings: Settings, tmp: Path) -> tuple[uuid.UUID, str, Path]:
+    """Rename a fsynced file to ASSETS_DIR/<id[:2]>/<id>, then fsync the directory. Returns (id, key, path)."""
+    aid = uuid.uuid4()
+    key = storage_key(aid)
+    final = path_of(settings, key)
+    final.parent.mkdir(parents=True, exist_ok=True)
+    os.replace(tmp, final)
+    _fsync_dir(final.parent)
+    return aid, key, final
+
+
+def _insert(conn: Any, values: tuple[Any, ...]) -> dict[str, Any]:
+    """Insert one assets row from (id, sha256, size, mime, name, key, source, source_ref, pages, by, packet_id)."""
+    v = list(values)
+    if v[7] is not None and not isinstance(v[7], Jsonb):
+        v[7] = Jsonb(v[7])
+    with conn.cursor(row_factory=dict_row) as cur:
+        row = cur.execute(
+            "INSERT INTO assets (id, sha256, size, mime, original_name, storage_key, source, source_ref, pages,"
+            " uploaded_by, packet_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING *",
+            v,
+        ).fetchone()
+    return dict(row)
+
+
 def store_upload(
     conn: Any,
     settings: Settings,
@@ -166,29 +227,15 @@ def store_upload(
         if mime is None or size == 0:
             raise ApiError(415, "unsupported_type", "Only PDF, PNG and JPEG files can be stored.")
         sha = digest.digest()
-        with conn.cursor(row_factory=dict_row) as cur:
-            existing = cur.execute(
-                "SELECT * FROM assets WHERE sha256 = %s AND size = %s ORDER BY uploaded_at LIMIT 1", (sha, size)
-            ).fetchone()
+        existing = _existing(conn, sha, size)
         if existing is not None:
-            return public(dict(existing))
+            return public(existing)
         pages = pdf_pages(tmp) if mime == PDF else None
-        aid = uuid.uuid4()
-        key = storage_key(aid)
-        final = path_of(settings, key)
-        final.parent.mkdir(parents=True, exist_ok=True)
-        os.replace(tmp, final)
+        aid, key, placed = _place(settings, tmp)
         keep = True
-        placed = final
-        _fsync_dir(final.parent)
         name = safe_name(original_name)
         with conn.transaction():
-            with conn.cursor(row_factory=dict_row) as cur:
-                row = cur.execute(
-                    "INSERT INTO assets (id, sha256, size, mime, original_name, storage_key, source, source_ref, pages,"
-                    " uploaded_by, packet_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING *",
-                    (aid, sha, size, mime, name, key, source, source_ref, pages, user, packet_id),
-                ).fetchone()
+            row = _insert(conn, (aid, sha, size, mime, name, key, source, source_ref, pages, user, packet_id))
             audit.append(
                 conn,
                 action="asset.uploaded",
@@ -213,6 +260,69 @@ def store_upload(
                 tmp.unlink()
 
 
+def store_file(
+    conn: Any,
+    settings: Settings,
+    path: Path,
+    *,
+    mime: str = PDF,
+    original_name: Any,
+    source: str,
+    source_ref: Any = None,
+    uploaded_by: Any = None,
+    packet_id: int | None = None,
+) -> dict[str, Any]:
+    """Store a file the server wrote itself (the worker's M-Files download). Returns the full assets row.
+
+    The file is consumed: renamed into ASSETS_DIR when its bytes are new, deleted when the same bytes are already
+    stored (same sha256 and size; that row comes back). Write it under ASSETS_DIR (ASSETS_DIR/tmp) so the rename
+    stays on one file system. It is hashed in chunks, its first bytes must match `mime`, and it is fsynced before
+    the rename. The row carries one extra key, `reused`. No audit row: the caller writes its own.
+    Raises 413 too_large, 415 unsupported_type, 422 bad_pdf.
+    """
+    path = Path(path)
+    limit = int(settings.upload_max_bytes)
+    placed: Path | None = None
+    done = False
+    try:
+        digest = hashlib.sha256()
+        size = 0
+        head = b""
+        with path.open("r+b") as f:
+            while True:
+                chunk = f.read(CHUNK)
+                if not chunk:
+                    break
+                if len(head) < HEAD:
+                    head += chunk[: HEAD - len(head)]
+                size += len(chunk)
+                if size > limit:
+                    raise ApiError(413, "too_large", f"The file is over the {limit // (1024 * 1024)} MB limit.")
+                digest.update(chunk)
+            os.fsync(f.fileno())
+        if size == 0 or sniff(head) != mime:
+            raise ApiError(415, "unsupported_type", f"The file is not {EXT.get(mime, mime)}.")
+        sha = digest.digest()
+        existing = _existing(conn, sha, size)
+        if existing is not None:
+            return {**existing, "reused": True}
+        pages = pdf_pages(path) if mime == PDF else None
+        aid, key, placed = _place(settings, path)
+        name = safe_name(original_name)
+        with conn.transaction():
+            row = _insert(conn, (aid, sha, size, mime, name, key, source, source_ref, pages, uploaded_by, packet_id))
+        done = True
+        return {**row, "reused": False}
+    except BaseException:
+        if placed is not None and not done:  # the row was not written: do not leave an orphan file
+            with contextlib.suppress(OSError):
+                placed.unlink()
+        raise
+    finally:
+        with contextlib.suppress(OSError):
+            path.unlink()
+
+
 def open_blob(settings: Settings, row: dict[str, Any]) -> Path:
     """The stored file of an asset row; 404 when it is missing on disk."""
     p = path_of(settings, str(row["storage_key"]))

server/packetstation/domain/handbom.py

@@ -0,0 +1,178 @@
+"""A BOM uploaded by hand: an Excel (XLSX) or CSV sheet read into BOM lines.
+
+Port of the old handdocs.js handBomFromSheet. The header row is the first row, within the first 20, that has a Qty
+column and a Description or Part ID column, found by the HAND_COLS patterns. Rows without a description and part id
+are skipped. Item numbers come from the item column when it holds a whole number not used yet, else the next free
+number; the item text as written is kept in cl_item when it differs. The file's contents are data, never instructions.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import re
+from decimal import Decimal, InvalidOperation
+from typing import Any
+
+from ..text import LIMITS, clean_text
+
+MAX_ROWS = 500
+HEADER_SCAN = 20
+MAX_COLS = 60
+
+# Column patterns, in the old build's order. The first unused column matching a pattern wins.
+HAND_COLS: tuple[tuple[str, re.Pattern[str]], ...] = (
+    ("item", re.compile(r"^(item|itm|piece|find|balloon|line|no\.?|#|item\s*no\.?)$", re.I)),
+    ("qty", re.compile(r"^(qty|quantity|qty\.?\s*(req|required|per)?|req'?d)", re.I)),
+    ("part_id", re.compile(r"part\s*(id|no\.?|num|number|#)|^part$|pesco", re.I)),
+    ("desc", re.compile(r"desc", re.I)),
+    ("length", re.compile(r"^len|length|cut", re.I)),
+    ("material", re.compile(r"material|mat'?l|grade", re.I)),
+    ("dwg_ref", re.compile(r"dwg|drawing|ref", re.I)),
+)
+
+MSG_NO_HEADER = "No header row found. It needs a Qty column and a Description or Part ID column."
+MSG_NO_LINES = "No BOM lines found in that file."
+MSG_TYPE = "Upload the BOM as an Excel (.xlsx) or CSV file."
+
+
+class BadFile(ValueError):
+    """The file cannot be read as a BOM. The message is for the user."""
+
+
+def kind_of(data: bytes, name: str | None) -> str:
+    """xlsx (a ZIP) or csv (text). Anything else is refused."""
+    if data[:4] == b"PK\x03\x04":
+        return "xlsx"
+    lower = (name or "").lower()
+    if data[:5] == b"%PDF-" or data[:8] == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1":
+        raise BadFile(MSG_TYPE)
+    if lower.endswith((".csv", ".txt")) or b"\x00" not in data[:4096]:
+        return "csv"
+    raise BadFile(MSG_TYPE)
+
+
+def _cell(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, float) and value.is_integer():
+        return str(int(value))
+    return str(value).strip()
+
+
+def _rows_xlsx(data: bytes) -> list[list[str]]:
+    try:
+        from openpyxl import load_workbook
+    except ImportError as e:  # pragma: no cover - openpyxl is a dependency
+        raise BadFile("The server cannot read Excel files right now. Use a CSV file.") from e
+    try:
+        wb = load_workbook(io.BytesIO(data), read_only=True, data_only=True)
+    except Exception as e:  # openpyxl raises many types on a bad file
+        raise BadFile("That Excel file could not be read.") from e
+    try:
+        ws = wb.worksheets[0] if wb.worksheets else None
+        if ws is None:
+            raise BadFile(MSG_NO_LINES)
+        out: list[list[str]] = []
+        for row in ws.iter_rows(values_only=True):
+            out.append([_cell(v) for v in row[:MAX_COLS]])
+            if len(out) > MAX_ROWS + HEADER_SCAN:
+                break
+        return out
+    finally:
+        wb.close()
+
+
+def _rows_csv(data: bytes) -> list[list[str]]:
+    try:
+        text = data.decode("utf-8-sig")
+    except UnicodeDecodeError:
+        text = data.decode("cp1252", errors="replace")
+    sample = text[:8192]
+    try:
+        dialect: Any = csv.Sniffer().sniff(sample, delimiters=",;\t|")
+    except csv.Error:
+        dialect = csv.excel
+    out: list[list[str]] = []
+    for row in csv.reader(io.StringIO(text), dialect):
+        out.append([_cell(v) for v in row[:MAX_COLS]])
+        if len(out) > MAX_ROWS + HEADER_SCAN:
+            break
+    return out
+
+
+def find_header(rows: list[list[str]]) -> tuple[int, dict[str, int]]:
+    """The header row index and the column map, or BadFile."""
+    for i, row in enumerate(rows[:HEADER_SCAN]):
+        m: dict[str, int] = {}
+        for key, rx in HAND_COLS:
+            for j, h in enumerate(row):
+                if h and rx.search(h) and j not in m.values():
+                    m[key] = j
+                    break
+        if "qty" in m and ("desc" in m or "part_id" in m):
+            return i, m
+    raise BadFile(MSG_NO_HEADER)
+
+
+def _qty(text: str) -> Decimal:
+    digits = re.sub(r"[^\d.\-]", "", text)
+    try:
+        q = Decimal(digits) if digits else Decimal(0)
+    except InvalidOperation:
+        return Decimal(0)
+    return q if q.is_finite() else Decimal(0)
+
+
+def lines_from_rows(rows: list[list[str]]) -> list[dict[str, Any]]:
+    hi, m = find_header(rows)
+
+    def cell(r: list[str], key: str) -> str:
+        j = m.get(key)
+        return r[j].strip() if j is not None and j < len(r) else ""
+
+    out: list[dict[str, Any]] = []
+    used: set[int] = set()
+    for r in rows[hi + 1 :]:
+        desc, pid, qty = cell(r, "desc"), cell(r, "part_id"), cell(r, "qty")
+        if not desc and not pid:
+            continue  # blank rows, totals, notes
+        raw_item = cell(r, "item")
+        item: int | None = None
+        if re.fullmatch(r"\d{1,5}", raw_item) and 1 <= int(raw_item) <= 100_000 and int(raw_item) not in used:
+            item = int(raw_item)
+        if item is None:
+            item = len(out) + 1
+            while item in used:
+                item += 1
+        used.add(item)
+        cl_item = clean_text(raw_item, LIMITS["part_id"]) or None if raw_item != str(item) else None
+        out.append(
+            {
+                "item": item,
+                "cl_item": cl_item,
+                "qty": _qty(qty),
+                "part_id": clean_text(pid, LIMITS["part_id"]) or None,
+                "description": clean_text(desc, LIMITS["description"]) or None,
+                "length": clean_text(cell(r, "length"), LIMITS["length"]) or None,
+                "material": clean_text(cell(r, "material"), LIMITS["material"]) or None,
+                "dwg_ref": clean_text(cell(r, "dwg_ref"), LIMITS["drawing"]) or None,
+            }
+        )
+        if len(out) >= MAX_ROWS:
+            break
+    if not out:
+        raise BadFile(MSG_NO_LINES)
+    return out
+
+
+def parse(data: bytes, name: str | None) -> list[dict[str, Any]]:
+    """BOM lines from an XLSX or CSV file: [{item, cl_item, qty, part_id, description, length, material, dwg_ref}].
+
+    Raises BadFile with the reason for the user.
+    """
+    if not data:
+        raise BadFile("The file is empty.")
+    kind = kind_of(data, name)
+    rows = _rows_xlsx(data) if kind == "xlsx" else _rows_csv(data)
+    return lines_from_rows(rows)

server/packetstation/domain/packets.py

@@ -7,6 +7,7 @@ boards from header columns, v_open_cos and the open short count only.
 
 from __future__ import annotations
 
+import re
 import uuid
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass, field
@@ -20,7 +21,7 @@ from ..api.errors import ApiError
 from . import checklists
 
 HAND_FLAG = "Made by hand - check it and send it on"
-DRAWING_NEEDED_FLAG = "Drawing needed - looking it up in the vault"
+DRAWING_NEEDED_FLAG = "Made from VISUAL - drawing not attached yet"
 DRAWING_STATES_ATTACHED = ("attached", "hand")
 
 # Keys left out of the sealed content: they change without the packet changing (who can do what, the counter,
@@ -37,6 +38,7 @@ SEAL_EXCLUDE = (
     "checklist_items",
     "qc_items",
     "flag",
+    "visual_state",
 )
 
 
@@ -67,19 +69,102 @@ def load_row(conn: Any, packet_id: int, *, lock: bool = False) -> dict[str, Any]
     return one(conn, sql, (packet_id,))
 
 
-def flag_of(status: str, source_kind: str, flags: Iterable[str] | None, drawing_state: str | None = None) -> str | None:
-    """The draft flag shown on cards and the packet page. The old packetFlag(), phase 2 part."""
+# Flags the drawing job owns; a hand drawing or an approved rev clears them.
+DRAWING_FLAGS = ("drawing_needed", "rev_mismatch", "not_released", "not_found", "lookup_failed", "bad_rev")
+# drawing_state values that mean the drawing still needs attention.
+DRAWING_STATES_FLAGGED = ("needed", "rev_mismatch", "not_released", "not_found", "lookup_failed")
+HAND_UPLOAD_FLAGS = ("hand_drawing", "hand_bom")
+
+# Flag kinds in display priority (the first one present gives the text).
+FLAG_PRIORITY = (
+    "hand_drawing",
+    "hand_bom",
+    "rev_mismatch",
+    "not_released",
+    "not_found",
+    "lookup_failed",
+    "bad_rev",
+    "drawing_needed",
+    "no_drawing",
+    "wo_gone",
+    "cancelled",
+)
+FLAG_TEXTS = {
+    "not_found": "Drawing not in M-Files - check the drawing number in VISUAL",
+    "bad_rev": "VISUAL rev is not a number - drawing not attached",
+    "lookup_failed": "M-Files lookup failed",
+    "drawing_needed": DRAWING_NEEDED_FLAG,
+    "no_drawing": "No drawing number in VISUAL - add it there or upload the drawing by hand",
+    "wo_gone": "Work order no longer in VISUAL",
+    "cancelled": "Work order cancelled in VISUAL",
+}
+# drawing_state -> flag kind, for a visual draft whose flags list does not name the problem.
+STATE_KINDS = {
+    "needed": "drawing_needed",
+    "rev_mismatch": "rev_mismatch",
+    "not_released": "not_released",
+    "not_found": "not_found",
+    "lookup_failed": "lookup_failed",
+}
+RX_FLAG_KIND = re.compile(r"^[a-z_]{1,30}$")
+
+
+def _appr_note(visual: Mapping[str, Any] | None) -> str:
+    appr = (visual or {}).get("appr") if isinstance(visual, Mapping) else None
+    if isinstance(appr, Mapping):
+        note = appr.get("note")
+        if isinstance(note, str) and note.strip():
+            return note.strip()
+    return ""
+
+
+def flag_of(
+    status: str,
+    source_kind: str,
+    flags: Iterable[str] | None,
+    drawing_state: str | None = None,
+    visual: Mapping[str, Any] | None = None,
+) -> str | None:
+    """The draft flag shown on cards and the packet page: the old packetFlag(), handFlag() and revMismatch() texts.
+
+    Only drafts carry a flag. A hand draft keeps the phase 2 text. A VISUAL draft shows its most important flag kind.
+    """
     if status != "draft":
         return None
     if source_kind == "hand":
         return HAND_FLAG
-    if "drawing_needed" in (flags or ()):
-        return DRAWING_NEEDED_FLAG
-    return None
+    kinds = list(flags or ())
+    if not kinds and drawing_state in STATE_KINDS:
+        kinds = [STATE_KINDS[drawing_state]]
+    if not kinds:
+        return None
+    dwg, bom = "hand_drawing" in kinds, "hand_bom" in kinds
+    if dwg or bom:
+        what = "Drawing and BOM" if dwg and bom else "Drawing" if dwg else "BOM"
+        return f"{what} uploaded by hand - check it and send it on"
+    note = _appr_note(visual)
+    for kind in FLAG_PRIORITY:
+        if kind not in kinds:
+            continue
+        if kind == "rev_mismatch":
+            return note or "Rev mismatch: VISUAL and M-Files show different revs"
+        if kind == "not_released":
+            return note or "Not released in M-Files yet"
+        if kind in ("lookup_failed", "bad_rev"):
+            # The drawing job writes the whole text ("M-Files lookup failed: ...", "VISUAL rev A is not a number;
+            # M-Files has rev 3 - not attached").
+            return note or FLAG_TEXTS[kind]
+        return FLAG_TEXTS[kind]
+    return "Flagged: " + ", ".join(k.replace("_", " ") for k in kinds)
 
 
 # SQL for the same rule, for the list filter.
-FLAG_SQL = "(p.status = 'draft' AND (p.source_kind = 'hand' OR 'drawing_needed' = ANY(p.flags)))"
+FLAG_SQL = "(p.status = 'draft' AND (p.source_kind = 'hand' OR cardinality(p.flags) > 0))"
+# A pending change order from the feed (VISUAL or M-Files): the "Changes to approve" board.
+CO_PENDING_SQL = (
+    "EXISTS (SELECT 1 FROM change_orders c WHERE c.packet_id = p.id AND c.review = 'pending'"
+    " AND c.origin IN ('visual', 'drawing'))"
+)
 
 
 def policy_principal_for(conn: Any, p: deps.Principal) -> policy.Principal:
@@ -378,6 +463,57 @@ def _unit(conn: Any, unit_id: int, *, seal: bool) -> dict[str, Any]:
     }
 
 
+def _visual(conn: Any, row: Mapping[str, Any], *, seal: bool) -> dict[str, Any]:
+    """The packet's visual jsonb. On a VISUAL draft (outside seal mode) it gains `staged_rev`: the M-Files copy the
+    drawing job staged for this drawing ({rev, name}), or None. It changes without the packet changing, so the seal
+    leaves it out."""
+    out = dict(row["visual"] or {})
+    if seal or row["source_kind"] != "visual" or row["status"] != "draft":
+        return out
+    out["staged_rev"] = staged_rev(conn, row["drawing"])
+    return out
+
+
+def staged_rev(conn: Any, drawing: Any) -> dict[str, Any] | None:
+    lk = staged_lookup(conn, drawing)
+    if lk is None:
+        return None
+    return {"rev": lk["rev"], "name": lk["name"]}
+
+
+def staged_lookup(conn: Any, drawing: Any) -> dict[str, Any] | None:
+    """The drawing_lookups row for a drawing number when it holds a fetched copy (asset and rev set)."""
+    dwg = str(drawing or "").strip()
+    if not dwg:
+        return None
+    return one(
+        conn,
+        "SELECT * FROM drawing_lookups WHERE dwg = %s AND asset_id IS NOT NULL AND rev IS NOT NULL",
+        (dwg,),
+    )
+
+
+def visual_state(conn: Any, row: Mapping[str, Any]) -> dict[str, Any] | None:
+    """What the feed last observed for a VISUAL packet: leg statuses, ticket hours, the chosen order, and the closed,
+    cancelled and gone times. None for a hand packet."""
+    if row["source_kind"] != "visual":
+        return None
+    vs = one(
+        conn,
+        "SELECT legs, ticket_hours, order_ref, observed_at FROM packet_visual_state WHERE packet_id = %s",
+        (row["id"],),
+    )
+    return {
+        "legs": (vs or {}).get("legs") or [],
+        "ticket_hours": (vs or {}).get("ticket_hours"),
+        "order_ref": (vs or {}).get("order_ref"),
+        "observed_at": (vs or {}).get("observed_at"),
+        "closed_at": row.get("visual_closed_at"),
+        "cancelled_at": row.get("visual_cancelled_at"),
+        "wo_gone_at": row.get("wo_gone_at"),
+    }
+
+
 def build(
     conn: Any,
     row: Mapping[str, Any],
@@ -424,7 +560,7 @@ def build(
         "source": row["source"] or {},
         "drawing_state": row["drawing_state"],
         "flags": list(row["flags"] or []),
-        "visual": row["visual"] or {},
+        "visual": _visual(conn, row, seal=seal),
         "removed": None
         if row["removed_at"] is None
         else {"at": row["removed_at"], "by": people.ref(row["removed_by"]), "reason": row["removed_reason"]},
@@ -451,6 +587,7 @@ def build(
                     "row_version",
                 )
             }
+            | {k: w.get(k) for k in ("dwg", "rev", "partspecs")}
             for w in ch.work_orders
         ]
         out["pages"] = [
@@ -513,7 +650,9 @@ def build(
     )
     out["checklist_items"] = checklists.checklist_for(weld, row["drawing"], row["rev"], row["wps"])
     out["qc_items"] = checklists.qc_items_for(row["drawing"], row["rev"], row["wps"], row["welder_stamp"])
-    out["flag"] = flag_of(row["status"], row["source_kind"], row["flags"], row["drawing_state"])
+    out["flag"] = flag_of(row["status"], row["source_kind"], row["flags"], row["drawing_state"], row["visual"])
+    if not seal:
+        out["visual_state"] = visual_state(conn, row)
     if pp is not None:
         allowed, blocked = policy.can(pp, state)
         out["can"] = [str(a) for a in allowed]
@@ -599,8 +738,12 @@ def _co(c: Mapping[str, Any], ch: Children, people: People) -> dict[str, Any]:
         "rejected_at",
         "rejected_reason",
         "row_version",
+        "vis",
+        "waiting",
+        "staged",
+        "target_fp",
     )
-    out = {k: c[k] for k in keys}
+    out = {k: c.get(k) for k in keys}
     out["issued_by"] = people.ref(c["issued_by"])
     out["approved_by"] = people.ref(c["approved_by"])
     out["rejected_by"] = people.ref(c["rejected_by"])
@@ -641,7 +784,10 @@ SELECT p.id, p.version, p.unit_id, u.lot, pr.number || u.lot AS unit_number, pr.
        st.name AS station_name, p.welder_id, wu.display_name AS welder_name, p.welder_stamp,
        p.routed_to::text AS routed_to, p.vclosed_at, p.vclosed_by, vu.display_name AS vclosed_name,
        p.source_kind::text AS source_kind, p.flags, p.drawing_state::text AS drawing_state, p.wh_skipped,
-       p.heat_map, p.removed_at, coalesce(u.due, pr.due) AS due_effective, p.updated_at,
+       p.heat_map, p.removed_at, coalesce(u.due, pr.due) AS due_effective, p.updated_at, p.visual,
+       p.visual_closed_at, p.visual_cancelled_at, p.wo_gone_at,
+       EXISTS (SELECT 1 FROM change_orders c WHERE c.packet_id = p.id AND c.review = 'pending'
+               AND c.origin IN ('visual', 'drawing')) AS co_pending,
        (SELECT w.stage::text FROM packet_wh w WHERE w.packet_id = p.id AND w.closed_at IS NULL) AS wh_stage,
        (SELECT count(*) FROM bom_shorts s WHERE s.packet_id = p.id AND s.closed_at IS NULL) AS open_shorts,
        (SELECT count(*) FROM packet_work_orders wo WHERE wo.packet_id = p.id) AS wo_count,
@@ -697,7 +843,8 @@ def card_state(r: Mapping[str, Any], names: Mapping[str, str]) -> policy.PacketS
 
 
 def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[str, Any]) -> list[dict[str, Any]]:
-    """The board cards. Filters: status (comma list), dept, unit_id, project_id, q, flag, limit, removed."""
+    """The board cards. Filters: status (comma list), dept, unit_id, project_id, q, flag (1|0|hand|hand_upload|
+    <kind>), co (pending), limit, removed."""
     where: list[str] = []
     params: list[Any] = []
     if not filters.get("removed"):
@@ -721,7 +868,24 @@ def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[s
     elif flag in FLAG_OFF:
         where.append("NOT " + FLAG_SQL)
     elif flag == "hand":
-        where.append("(p.status = 'draft' AND p.source_kind = 'hand')")
+        # Made by hand, or a VISUAL draft with a drawing or BOM uploaded by hand.
+        where.append("(p.status = 'draft' AND (p.source_kind = 'hand' OR p.flags && %s::text[]))")
+        params.append(list(HAND_UPLOAD_FLAGS))
+    elif flag in ("hand_upload", "hand_uploads"):
+        where.append("(p.status = 'draft' AND p.flags && %s::text[])")
+        params.append(list(HAND_UPLOAD_FLAGS))
+    elif flag and RX_FLAG_KIND.match(flag):
+        # One flag kind on the Flagged board (drafts). A visual draft whose flags are empty but whose drawing state
+        # still names the problem counts too, the same way flag_of reads it.
+        states = [st for st, k in STATE_KINDS.items() if k == flag]
+        where.append(
+            "(p.status = 'draft' AND (%s = ANY(p.flags) OR (cardinality(p.flags) = 0 AND p.source_kind = 'visual'"
+            " AND p.drawing_state::text = ANY(%s))))"
+        )
+        params += [flag, states]
+    co = str(filters.get("co") or "").strip().lower()
+    if co == "pending":
+        where.append(CO_PENDING_SQL)
     for word in str(filters.get("q") or "").split()[:8]:
         pat = _like(word)
         where.append(
@@ -771,7 +935,11 @@ def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[s
                     else {"id": str(r["vclosed_by"]), "display_name": r["vclosed_name"]},
                 },
                 "source_kind": r["source_kind"],
-                "flag": flag_of(r["status"], r["source_kind"], r["flags"], r["drawing_state"]),
+                "flag": flag_of(r["status"], r["source_kind"], r["flags"], r["drawing_state"], r["visual"]),
+                "flag_kinds": list(r["flags"] or []),
+                "co_pending": bool(r["co_pending"]),
+                "visual_closed": r["visual_closed_at"] is not None,
+                "drawing_state": r["drawing_state"],
                 "open_cos": [c["co_no"] for c in r["open_co_rows"] or []],
                 "open_shorts": int(r["open_shorts"] or 0),
                 "wh_stage": r["wh_stage"],

server/packetstation/domain/projects.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 import re
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
-from datetime import date
+from datetime import UTC, date, datetime
 from typing import Any
 
 from psycopg import errors as pgerr
@@ -28,8 +28,11 @@ MAX_UNITS_AT_ONCE = 200
 MAX_WORDS = 8
 LIST_LIMIT = 500
 
-PROJECT_FIELDS = ("name", "customer", "po", "due", "status", "notes")
-UNIT_FIELDS = ("name", "due", "status", "notes")
+PROJECT_FIELDS = ("name", "customer", "po", "due", "status", "notes", "order_pin", "order_pin_reason")
+UNIT_FIELDS = ("name", "customer", "po", "due", "status", "notes", "order_pin", "order_pin_reason")
+# Order-derived fields: the value in use, where it came from (visual | hand) and what VISUAL last said.
+ORDER_FIELDS = ("customer", "po", "order_date", "ship_date")
+RX_ORDER_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ./_-]{0,39}$")
 
 _DIGITS = re.compile(r"^[0-9]+$")
 _ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
@@ -159,6 +162,28 @@ def _removed(r: Mapping[str, Any]) -> dict[str, Any] | None:
 # ---------------------------------------------------------------- shapes
 
 
+def _order_out(r: Mapping[str, Any]) -> dict[str, Any]:
+    """The order-derived columns with their provenance, and the order pin. Shared by projects and units."""
+    pin_by = r.get("order_pin_by")
+    return {
+        "customer_src": r.get("customer_src"),
+        "customer_visual": r.get("customer_visual"),
+        "po_src": r.get("po_src"),
+        "po_visual": r.get("po_visual"),
+        "order_date": r.get("order_date"),
+        "order_date_src": r.get("order_date_src"),
+        "order_date_visual": r.get("order_date_visual"),
+        "ship_date": r.get("ship_date"),
+        "ship_date_src": r.get("ship_date_src"),
+        "ship_date_visual": r.get("ship_date_visual"),
+        "order_ref": r.get("order_ref"),
+        "order_pin": r.get("order_pin"),
+        "order_pin_by": str(pin_by) if pin_by else None,
+        "order_pin_at": r.get("order_pin_at"),
+        "order_pin_reason": r.get("order_pin_reason"),
+    }
+
+
 def project_out(r: Mapping[str, Any]) -> dict[str, Any]:
     return {
         "id": r["id"],
@@ -171,6 +196,7 @@ def project_out(r: Mapping[str, Any]) -> dict[str, Any]:
         "notes": r["notes"],
         "source": r["source"],
         "needs_details": r["needs_details"],
+        **_order_out(r),
         "units": int(r.get("unit_count") or 0),
         "packets": _counts(r),
         "version": r["version"],
@@ -193,6 +219,9 @@ def unit_out(r: Mapping[str, Any]) -> dict[str, Any]:
         "status": status,
         "status_effective": "closed" if r["project_status"] == "closed" else status,
         "notes": r["notes"],
+        "customer": r.get("customer"),
+        "po": r.get("po"),
+        **_order_out(r),
         "packets": _counts(r),
         "version": r["version"],
         "created_at": r["created_at"],
@@ -322,6 +351,7 @@ def project_detail(conn: Any, project_id: int) -> dict[str, Any]:
     out["unit_count"] = len(rows)
     out["aliases"] = aliases_of(conn, project_id=project_id)
     out["next_lot"] = next_lot(conn, project_id, out["number"])
+    out["orders"] = cached_orders(conn, out["number"], _project_bases(conn, project_id))
     return out
 
 
@@ -370,6 +400,7 @@ def unit_detail(conn: Any, unit_id: int) -> dict[str, Any]:
         raise ApiError(404, "not_found", "No such unit.")
     out["project"] = {k: p[k] for k in ("id", "number", "name", "customer", "po", "due", "status", "version")}
     out["aliases"] = aliases_of(conn, unit_id=unit_id)
+    out["orders"] = cached_orders(conn, p["number"], _unit_bases(conn, unit_id))
     return out
 
 
@@ -479,6 +510,13 @@ def _norm_field(key: str, value: Any, *, unit: bool) -> Any:
         return clean_opt(value, LIMITS["po"])
     if key == "notes":
         return clean_opt(value, LIMITS["notes"])
+    if key == "order_pin":
+        pin = clean_opt(value, LIMITS["order_pin"] + 20)
+        if pin is not None and not RX_ORDER_ID.match(pin):
+            raise ApiError(422, "invalid_order", "Give a VISUAL customer order ID, or nothing to clear the pin.")
+        return pin
+    if key == "order_pin_reason":
+        return clean_opt(value, LIMITS["reason"])
     raise ApiError(422, "invalid_field", f"{key} cannot be changed here.")
 
 
@@ -515,6 +553,100 @@ def _change_detail(row: Mapping[str, Any], changes: Mapping[str, Any]) -> dict[s
     return out
 
 
+def _columns(row: Mapping[str, Any], changes: Mapping[str, Any], who: Who) -> dict[str, Any]:
+    """The columns a field edit writes: a hand customer or po marks its source, a pin records who and when."""
+    cols = dict(changes)
+    for k in ("customer", "po"):
+        if k in cols:
+            cols[f"{k}_src"] = "hand" if cols[k] else None
+    if "order_pin" in cols:
+        if cols["order_pin"] is None:
+            cols.update(order_pin_by=None, order_pin_at=None, order_pin_reason=None)
+        else:
+            cols["order_pin_by"] = who.user_id
+            cols["order_pin_at"] = datetime.now(UTC)
+    return cols
+
+
+def _check_pin(
+    conn: Any, row: Mapping[str, Any], changes: Mapping[str, Any], *, project_number: str, bases: list[str]
+) -> None:
+    """A new pin needs a reason and, when VISUAL's orders are cached for this project, must name one of them."""
+    if "order_pin" not in changes and "order_pin_reason" in changes and row.get("order_pin") is None:
+        raise ApiError(422, "invalid_field", "There is no order pin to give a reason for.")
+    pin = changes.get("order_pin")
+    if pin is None:
+        return
+    if not changes.get("order_pin_reason"):
+        raise ApiError(400, "reason_required", "Say why this order is pinned.")
+    known = [o["cust_order_id"] for o in cached_orders(conn, project_number, bases)]
+    if known and pin.upper() not in {k.upper() for k in known}:
+        raise ApiError(
+            422,
+            "unknown_order",
+            f"Order {pin} is not one of the VISUAL orders for {project_number}.",
+            extra={"orders": known},
+        )
+
+
+def _project_bases(conn: Any, project_id: int) -> list[str]:
+    return [
+        r[0]
+        for r in conn.execute(
+            "SELECT DISTINCT wo.wo_base FROM packet_work_orders wo JOIN packets k ON k.id = wo.packet_id"
+            " JOIN units u ON u.id = k.unit_id WHERE u.project_id = %s ORDER BY 1",
+            (project_id,),
+        )
+    ]
+
+
+def _unit_bases(conn: Any, unit_id: int) -> list[str]:
+    return [
+        r[0]
+        for r in conn.execute(
+            "SELECT DISTINCT wo.wo_base FROM packet_work_orders wo JOIN packets k ON k.id = wo.packet_id"
+            " WHERE k.unit_id = %s ORDER BY 1",
+            (unit_id,),
+        )
+    ]
+
+
+def cached_orders(conn: Any, project_number: str, bases: Iterable[str]) -> list[dict[str, Any]]:
+    """Every cached VISUAL order of a project: the ones linked to the project number, then the ones linked to its
+    work order bases. Each once, with `via` (project | base) and the link's lines and top jobs."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute(
+            "SELECT o.*, l.kind AS via, l.key, l.rank, l.lines, l.top_jobs FROM visual_order_links l"
+            " JOIN visual_orders o ON o.cust_order_id = l.cust_order_id"
+            " WHERE (l.kind = 'project' AND upper(l.key) = upper(%s)) OR (l.kind = 'base' AND l.key = ANY(%s))"
+            " ORDER BY CASE l.kind WHEN 'project' THEN 0 ELSE 1 END, l.rank, l.key, o.cust_order_id",
+            (project_number, list(bases)),
+        ).fetchall()
+    out: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for r in rows:
+        if r["cust_order_id"] in seen:
+            continue
+        seen.add(r["cust_order_id"])
+        out.append(
+            {
+                "cust_order_id": r["cust_order_id"],
+                "status": r["status"],
+                "customer_id": r["customer_id"],
+                "customer_name": r["customer_name"],
+                "customer_po_ref": r["customer_po_ref"],
+                "order_date": r["order_date"],
+                "desired_ship_date": r["desired_ship_date"],
+                "promise_date": r["promise_date"],
+                "via": r["via"],
+                "key": r["key"],
+                "lines": list(r["lines"] or []),
+                "top_jobs": list(r["top_jobs"] or []),
+            }
+        )
+    return out
+
+
 # ---------------------------------------------------------------- writes: projects
 
 
@@ -567,11 +699,8 @@ def update_project(
         row = _lock_project(conn, project_id)
         changes = field_cas(row, values, base, PROJECT_FIELDS, unit=False)
         if changes:
-            cols = dict(changes)
-            if "customer" in cols:
-                cols["customer_src"] = "hand" if cols["customer"] else None
-            if "po" in cols:
-                cols["po_src"] = "hand" if cols["po"] else None
+            _check_pin(conn, row, changes, project_number=row["number"], bases=_project_bases(conn, project_id))
+            cols = _columns(row, changes, who)
             sets = ", ".join(f"{k} = %s" for k in cols)
             conn.execute(
                 f"UPDATE projects SET {sets}, version = version + 1, updated_at = now() WHERE id = %s",  # noqa: S608
@@ -800,10 +929,12 @@ def update_unit(
         row, project = _lock_unit(conn, unit_id)
         changes = field_cas(row, values, base, UNIT_FIELDS, unit=True)
         if changes:
-            sets = ", ".join(f"{k} = %s" for k in changes)
+            _check_pin(conn, row, changes, project_number=project["number"], bases=_unit_bases(conn, unit_id))
+            cols = _columns(row, changes, who)
+            sets = ", ".join(f"{k} = %s" for k in cols)
             conn.execute(
                 f"UPDATE units SET {sets}, version = version + 1, updated_at = now() WHERE id = %s",  # noqa: S608
-                [*changes.values(), unit_id],
+                [*cols.values(), unit_id],
             )
             _audit(
                 conn,

server/packetstation/drawings/__init__.py

@@ -0,0 +1,17 @@
+"""Drawings from M-Files, read-only: the lookup, the decision table, the queue and the fetch into assets.
+
+Packet Station never writes to M-Files. MFilesKit enforces that in its own code; this package only calls its
+read methods (`DrawingLookup.status`, `DrawingLookup.download`).
+"""
+
+from __future__ import annotations
+
+# The packet flags this package owns. The drawing job replaces only these and keeps every other flag.
+DRAWING_FLAGS: tuple[str, ...] = (
+    "drawing_needed",
+    "rev_mismatch",
+    "not_released",
+    "not_found",
+    "lookup_failed",
+    "bad_rev",
+)

server/packetstation/drawings/decide.py

@@ -0,0 +1,190 @@
+"""The drawing decision table. Pure: no database, no M-Files.
+
+`decide(target, found, same_file=...)` says what the drawing job does to one packet for one lookup. Note texts are
+the app's own (from the old revMismatch and draftLine in visual.js, "-" in place of the em-dash), never M-Files'
+note passed through, except inside the lookup_failed text.
+
+Rows, in the order they are tried:
+ 1. lookup failed -> flag lookup_failed, keep what is attached (state lookup_failed only when it was needed).
+ 2. past draft, a pending feed change order waits for this drawing at the released rev -> stage_for_co.
+    Past draft, the pending change order already stages this drawing at the released rev -> unchanged, or
+    stage_for_co again when M-Files has a different file for that rev.
+ 3. no released rev, a draft rev or a drafting assignment -> flag not_released with the drafting status.
+ 4. no released rev otherwise -> flag not_found.
+ 5. VISUAL rev not a number (and no approved rev equal to M-Files') -> flag bad_rev, not attached.
+ 6. revs differ (and no approved rev equal to M-Files') -> flag rev_mismatch. The job stages the M-Files copy
+    for Clerical Supervisor approval on drafts only.
+ 7. revs match, draft with a hand drawing -> unchanged (the feed already took the hand drawing off on a rev move).
+ 8. revs match, draft with this very file attached at this rev -> unchanged.
+ 9. revs match, draft -> attach ("sheet of drawing <base>" for an H sheet).
+10. revs match, past draft, nothing attached -> unchanged.
+11. revs match, past draft, same file as attached -> unchanged.
+12. revs match, past draft, a pending change order already stages this file, or Clerical rejected a change order
+    for this file -> unchanged.
+13. revs match, past draft, attached rev lower -> open_co "M-Files released rev N of <dwg>; this packet has rev M".
+14. revs match, past draft, attached rev equal, new file -> open_co "M-Files re-released rev N of <dwg> (new file)".
+15. revs match, past draft, attached rev higher or not a number -> unchanged.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .lookup import Found
+
+LOOKUP_FAILED = "lookup_failed"
+NOT_RELEASED = "not_released"
+NOT_FOUND = "not_found"
+BAD_REV = "bad_rev"
+REV_MISMATCH = "rev_mismatch"
+ATTACH = "attach"
+UNCHANGED = "unchanged"
+OPEN_CO = "open_co"
+STAGE_FOR_CO = "stage_for_co"
+
+KINDS = (ATTACH, UNCHANGED, REV_MISMATCH, NOT_RELEASED, NOT_FOUND, LOOKUP_FAILED, BAD_REV, OPEN_CO, STAGE_FOR_CO)
+FLAG_KINDS = (LOOKUP_FAILED, NOT_RELEASED, NOT_FOUND, BAD_REV, REV_MISMATCH)
+
+# drawing_state enum values (0003). bad_rev has no state of its own: it is a rev mismatch Clerical can approve.
+STATE_OF = {NOT_RELEASED: "not_released", NOT_FOUND: "not_found", BAD_REV: "rev_mismatch", REV_MISMATCH: "rev_mismatch"}
+HELD_STATES = ("attached", "hand")
+
+NOT_FOUND_NOTE = "Drawing not in M-Files - check the drawing number in VISUAL"
+_FAILED_PREFIX = "M-Files lookup failed: "
+
+
+@dataclass(frozen=True)
+class Target:
+    dwg: str
+    visual_rev: str
+    attached_rev: str | None  # rev of the live drawing pages; None when no drawing page is attached
+    attached_sha: bytes | None
+    status: str  # packet status
+    drawing_state: str
+    has_hand_drawing: bool
+    rev_ok: str | None  # the rev Clerical approved for this VISUAL rev, if any
+    # Additions to the contract, all defaulted: the pending feed change order (origin visual or drawing), if any.
+    co_waiting: tuple[str, str] | None = None  # (dwg, rev) the pending CO waits for
+    co_staged: tuple[str, str] | None = None  # (dwg, rev) the pending CO already stages
+    co_pending: bool = False
+    co_staged_sha: bytes | None = None  # sha256 of the file the pending CO already stages
+    rejected_sha: bytes | None = None  # sha256 staged on the last drawing-origin CO Clerical rejected
+
+
+@dataclass(frozen=True)
+class Decision:
+    kind: str
+    flag: str | None
+    drawing_state: str
+    note: str
+
+
+def _int(v: object) -> int | None:
+    s = "" if v is None else str(v).strip()
+    return int(s) if s.isdigit() else None
+
+
+def draft_line(f: Found) -> str:
+    """The old draftLine: "rev N <where>", or "in drafting, no released PDF yet", or ""."""
+    if f.draft_rev is not None and f.draft_where:
+        return f"rev {f.draft_rev} {f.draft_where}"
+    if f.in_drafting:
+        return "in drafting, no released PDF yet"
+    if f.draft_rev is not None:
+        return f"rev {f.draft_rev}"
+    return ""
+
+
+def failed_note(f: Found) -> str:
+    detail = (f.note or "").strip()
+    if detail.startswith(_FAILED_PREFIX):  # MFilesKit's own note already carries the prefix
+        detail = detail[len(_FAILED_PREFIX) :]
+    return f"{_FAILED_PREFIX}{detail or 'no detail'}"
+
+
+def rev_confirmed(t: Target, f: Found) -> bool:
+    """The old revConfirmed: M-Files' released rev is the rev VISUAL calls, or Clerical approved that rev."""
+    if not f.ok or f.rev is None:
+        return False
+    v = _int(t.visual_rev)
+    if v is not None and v == f.rev:
+        return True
+    return t.rev_ok is not None and str(t.rev_ok).strip() == str(f.rev)
+
+
+def decide(t: Target, f: Found, *, same_file: bool, fetched_sha: bytes | None = None) -> Decision:
+    """What to do to one packet.
+
+    `same_file`: the M-Files file has the sha256 of the attached drawing asset. `fetched_sha`: the sha256 of the
+    M-Files file, when it was fetched (row 12 compares it with the pending and the rejected change orders).
+    """
+    held = t.drawing_state in HELD_STATES
+    draft = t.status == "draft"
+
+    def flag(kind: str, note: str) -> Decision:
+        return Decision(kind, kind, t.drawing_state if held else STATE_OF[kind], note)
+
+    def keep(kind: str, note: str = "") -> Decision:
+        return Decision(kind, None, t.drawing_state, note)
+
+    # 1
+    if not f.ok:
+        return Decision(
+            LOOKUP_FAILED,
+            LOOKUP_FAILED,
+            LOOKUP_FAILED if t.drawing_state == "needed" else t.drawing_state,
+            failed_note(f),
+        )
+    # 2
+    if not draft and f.rev is not None and t.co_waiting is not None:
+        w_dwg, w_rev = t.co_waiting
+        if w_dwg.strip() == t.dwg.strip() and str(w_rev).strip() == str(f.rev):
+            return keep(STAGE_FOR_CO, f"Drawing for the change order is in M-Files (rev {f.rev})")
+    if not draft and f.rev is not None and t.co_staged is not None:
+        s_dwg, s_rev = t.co_staged
+        if s_dwg.strip() == t.dwg.strip() and str(s_rev).strip() == str(f.rev):
+            if fetched_sha is not None and t.co_staged_sha is not None and fetched_sha != t.co_staged_sha:
+                return keep(STAGE_FOR_CO, f"New file for rev {f.rev} in M-Files; staged again for the change order")
+            return keep(UNCHANGED, f"Rev {f.rev} is staged on the pending change order")
+    # 3, 4
+    if f.rev is None:
+        d = draft_line(f)
+        if d:
+            on = f" (sheet of {f.base})" if f.base else ""
+            return flag(NOT_RELEASED, f"Not released in M-Files yet{on} - drafting status: {d}")
+        return flag(NOT_FOUND, NOT_FOUND_NOTE)
+    approved = t.rev_ok is not None and str(t.rev_ok).strip() == str(f.rev)
+    vrev = (t.visual_rev or "").strip()
+    v = _int(vrev)
+    if not approved:
+        # 5
+        if v is None:
+            return flag(BAD_REV, f"VISUAL rev {vrev or '-'} is not a number; M-Files has rev {f.rev} - not attached")
+        # 6
+        if v != f.rev:
+            d = f"; drafting status: {draft_line(f)}" if f.draft_rev is not None and f.draft_rev > f.rev else ""
+            return flag(REV_MISMATCH, f"Rev mismatch: VISUAL calls rev {vrev}, M-Files has rev {f.rev}{d}")
+    sheet = f"sheet of drawing {f.base}" if f.base else ""
+    if draft:
+        # 7
+        if t.has_hand_drawing:
+            return keep(UNCHANGED, "A drawing uploaded by hand stays at the same rev")
+        # 8
+        if t.drawing_state == "attached" and same_file and t.attached_rev == str(f.rev):
+            return keep(UNCHANGED, sheet)
+        # 9
+        return Decision(ATTACH, None, "attached", sheet)
+    # 10, 11
+    if t.attached_rev is None or same_file:
+        return keep(UNCHANGED, sheet)
+    # 12
+    if fetched_sha is not None and fetched_sha in (t.co_staged_sha, t.rejected_sha):
+        return keep(UNCHANGED, sheet)
+    att = _int(t.attached_rev)
+    if att is None or att > f.rev:
+        return keep(UNCHANGED, sheet)  # 15
+    if att < f.rev:
+        reason = f"M-Files released rev {f.rev} of {t.dwg}; this packet has rev {att}"
+    else:
+        reason = f"M-Files re-released rev {f.rev} of {t.dwg} (new file)"
+    return keep(OPEN_CO, reason)

server/packetstation/drawings/fake.py

@@ -0,0 +1,118 @@
+"""The fixture lookup for CI and tests: `dir/drawings.<state>.json` and the PDFs it names.
+
+`dir/state` holds the state number (default "1"). The JSON maps a drawing number to
+`{"rev": 3, "file": "L073762X r3.pdf", "draft_rev": null, "draft_where": "", "in_drafting": false, "ok": true,
+"note": "", "obj_id": 1, "obj_version": 1, "file_id": 1, "file_version": 1}`; every key is optional. A drawing
+not in the file is not found, after the same H-sheet fallback MFilesKit uses (K978-01-6H13 -> K978-01-6).
+`download` copies `dir/<file>`.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+from pathlib import Path
+from typing import Any
+
+from .lookup import DrawingFetchError, Found, failed
+
+NOT_FOUND_NOTE = "not in M-Files - check the drawing number in VISUAL"
+_H_SHEET = re.compile(r"^(.*\d)H\d{1,3}$")
+
+
+def base_drawing(dwg: str) -> str | None:
+    m = _H_SHEET.match(dwg)
+    return m.group(1) if m else None
+
+
+def _int(v: Any) -> int | None:
+    if v is None or isinstance(v, bool):
+        return None
+    try:
+        return int(str(v).strip())
+    except ValueError:
+        return None
+
+
+class FixtureLookup:
+    def __init__(self, directory: Path, state: str | None = None) -> None:
+        self.dir = Path(directory)
+        if state is None:
+            try:
+                state = (self.dir / "state").read_text(encoding="utf-8").strip() or "1"
+            except OSError:
+                state = "1"
+        self.state = state
+        self._data: dict[str, dict[str, Any]] | None = None
+        self._error: str | None = None
+
+    def _load(self) -> dict[str, dict[str, Any]]:
+        if self._data is None:
+            path = self.dir / f"drawings.{self.state}.json"
+            try:
+                raw = json.loads(path.read_text(encoding="utf-8"))
+                self._data = {str(k).strip(): v for k, v in raw.items() if isinstance(v, dict)}
+            except (OSError, ValueError, AttributeError) as e:
+                self._error = f"fixture {path.name} could not be read: {type(e).__name__}"
+                self._data = {}
+        return self._data
+
+    def status(self, dwg: str) -> Found:
+        dwg = (dwg or "").strip()
+        data = self._load()
+        if self._error:
+            return failed(dwg, self._error)
+        if not dwg:
+            return Found(dwg="", note="no drawing number")
+        entry, base = data.get(dwg), None
+        if entry is None:
+            b = base_drawing(dwg)
+            if b and b in data:
+                entry, base = data[b], b
+        if entry is None:
+            return Found(dwg=dwg, note=NOT_FOUND_NOTE)
+        if entry.get("ok") is False:
+            return failed(dwg, str(entry.get("note") or "M-Files lookup failed: fixture says so"))
+        rev = _int(entry.get("rev"))
+        file = entry.get("file") or None
+        size = None
+        if file and rev is not None:
+            p = self._file(str(file))
+            size = p.stat().st_size if p is not None and p.is_file() else None
+        has_file = rev is not None and bool(file)
+        return Found(
+            dwg=dwg,
+            ok=True,
+            rev=rev,
+            name=str(entry.get("name") or file or "") or None if rev is not None else None,
+            obj_id=_int(entry.get("obj_id", 1)) if has_file else None,
+            obj_version=_int(entry.get("obj_version", 1)) if has_file else None,
+            file_id=_int(entry.get("file_id", 1)) if has_file else None,
+            file_version=_int(entry.get("file_version", 1)) if has_file else None,
+            file_size=size,
+            base=base,
+            draft_rev=_int(entry.get("draft_rev")),
+            draft_where=str(entry.get("draft_where") or ""),
+            in_drafting=bool(entry.get("in_drafting")),
+            note=str(entry.get("note") or ""),
+        )
+
+    def _file(self, name: str) -> Path | None:
+        root = self.dir.resolve()
+        p = (root / name).resolve()
+        return p if p.is_relative_to(root) else None
+
+    def download(self, found: Found, dest: Path) -> int:
+        entry = self._load().get(found.base or found.dwg) or self._load().get(found.dwg) or {}
+        name = entry.get("file")
+        src = self._file(str(name)) if name else None
+        if src is None or not src.is_file():
+            raise DrawingFetchError(f"{found.dwg}: fixture file {name!r} is missing")
+        dest = Path(dest)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(src, dest)
+        return dest.stat().st_size
+
+    def describe(self) -> str:
+        return f"fixture {self.dir} state {self.state}"

server/packetstation/drawings/fetch.py

@@ -0,0 +1,125 @@
+"""Copy a released drawing PDF from M-Files into assets.
+
+Downloads only when the M-Files identity (object id and version, file id and version) differs from what the
+`drawing_lookups` row last stored, or that row has no asset (or the asset's file is gone). The download goes to
+ASSETS_DIR/tmp/<uuid>; the PDF magic and the size limit are checked, then `assets.store_file` hashes it in chunks,
+fsyncs and renames it into place, and reuses an existing asset row with the same sha256 and size.
+
+Read-only to M-Files: only `Lookup.download` is called.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from psycopg.rows import dict_row
+
+from ..api.errors import ApiError
+from ..domain import assets
+from .lookup import Found, Lookup
+
+PDF_MAGIC = b"%PDF-"
+
+
+class FetchError(Exception):
+    """The drawing could not be copied. The message is safe to log."""
+
+
+@dataclass(frozen=True, slots=True)
+class FetchResult:
+    asset_id: uuid.UUID
+    sha256: bytes
+    pages: int
+    downloaded: bool  # False when the stored asset was still current
+    name: str | None = None
+    reused: bool = False  # downloaded, but the same bytes were already stored
+
+
+def _current(conn: Any, settings: Any, found: Found, dwg: str) -> FetchResult | None:
+    with conn.cursor(row_factory=dict_row) as cur:
+        row = cur.execute(
+            "SELECT l.obj_id, l.obj_version, l.file_id, l.file_version, l.asset_id, a.sha256, a.pages,"
+            " a.storage_key, a.original_name FROM drawing_lookups l LEFT JOIN assets a ON a.id = l.asset_id"
+            " WHERE l.dwg = %s",
+            (dwg,),
+        ).fetchone()
+    if row is None or row["asset_id"] is None or row["sha256"] is None:
+        return None
+    if (row["obj_id"], row["obj_version"], row["file_id"], row["file_version"]) != found.identity:
+        return None
+    if None in found.identity:
+        return None
+    try:
+        if not assets.path_of(settings, str(row["storage_key"])).is_file():
+            return None
+    except ApiError:
+        return None
+    return FetchResult(
+        asset_id=row["asset_id"],
+        sha256=bytes(row["sha256"]),
+        pages=int(row["pages"] or 0),
+        downloaded=False,
+        name=row["original_name"],
+    )
+
+
+def fetch(conn: Any, settings: Any, lookup: Lookup, found: Found, *, dwg: str) -> FetchResult:
+    """The asset for M-Files' released file of `dwg`. Raises FetchError."""
+    if not found.ok or found.rev is None:
+        raise FetchError(f"{dwg}: no released file in M-Files")
+    cur = _current(conn, settings, found, dwg)
+    if cur is not None:
+        return cur
+    limit = int(settings.upload_max_bytes)
+    if found.file_size is not None and found.file_size > limit:
+        raise FetchError(f"{dwg}: the M-Files file is {found.file_size} bytes, over the {limit} byte limit")
+    tmp_dir = Path(settings.assets_dir) / "tmp"
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+    tmp = tmp_dir / uuid.uuid4().hex
+    try:
+        try:
+            lookup.download(found, tmp)
+        except LookupError as e:
+            raise FetchError(str(e)) from None
+        except OSError as e:
+            raise FetchError(f"{dwg}: could not write the download: {type(e).__name__}") from None
+        try:
+            size = tmp.stat().st_size
+            with tmp.open("rb") as f:
+                head = f.read(len(PDF_MAGIC))
+        except OSError:
+            raise FetchError(f"{dwg}: the download left no file") from None
+        if size > limit:
+            raise FetchError(f"{dwg}: the M-Files file is {size} bytes, over the {limit} byte limit")
+        if head != PDF_MAGIC:
+            raise FetchError(f"{dwg}: the M-Files file is not a PDF")
+        name = found.name or f"{dwg} r{found.rev}.pdf"
+        source_ref = {
+            "dwg": dwg,
+            "obj_id": found.obj_id,
+            "obj_version": found.obj_version,
+            "file_id": found.file_id,
+            "file_version": found.file_version,
+            "rev": found.rev,
+        }
+        try:
+            row = assets.store_file(
+                conn, settings, tmp, mime=assets.PDF, original_name=name, source="mfiles", source_ref=source_ref
+            )
+        except ApiError as e:
+            raise FetchError(f"{dwg}: {e.detail}") from None
+        return FetchResult(
+            asset_id=row["id"],
+            sha256=bytes(row["sha256"]),
+            pages=int(row["pages"] or 0),
+            downloaded=True,
+            name=row.get("original_name") or name,
+            reused=bool(row.get("reused")),
+        )
+    finally:
+        with contextlib.suppress(OSError):
+            tmp.unlink()

server/packetstation/drawings/lookup.py

@@ -0,0 +1,183 @@
+"""The drawing lookup the job talks to: M-Files through MFilesKit, or the fixture files (fake.py).
+
+One M-Files `Vault` per worker process (each sign-in can use a licence), kept in the dict the scheduler passes
+(`JobContext.shared`) or, when there is none, in this module's own cache keyed by MFILES_URL. A new
+`DrawingLookup` per run, because its result cache lives on the instance.
+
+Read-only. Nothing here writes to M-Files.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import threading
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Protocol
+
+if TYPE_CHECKING:
+    from mfileskit.drawings import DrawingStatus
+    from mfileskit.vault import Vault
+
+    from ..settings import Settings
+
+log = logging.getLogger(__name__)
+
+VAULT_KEY = "mfiles_vault"
+NOTE_MAX = 500
+
+
+class DrawingFetchError(LookupError):
+    """A download did not work. The message is safe to log (no credentials)."""
+
+
+@dataclass(slots=True)
+class Found:
+    """What M-Files has for one drawing number: the subset the decision needs, plus identity for change detection."""
+
+    dwg: str
+    ok: bool = True
+    rev: int | None = None
+    name: str | None = None
+    obj_id: int | None = None
+    obj_version: int | None = None
+    file_id: int | None = None
+    file_version: int | None = None
+    file_size: int | None = None
+    base: str | None = None
+    draft_rev: int | None = None
+    draft_where: str = ""
+    in_drafting: bool = False
+    note: str = ""
+
+    @property
+    def identity(self) -> tuple[int | None, int | None, int | None, int | None]:
+        return (self.obj_id, self.obj_version, self.file_id, self.file_version)
+
+
+def failed(dwg: str, note: str) -> Found:
+    return Found(dwg=dwg, ok=False, note=note[:NOTE_MAX])
+
+
+class Lookup(Protocol):
+    def status(self, dwg: str) -> Found: ...
+
+    def download(self, found: Found, dest: Path) -> int:
+        """Write the released PDF to `dest`; bytes written. Raises DrawingFetchError (a LookupError)."""
+        ...
+
+    def describe(self) -> str: ...
+
+
+def from_status(st: DrawingStatus) -> Found:
+    """MFilesKit's DrawingStatus -> Found."""
+    ov, fi = st.obj_ver, st.file
+    return Found(
+        dwg=st.drawing,
+        ok=bool(st.ok),
+        rev=st.rev,
+        name=st.name,
+        obj_id=ov.id if ov is not None else None,
+        obj_version=ov.version if ov is not None else None,
+        file_id=fi.id if fi is not None else None,
+        file_version=fi.version if fi is not None else None,
+        file_size=fi.size if fi is not None else None,
+        base=st.base,
+        draft_rev=st.draft_rev,
+        draft_where=st.draft_where or "",
+        in_drafting=bool(st.in_drafting),
+        note=(st.note or "")[:NOTE_MAX],
+    )
+
+
+class MFilesLookup:
+    """M-Files through `mfileskit.drawings.DrawingLookup`. Read-only."""
+
+    def __init__(self, vault: Vault) -> None:
+        from mfileskit.drawings import DrawingLookup
+
+        self.vault = vault
+        self._lookup = DrawingLookup(vault)  # reads the vault schema; raises MFilesError when the vault is down
+        self._seen: dict[str, DrawingStatus] = {}
+
+    def status(self, dwg: str) -> Found:
+        from mfileskit.errors import MFilesError
+
+        dwg = (dwg or "").strip()
+        try:
+            st = self._lookup.status(dwg)
+        except MFilesError as e:  # status() catches these itself; this is a second net
+            return failed(dwg, f"M-Files lookup failed: {e}")
+        self._seen[dwg] = st
+        return from_status(st)
+
+    def download(self, found: Found, dest: Path) -> int:
+        from mfileskit.errors import MFilesError
+
+        st = self._seen.get(found.dwg)
+        if st is None or st.obj_ver is None or st.file is None:
+            raise DrawingFetchError(f"{found.dwg}: no released file to download")
+        try:
+            return int(self._lookup.download(st, dest))
+        except MFilesError as e:
+            raise DrawingFetchError(f"{found.dwg}: download failed: {e}"[:NOTE_MAX]) from None
+
+    def describe(self) -> str:
+        url = getattr(getattr(self.vault, "settings", None), "url", "") or ""
+        return f"M-Files {url}".strip()
+
+
+# ---------------------------------------------------------------- one vault per process
+
+_LOCK = threading.Lock()
+_CACHE: dict[str, Any] = {}
+
+
+def _cache_for(cache: dict[str, Any] | None) -> tuple[dict[str, Any], str]:
+    if cache is not None:
+        return cache, VAULT_KEY
+    return _CACHE, f"{VAULT_KEY}:{os.environ.get('MFILES_URL', '')}"
+
+
+def vault_for(settings: Settings, cache: dict[str, Any] | None = None) -> Vault:
+    """The process's Vault, made once with `Vault.from_env()` (MFILES_* from the environment; the entrypoint
+    exports MFILES_PASSWORD from its secret file) and signed in."""
+    from mfileskit.vault import Vault
+
+    store, key = _cache_for(cache)
+    with _LOCK:
+        vault = store.get(key)
+        if vault is None:
+            vault = Vault.from_env().connect()
+            store[key] = vault
+        return vault
+
+
+def forget_vault(cache: dict[str, Any] | None = None) -> None:
+    """Drop (and close) the cached Vault, so the next run signs in again."""
+    store, key = _cache_for(cache)
+    with _LOCK:
+        vault = store.pop(key, None)
+    if vault is not None:
+        try:
+            vault.close()
+        except Exception as e:  # closing a broken connection may fail; nothing to do
+            log.info("closing the M-Files vault failed: %s", type(e).__name__)
+
+
+def make(settings: Settings, cache: dict[str, Any] | None = None) -> Lookup:
+    """The lookup for one run. `drawing_source = fixture` reads `fixture_dir`; otherwise M-Files.
+
+    Raises when M-Files cannot be opened (the job then records a degraded run). A failed open drops the cached
+    Vault so the next run tries a fresh sign-in.
+    """
+    if getattr(settings, "drawing_source", "mfiles") == "fixture":
+        from .fake import FixtureLookup
+
+        return FixtureLookup(Path(settings.fixture_dir))
+    try:
+        return MFilesLookup(vault_for(settings, cache))
+    except Exception:
+        forget_vault(cache)
+        raise

server/packetstation/drawings/queue.py

@@ -0,0 +1,113 @@
+"""Which drawings the drawing job looks up this run, in order.
+
+Over live packets made from VISUAL that are not filed, have a drawing number, and sit in a department in scope
+(feed setting `departments`; empty = every department). One item per drawing number (trimmed), the best priority
+winning:
+
+1. the VISUAL rev changed since the last lookup (`drawing_lookups.visual_rev_seen`), or a pending feed change
+   order waits for a drawing (the item is the CO's target drawing and rev; at most every 15 minutes);
+2. drafts still needing a drawing (`drawing_state = 'needed'`, or never looked up), unless a failed fetch set a
+   later `next_check_at`;
+3. failed lookups whose backoff ran out (`ok = false and next_check_at <= now`);
+4. rechecks due (`checked_at` older than `drawing_recheck_hours`).
+
+A packet with a hand drawing joins only through (1) or (4) once the feed has taken the hand drawing off; while
+`drawing_state = 'hand'` it is skipped.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Any
+
+BACKOFF_FIRST = timedelta(minutes=15)
+BACKOFF_MAX = timedelta(hours=24)
+CO_RECHECK = timedelta(minutes=15)  # a change order's target drawing is looked up at most this often
+
+QUEUE_SQL = """
+WITH own AS (
+    SELECT p.id AS packet_id, btrim(p.drawing) AS dwg, coalesce(btrim(p.rev), '') AS visual_rev, NULL::bigint AS co_id,
+        CASE
+            WHEN d.dwg IS NOT NULL AND d.checked_at IS NOT NULL
+                 AND d.visual_rev_seen IS DISTINCT FROM coalesce(btrim(p.rev), '') THEN 1
+            WHEN p.status = 'draft' AND (d.dwg IS NULL OR p.drawing_state = 'needed')
+                 AND (d.next_check_at IS NULL OR d.next_check_at <= %(now)s) THEN 2
+            WHEN d.ok = false AND (d.next_check_at IS NULL OR d.next_check_at <= %(now)s) THEN 3
+            WHEN d.checked_at IS NULL OR d.checked_at < %(recheck_before)s THEN 4
+        END AS priority,
+        d.checked_at
+    FROM packets p
+    LEFT JOIN drawing_lookups d ON d.dwg = btrim(p.drawing)
+    WHERE p.removed_at IS NULL AND p.source_kind = 'visual' AND p.status <> 'filed'
+      AND coalesce(btrim(p.drawing), '') <> '' AND p.drawing_state <> 'hand'
+      AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[]))
+),
+co AS (
+    SELECT p.id AS packet_id, btrim(c.waiting->>'dwg') AS dwg, coalesce(btrim(c.waiting->>'rev'), '') AS visual_rev,
+        c.id AS co_id, 1 AS priority, d.checked_at
+    FROM change_orders c
+    JOIN packets p ON p.id = c.packet_id
+    LEFT JOIN drawing_lookups d ON d.dwg = btrim(c.waiting->>'dwg')
+    WHERE c.review = 'pending' AND c.origin IN ('visual', 'drawing') AND c.waiting IS NOT NULL
+      AND jsonb_typeof(c.waiting) = 'object' AND coalesce(btrim(c.waiting->>'dwg'), '') <> ''
+      AND p.removed_at IS NULL AND p.status NOT IN ('draft', 'filed')
+      AND (d.checked_at IS NULL OR d.checked_at < %(co_before)s)
+      AND (d.ok IS NOT false OR d.next_check_at IS NULL OR d.next_check_at <= %(now)s)
+      AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[]))
+),
+items AS (
+    SELECT * FROM own WHERE priority IS NOT NULL
+    UNION ALL
+    SELECT * FROM co
+),
+best AS (
+    SELECT DISTINCT ON (dwg) packet_id, dwg, visual_rev, priority, co_id, checked_at
+    FROM items
+    ORDER BY dwg, priority, co_id NULLS LAST, packet_id
+)
+SELECT packet_id, dwg, visual_rev, priority, co_id
+FROM best
+ORDER BY priority, checked_at NULLS FIRST, dwg
+"""
+
+
+@dataclass(frozen=True, slots=True)
+class QueueItem:
+    packet_id: int
+    dwg: str
+    visual_rev: str
+    priority: int
+    co_id: int | None
+
+
+def backoff(fail_count: int) -> timedelta:
+    """15 minutes x 2^(fail_count - 1), at most 24 hours. fail_count counts this failure (1 = first)."""
+    n = max(0, int(fail_count) - 1)
+    if n >= 10:
+        return BACKOFF_MAX
+    return min(BACKOFF_FIRST * (2**n), BACKOFF_MAX)
+
+
+def params(feed_settings: Any, now: datetime) -> dict[str, Any]:
+    hours = int(getattr(feed_settings, "drawing_recheck_hours", 4))
+    depts = [str(d) for d in (getattr(feed_settings, "departments", None) or [])]
+    return {
+        "now": now,
+        "recheck_before": now - timedelta(hours=hours),
+        "co_before": now - CO_RECHECK,
+        "depts": depts,
+    }
+
+
+def queue(conn: Any, feed_settings: Any, now: datetime, *, limit: int | None = None) -> list[QueueItem]:
+    """The ordered lookup list. `limit` cuts it (the job passes the budget)."""
+    sql = QUEUE_SQL
+    p = params(feed_settings, now)
+    if limit is not None:
+        sql += " LIMIT %(limit)s"
+        p["limit"] = max(0, int(limit))
+    rows = conn.execute(sql, p).fetchall()
+    return [
+        QueueItem(int(r[0]), str(r[1]), str(r[2] or ""), int(r[3]), None if r[4] is None else int(r[4])) for r in rows
+    ]

server/packetstation/feed/__init__.py

@@ -0,0 +1 @@
+"""The VISUAL feed: settings, the read, the cache, candidates, fingerprints and the apply."""

server/packetstation/feed/cache.py

@@ -0,0 +1,564 @@
+"""The VISUAL cache in PostgreSQL: one batch per read, the rows and material lines as VISUAL last showed them,
+the orders and their links, labor per operation, and a change row for everything VISUAL added, changed, dropped
+or brought back.
+
+`write_batch` runs in one transaction. Rows and materials go in through COPY into a temp table, then an upsert
+keyed on the typed key (wo_base, lot, split, sub, seq[, piece]), all text but seq and piece. Rows in play that this
+read did not return get `gone_batch`, unless the batch is suspect: no window rows where the previous batch had
+some, or more than 5 tracked keys missing and more than 10 % of them.
+
+The apply reads back from here (`rows_for_apply`, `orders`, `labor_hours`), so it works on one consistent snapshot
+and keeps the last good orders when this run's order read failed.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Collection, Iterable, Mapping
+from dataclasses import dataclass, field
+from datetime import date
+from decimal import Decimal
+from typing import Any
+
+from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
+
+from .fingerprint import canonical, norm_decimal
+from .materials import material_from_row
+from .types import FeedData, Material, Order, Row, WoKey, num
+
+ROW_COLS = (
+    "wo_base",
+    "lot",
+    "split",
+    "sub",
+    "seq",
+    "res",
+    "dwg",
+    "rev",
+    "ps",
+    "descr",
+    "qty",
+    "tgt",
+    "rel",
+    "want",
+    "st",
+    "part",
+    "barcode",
+)
+MAT_COLS = (
+    "wo_base",
+    "lot",
+    "split",
+    "sub",
+    "seq",
+    "piece",
+    "part",
+    "qty",
+    "issued",
+    "whs",
+    "loc",
+    "st",
+    "istat",
+    "dwg",
+    "descr",
+    "comm",
+    "cons",
+)
+SUSPECT_MISSING_MIN = 5
+SUSPECT_MISSING_SHARE = Decimal("0.10")
+
+
+@dataclass
+class Batch:
+    id: int
+    suspect: bool
+    counts: dict[str, Any] = field(default_factory=dict)
+    reason: str = ""
+
+
+# ---------------------------------------------------------------- row dicts and hashes
+
+
+def _d(v: Any) -> str | None:
+    return v.isoformat() if isinstance(v, date) else None
+
+
+def row_doc(r: Row) -> dict[str, Any]:
+    """The row as a change-row dict: text values, numbers as text, dates ISO."""
+    return {
+        "wo": r.wo,
+        "lot": r.lot,
+        "split": r.split,
+        "sub": r.sub,
+        "seq": r.seq,
+        "res": r.res,
+        "dwg": r.dwg,
+        "rev": r.rev,
+        "ps": r.ps,
+        "desc": r.desc,
+        "qty": norm_decimal(r.qty),
+        "tgt": norm_decimal(r.tgt),
+        "rel": _d(r.rel),
+        "want": _d(r.want),
+        "st": r.st,
+        "part": r.part,
+        "barcode": r.barcode,
+    }
+
+
+def mat_doc(m: Material) -> dict[str, Any]:
+    return {
+        "wo": m.wo,
+        "lot": m.lot,
+        "split": m.split,
+        "sub": m.sub,
+        "seq": m.seq,
+        "piece": m.piece,
+        "part": m.part,
+        "qty": norm_decimal(m.qty),
+        "issued": norm_decimal(m.issued),
+        "whs": m.whs,
+        "loc": m.loc,
+        "st": m.st,
+        "istat": m.istat,
+        "dwg": m.dwg,
+        "desc": m.desc,
+        "comm": m.comm,
+        "cons": m.cons,
+    }
+
+
+def doc_hash(doc: Mapping[str, Any]) -> bytes:
+    return hashlib.sha256(canonical(dict(doc))).digest()
+
+
+# SQL that rebuilds the same dict from a cached row, for the change rows' `before`.
+_ROW_BEFORE = (
+    "jsonb_build_object('wo', v.wo_base, 'lot', v.lot, 'split', v.split, 'sub', v.sub, 'seq', v.seq, 'res', v.res,"
+    " 'dwg', v.dwg, 'rev', v.rev, 'ps', v.ps, 'desc', v.descr, 'qty', v.qty::text, 'tgt', v.tgt::text,"
+    " 'rel', v.rel, 'want', v.want, 'st', v.st, 'part', v.part, 'barcode', v.barcode)"
+)
+_MAT_BEFORE = (
+    "jsonb_build_object('wo', v.wo_base, 'lot', v.lot, 'split', v.split, 'sub', v.sub, 'seq', v.seq,"
+    " 'piece', v.piece, 'part', v.part, 'qty', v.qty::text, 'issued', v.issued::text, 'whs', v.whs, 'loc', v.loc,"
+    " 'st', v.st, 'istat', v.istat, 'dwg', v.dwg, 'desc', v.descr, 'comm', v.comm, 'cons', v.cons)"
+)
+
+
+def _key_arrays(keys: Iterable[WoKey]) -> tuple[list[str], list[str], list[str], list[str]]:
+    ks = list(dict.fromkeys(keys))
+    return [k.base for k in ks], [k.lot for k in ks], [k.split for k in ks], [k.sub for k in ks]
+
+
+_KEYS_UNNEST = "unnest(%s::text[], %s::text[], %s::text[], %s::text[]) AS k(b, l, sp, su)"
+
+
+# ---------------------------------------------------------------- write
+
+
+def is_suspect(window_rows: int, prev_window_rows: int | None, missing: int, tracked: int) -> str:
+    """The reason a batch is suspect, or ''."""
+    if window_rows == 0 and (prev_window_rows or 0) > 0:
+        return f"the window read returned nothing (the previous read had {prev_window_rows})"
+    if missing > SUSPECT_MISSING_MIN and Decimal(missing) > SUSPECT_MISSING_SHARE * tracked:
+        return f"{missing} of {tracked} tracked work orders missing"
+    return ""
+
+
+def write_batch(conn: Any, run_id: int | None, data: FeedData, source_desc: str, window: tuple[date, date]) -> Batch:
+    """Record one read: the batch row, rows, materials, orders, labor and the change rows. One transaction."""
+    since, until = window
+    with conn.transaction():
+        prev = conn.execute("SELECT window_rows FROM visual_batches ORDER BY id DESC LIMIT 1").fetchone()
+        missing = len(set(data.tracked_missing))
+        tracked_n = len(set(data.tracked)) or data.tracked_count
+        reason = is_suspect(data.window_count, prev[0] if prev else None, missing, tracked_n)
+        suspect = bool(reason)
+        server, _, database = source_desc.partition(" / ")
+        bid = int(
+            conn.execute(
+                "INSERT INTO visual_batches (run_id, server, database, site, since, until, window_rows, tracked_keys,"
+                " missing_keys, suspect) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id",
+                (
+                    run_id,
+                    str(data.meta.get("server") or server),
+                    str(data.meta.get("database") or database) or None,
+                    data.meta.get("site"),
+                    since,
+                    until,
+                    data.window_count,
+                    tracked_n,
+                    missing,
+                    suspect,
+                ),
+            ).fetchone()[0]
+        )
+        counts: dict[str, Any] = {"rows": len(data.rows), "suspect": int(suspect)}
+        if reason:
+            counts["suspect_reason"] = reason
+        counts.update(_write_rows(conn, bid, data, window, suspect))
+        if data.materials is not None:
+            counts.update(_write_materials(conn, bid, data, suspect))
+            counts["materials"] = len(data.materials)
+        if data.orders_by_base is not None and data.orders_by_project is not None:
+            counts["orders"] = _write_orders(conn, bid, data)
+        if data.labor is not None:
+            counts["labor_legs"] = _write_labor(conn, data)
+            counts["labor_read"] = 1
+        conn.execute("UPDATE visual_batches SET counts = %s WHERE id = %s", (Jsonb(counts), bid))
+    return Batch(id=bid, suspect=suspect, counts=counts, reason=reason)
+
+
+def _write_rows(conn: Any, bid: int, data: FeedData, window: tuple[date, date], suspect: bool) -> dict[str, int]:
+    uniq: dict[tuple[WoKey, int], Row] = {}
+    for r in data.rows:
+        uniq[(r.key, r.seq)] = r
+    conn.execute(
+        "CREATE TEMP TABLE t_rows (wo_base text, lot text, split text, sub text, seq int, res text, dwg text,"
+        " rev text, ps text, descr text, qty numeric, tgt numeric, rel date, want date, st text, part text,"
+        " barcode text, row_hash bytea, doc jsonb) ON COMMIT DROP"
+    )
+    with (
+        conn.cursor() as cur,
+        cur.copy(
+            "COPY t_rows (wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st, part,"
+            " barcode, row_hash, doc) FROM STDIN"
+        ) as cp,
+    ):
+        for r in uniq.values():
+            doc = row_doc(r)
+            cp.write_row(
+                (
+                    r.wo,
+                    r.lot,
+                    r.split,
+                    r.sub,
+                    r.seq,
+                    r.res,
+                    r.dwg,
+                    r.rev,
+                    r.ps,
+                    r.desc,
+                    r.qty,
+                    r.tgt,
+                    r.rel,
+                    r.want,
+                    r.st,
+                    r.part,
+                    r.barcode or None,
+                    doc_hash(doc),
+                    Jsonb(doc),
+                )
+            )
+    out: dict[str, int] = {}
+    join = (
+        "visual_rows v ON v.wo_base = t.wo_base AND v.lot = t.lot AND v.split = t.split AND v.sub = t.sub"
+        " AND v.seq = t.seq"
+    )
+    out["rows_added"] = conn.execute(
+        "INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, after)"  # noqa: S608
+        f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'added', t.doc FROM t_rows t LEFT JOIN {join}"
+        " WHERE v.id IS NULL",
+        (bid,),
+    ).rowcount
+    out["rows_returned"] = conn.execute(
+        "INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before, after)"  # noqa: S608
+        f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'returned', {_ROW_BEFORE}, t.doc FROM t_rows t"
+        f" JOIN {join} WHERE v.gone_batch IS NOT NULL",
+        (bid,),
+    ).rowcount
+    out["rows_changed"] = conn.execute(
+        "INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before, after)"  # noqa: S608
+        f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'changed', {_ROW_BEFORE}, t.doc FROM t_rows t"
+        f" JOIN {join} WHERE v.gone_batch IS NULL AND v.row_hash <> t.row_hash",
+        (bid,),
+    ).rowcount
+    conn.execute(
+        "INSERT INTO visual_rows (wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st,"
+        " part, barcode, row_hash, first_batch, last_batch)"
+        " SELECT wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st, part, barcode,"
+        " row_hash, %s, %s FROM t_rows"
+        " ON CONFLICT (wo_base, lot, split, sub, seq) DO UPDATE SET res = EXCLUDED.res, dwg = EXCLUDED.dwg,"
+        " rev = EXCLUDED.rev, ps = EXCLUDED.ps, descr = EXCLUDED.descr, qty = EXCLUDED.qty, tgt = EXCLUDED.tgt,"
+        " rel = EXCLUDED.rel, want = EXCLUDED.want, st = EXCLUDED.st, part = EXCLUDED.part,"
+        " barcode = EXCLUDED.barcode, row_hash = EXCLUDED.row_hash, last_batch = EXCLUDED.last_batch,"
+        " gone_batch = NULL",
+        (bid, bid),
+    )
+    out["rows_gone"] = 0
+    if not suspect:
+        b, lot, sp, su = _key_arrays(data.tracked)
+        out["rows_gone"] = conn.execute(
+            "WITH gone AS (UPDATE visual_rows v SET gone_batch = %s"  # noqa: S608
+            " WHERE v.gone_batch IS NULL AND v.last_batch <> %s"
+            f" AND (EXISTS (SELECT 1 FROM {_KEYS_UNNEST} WHERE k.b = v.wo_base AND k.l = v.lot AND k.sp = v.split"
+            " AND k.su = v.sub) OR (v.st = 'F' AND v.rel BETWEEN %s AND %s)) RETURNING v.*)"
+            " INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before)"
+            f" SELECT %s, v.wo_base, v.lot, v.split, v.sub, v.seq, 'gone', {_ROW_BEFORE} FROM gone v",
+            (bid, bid, b, lot, sp, su, window[0], window[1], bid),
+        ).rowcount
+    return out
+
+
+def _write_materials(conn: Any, bid: int, data: FeedData, suspect: bool) -> dict[str, int]:
+    uniq: dict[tuple[WoKey, int, int], Material] = {}
+    for m in data.materials or []:
+        uniq[(m.key, m.seq, m.piece)] = m
+    conn.execute(
+        "CREATE TEMP TABLE t_mats (wo_base text, lot text, split text, sub text, seq int, piece int, part text,"
+        " qty numeric, issued numeric, whs text, loc text, st text, istat int, dwg text, descr text, comm text,"
+        " cons text, mat_hash bytea, doc jsonb) ON COMMIT DROP"
+    )
+    with (
+        conn.cursor() as cur,
+        cur.copy(
+            "COPY t_mats (wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat, dwg, descr,"
+            " comm, cons, mat_hash, doc) FROM STDIN"
+        ) as cp,
+    ):
+        for m in uniq.values():
+            doc = mat_doc(m)
+            cp.write_row(
+                (
+                    m.wo,
+                    m.lot,
+                    m.split,
+                    m.sub,
+                    m.seq,
+                    m.piece,
+                    m.part,
+                    m.qty,
+                    m.issued,
+                    m.whs,
+                    m.loc,
+                    m.st,
+                    m.istat,
+                    m.dwg,
+                    m.desc,
+                    m.comm,
+                    m.cons,
+                    doc_hash(doc),
+                    Jsonb(doc),
+                )
+            )
+    join = (
+        "visual_materials v ON v.wo_base = t.wo_base AND v.lot = t.lot AND v.split = t.split AND v.sub = t.sub"
+        " AND v.seq = t.seq AND v.piece = t.piece"
+    )
+    cols = "batch_id, wo_base, lot, split, sub, seq, piece, kind"
+    out: dict[str, int] = {}
+    out["mats_added"] = conn.execute(
+        f"INSERT INTO visual_material_changes ({cols}, after)"  # noqa: S608
+        f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'added', t.doc FROM t_mats t LEFT JOIN {join}"
+        " WHERE v.id IS NULL",
+        (bid,),
+    ).rowcount
+    out["mats_returned"] = conn.execute(
+        f"INSERT INTO visual_material_changes ({cols}, before, after)"  # noqa: S608
+        f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'returned', {_MAT_BEFORE}, t.doc"
+        f" FROM t_mats t JOIN {join} WHERE v.gone_batch IS NOT NULL",
+        (bid,),
+    ).rowcount
+    out["mats_changed"] = conn.execute(
+        f"INSERT INTO visual_material_changes ({cols}, before, after)"  # noqa: S608
+        f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'changed', {_MAT_BEFORE}, t.doc"
+        f" FROM t_mats t JOIN {join} WHERE v.gone_batch IS NULL AND v.mat_hash <> t.mat_hash",
+        (bid,),
+    ).rowcount
+    conn.execute(
+        "INSERT INTO visual_materials (wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat,"
+        " dwg, descr, comm, cons, mat_hash, first_batch, last_batch)"
+        " SELECT wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat, dwg, descr, comm,"
+        " cons, mat_hash, %s, %s FROM t_mats"
+        " ON CONFLICT (wo_base, lot, split, sub, seq, piece) DO UPDATE SET part = EXCLUDED.part, qty = EXCLUDED.qty,"
+        " issued = EXCLUDED.issued, whs = EXCLUDED.whs, loc = EXCLUDED.loc, st = EXCLUDED.st,"
+        " istat = EXCLUDED.istat, dwg = EXCLUDED.dwg, descr = EXCLUDED.descr, comm = EXCLUDED.comm,"
+        " cons = EXCLUDED.cons, mat_hash = EXCLUDED.mat_hash, last_batch = EXCLUDED.last_batch, gone_batch = NULL",
+        (bid, bid),
+    )
+    out["mats_gone"] = 0
+    if not suspect:
+        bases = sorted({r.wo for r in data.rows} | {m.wo for m in data.materials or []})
+        out["mats_gone"] = conn.execute(
+            "WITH gone AS (UPDATE visual_materials v SET gone_batch = %s WHERE v.gone_batch IS NULL"  # noqa: S608
+            " AND v.last_batch <> %s AND v.wo_base = ANY(%s) RETURNING v.*)"
+            f" INSERT INTO visual_material_changes ({cols}, before)"
+            f" SELECT %s, v.wo_base, v.lot, v.split, v.sub, v.seq, v.piece, 'gone', {_MAT_BEFORE} FROM gone v",
+            (bid, bid, bases, bid),
+        ).rowcount
+    return out
+
+
+def _write_orders(conn: Any, bid: int, data: FeedData) -> int:
+    by_base = data.orders_by_base or {}
+    by_project = data.orders_by_project or {}
+    every: dict[str, Order] = {}
+    for lists in (by_base, by_project):
+        for orders in lists.values():
+            for o in orders:
+                every.setdefault(o.cust_order_id, o)
+    for o in every.values():
+        conn.execute(
+            "INSERT INTO visual_orders (cust_order_id, status, customer_id, customer_name, customer_po_ref,"
+            " order_date, desired_ship_date, promise_date, last_batch) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)"
+            " ON CONFLICT (cust_order_id) DO UPDATE SET status = EXCLUDED.status, customer_id = EXCLUDED.customer_id,"
+            " customer_name = EXCLUDED.customer_name, customer_po_ref = EXCLUDED.customer_po_ref,"
+            " order_date = coalesce(EXCLUDED.order_date, visual_orders.order_date),"
+            " desired_ship_date = EXCLUDED.desired_ship_date, promise_date = EXCLUDED.promise_date,"
+            " last_batch = EXCLUDED.last_batch",
+            (
+                o.cust_order_id,
+                o.status,
+                o.customer_id,
+                o.customer_name,
+                o.customer_po_ref,
+                o.order_date,
+                o.desired_ship_date,
+                o.promise_date,
+                bid,
+            ),
+        )
+    bases = sorted({r.wo for r in data.rows} | set(by_base))
+    projects = sorted({r.wo.partition("-")[0] or r.wo for r in data.rows} | set(by_project))
+    conn.execute(
+        "DELETE FROM visual_order_links WHERE (kind = 'base' AND key = ANY(%s))"
+        " OR (kind = 'project' AND key = ANY(%s))",
+        (bases, projects),
+    )
+    n = 0
+    for kind, lists in (("base", by_base), ("project", by_project)):
+        for key, orders in lists.items():
+            seen: set[str] = set()
+            for rank, o in enumerate(orders):
+                if o.cust_order_id in seen:
+                    continue
+                seen.add(o.cust_order_id)
+                conn.execute(
+                    "INSERT INTO visual_order_links (kind, key, cust_order_id, rank, lines, top_jobs, last_batch)"
+                    " VALUES (%s, %s, %s, %s, %s, %s, %s)",
+                    (kind, key, o.cust_order_id, rank, o.lines, o.top_jobs, bid),
+                )
+                n += 1
+    return len(every)
+
+
+def _write_labor(conn: Any, data: FeedData) -> int:
+    n = 0
+    for lg in data.labor or []:
+        conn.execute(
+            "INSERT INTO visual_labor (wo_base, lot, split, sub, seq, hours, tickets, last_ticket, observed_at)"
+            " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now())"
+            " ON CONFLICT (wo_base, lot, split, sub, seq) DO UPDATE SET hours = EXCLUDED.hours,"
+            " tickets = EXCLUDED.tickets, last_ticket = EXCLUDED.last_ticket, observed_at = now()",
+            (lg.wo, lg.lot, lg.split, lg.sub, lg.seq, lg.hours, lg.tickets, lg.last_ticket),
+        )
+        n += 1
+    return n
+
+
+# ---------------------------------------------------------------- read back
+
+
+def row_from_cache(r: Mapping[str, Any]) -> Row:
+    return Row(
+        wo=r["wo_base"],
+        lot=r["lot"],
+        split=r["split"],
+        sub=r["sub"],
+        seq=int(r["seq"]),
+        res=r["res"],
+        dwg=r["dwg"],
+        rev=r["rev"],
+        ps=r["ps"],
+        desc=r["descr"],
+        qty=num(r["qty"]),
+        tgt=num(r["tgt"]),
+        rel=r["rel"],
+        want=r["want"],
+        st=r["st"],
+        part=r["part"],
+        barcode=r["barcode"] or "",
+    )
+
+
+def rows_for_apply(conn: Any, keys_in_play: Collection[WoKey]) -> tuple[list[Row], list[Material], set[WoKey]]:
+    """Live cached rows and material lines of every work order (base and lot) among these keys, so folded
+    operations and planner instructions of the same work order come along. material_legs = legs with live
+    material lines whose status is F or R."""
+    pairs = sorted({(k.base, k.lot) for k in keys_in_play})
+    if not pairs:
+        return [], [], set()
+    bases = [p[0] for p in pairs]
+    lots = [p[1] for p in pairs]
+    sql_join = "JOIN unnest(%s::text[], %s::text[]) AS k(b, l) ON v.wo_base = k.b AND v.lot = k.l"
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute(
+            f"SELECT v.* FROM visual_rows v {sql_join} WHERE v.gone_batch IS NULL"  # noqa: S608
+            " ORDER BY v.wo_base, v.lot, v.split, v.sub, v.seq",
+            (bases, lots),
+        ).fetchall()
+        mats = cur.execute(
+            f"SELECT v.* FROM visual_materials v {sql_join} WHERE v.gone_batch IS NULL"  # noqa: S608
+            " ORDER BY v.wo_base, v.lot, v.split, v.sub, v.seq, v.piece",
+            (bases, lots),
+        ).fetchall()
+    materials = [material_from_row(m) for m in mats]
+    legs = {m.key for m in materials if m.st in ("F", "R")}
+    return [row_from_cache(r) for r in rows], materials, legs
+
+
+def orders(
+    conn: Any, *, bases: Iterable[str] | None = None, projects: Iterable[str] | None = None
+) -> tuple[dict[str, list[Order]], dict[str, list[Order]]]:
+    """The cached order links as (by_base, by_project), in rank order. Without filters: every link."""
+    params: list[Any] = []
+    where = ""
+    if bases is not None or projects is not None:
+        where = " WHERE (l.kind = 'base' AND l.key = ANY(%s)) OR (l.kind = 'project' AND l.key = ANY(%s))"
+        params = [sorted(set(bases or ())), sorted(set(projects or ()))]
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute(
+            "SELECT l.kind, l.key, l.rank, l.lines, l.top_jobs, o.* FROM visual_order_links l"  # noqa: S608
+            f" JOIN visual_orders o ON o.cust_order_id = l.cust_order_id{where} ORDER BY l.kind, l.key, l.rank",
+            params,
+        ).fetchall()
+    by_base: dict[str, list[Order]] = {}
+    by_project: dict[str, list[Order]] = {}
+    for r in rows:
+        o = Order(
+            cust_order_id=r["cust_order_id"],
+            status=r["status"],
+            customer_id=r["customer_id"],
+            customer_name=r["customer_name"],
+            customer_po_ref=r["customer_po_ref"],
+            order_date=r["order_date"],
+            desired_ship_date=r["desired_ship_date"],
+            promise_date=r["promise_date"],
+            lines=list(r["lines"] or []),
+            top_jobs=list(r["top_jobs"] or []),
+        )
+        (by_base if r["kind"] == "base" else by_project).setdefault(r["key"], []).append(o)
+    return by_base, by_project
+
+
+def labor_hours(conn: Any) -> dict[tuple[WoKey, int], Decimal]:
+    """Cached ticket hours per (leg, seq)."""
+    out: dict[tuple[WoKey, int], Decimal] = {}
+    for r in conn.execute("SELECT wo_base, lot, split, sub, seq, hours FROM visual_labor"):
+        out[(WoKey(r[0], r[1], r[2], r[3]), int(r[4]))] = num(r[5])
+    return out
+
+
+def last_labor_read(conn: Any) -> Any:
+    """When labor was last read (a batch that carried labor), or None."""
+    r = conn.execute("SELECT max(started_at) FROM visual_batches WHERE counts ? 'labor_read'").fetchone()
+    return r[0] if r else None
+
+
+def finish_batch(conn: Any, batch_id: int, counts: Mapping[str, Any]) -> None:
+    """Merge the apply's counts into the batch row and set finished_at."""
+    conn.execute(
+        "UPDATE visual_batches SET finished_at = now(), counts = counts || %s WHERE id = %s",
+        (Jsonb(dict(counts)), batch_id),
+    )

server/packetstation/feed/candidates.py

@@ -0,0 +1,243 @@
+"""VISUAL rows to packet candidates: the old visBuild, typed.
+
+One candidate = unit + resource + drawing, holding every work order operation on that drawing. The key is
+`f"{unit}|{res}|{dwg or '(no drawing)'}"`, which is `packets.visual_key`. Project = the work order base before its
+first dash; unit = project + the lot zero-padded to 3. A non-numeric lot (7A) makes no candidate, as the old
+build: it is counted in `Built.skipped_lots`. Resources route through the resource map: a shop department code,
+`fold` (rides along on the packet of the same work order and lot), `ignore`, or no entry (counted as unmapped).
+`MAT_*` with no map entry folds.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Collection, Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from datetime import date
+from decimal import Decimal
+
+from ..text import RX_LOT, RX_PROJECT
+from . import materials as mats
+from .materials import BomLine
+from .types import Material, Row, WoKey
+
+FOLD = "fold"
+IGNORE = "ignore"
+NO_DRAWING = "(no drawing)"
+
+__all__ = [
+    "FOLD",
+    "IGNORE",
+    "NO_DRAWING",
+    "BomLine",
+    "Built",
+    "Candidate",
+    "Folded",
+    "Leg",
+    "build",
+    "project_of",
+    "route",
+    "unit_number",
+]
+
+
+@dataclass(slots=True)
+class Leg:
+    """One work order operation on the packet."""
+
+    key: WoKey
+    seq: int
+    op: str
+    qty: Decimal
+    tgt: Decimal
+    dwg: str
+    rev: str
+    partspecs: str
+    st: str
+
+
+@dataclass(slots=True)
+class Folded:
+    key: WoKey
+    seq: int
+    op: str
+    tgt: Decimal
+
+
+@dataclass(slots=True)
+class Candidate:
+    key: str  # f"{unit}|{res}|{dwg or '(no drawing)'}" == packets.visual_key
+    project: str
+    unit: str
+    lot: str
+    dept: str
+    op: str
+    drawing: str
+    rev: str
+    title: str
+    legs: list[Leg]
+    folded: list[Folded]
+    partspecs: list[str]
+    rel: date | None
+    want: date | None
+    bom: list[BomLine] | None  # None when materials are unknown for this candidate
+    statuses: dict[WoKey, str]  # header status per leg key
+    base: str  # the first leg's wo base, for the order lookup
+    in_window: bool  # at least one leg came from the window read
+
+    @property
+    def mat_known(self) -> bool:
+        return self.bom is not None
+
+    @property
+    def leg_keys(self) -> set[WoKey]:
+        return {lg.key for lg in self.legs}
+
+
+@dataclass(slots=True)
+class Built:
+    candidates: dict[str, Candidate]
+    unmapped: dict[str, int]
+    ignored: list[Row]  # rows below the go-live date (not tracked)
+    skipped_lots: dict[str, int] = field(default_factory=dict)  # f"{wo}/{lot}" -> rows (non-numeric lot)
+
+
+def route(res: str, resource_map: Mapping[str, str]) -> str | None:
+    """'fold' | 'ignore' | a dept code | None (no map entry). MAT_* with no entry folds."""
+    if res in resource_map:
+        return resource_map[res]
+    if res.upper().startswith("MAT_"):
+        return FOLD
+    return None
+
+
+def project_of(wo: str) -> str:
+    """'L489-05-5GA' -> 'L489' (upper-cased); a base with no dash is its own project."""
+    head = wo.partition("-")[0].strip()
+    return (head or wo).upper()
+
+
+def unit_number(project: str, lot: str) -> str | None:
+    """project + lot zero-padded to 3 when the lot is numeric; None for a non-numeric lot (the old unit_no)."""
+    lot = lot.strip()
+    if not lot.isdigit():
+        return None
+    padded = lot.zfill(3)
+    if not RX_LOT.match(padded):
+        return None
+    return project + padded
+
+
+def _rev_wins(new: str, cur: str) -> bool:
+    """The old rule: a numeric rev higher than the current wins; anything wins over an empty current."""
+    if not new or new == cur:
+        return False
+    if not cur:
+        return True
+    try:
+        return float(new) > float(cur)
+    except ValueError:
+        return False
+
+
+def build(
+    rows: Sequence[Row],
+    materials: Iterable[Material] | None,
+    material_legs: Collection[WoKey] | None,
+    *,
+    resource_map: Mapping[str, str],
+    window: tuple[date, date],
+    golive: date | None,
+    departments: Collection[str],
+    tracked_keys: Collection[WoKey],
+    today: date | None = None,
+) -> Built:
+    """Group rows into candidates. See the module docstring and phase3-design "Candidates"."""
+    del today  # kept for the contract signature; the window already carries it
+    tracked = set(tracked_keys)
+    depts = set(departments)
+    by_wo_lot: dict[tuple[str, str], list[Row]] = {}
+    for r in rows:
+        by_wo_lot.setdefault((r.wo, r.lot), []).append(r)
+
+    cands: dict[str, Candidate] = {}
+    unmapped: dict[str, int] = {}
+    ignored: list[Row] = []
+    skipped_lots: dict[str, int] = {}
+    leg_seen: dict[str, set[tuple[WoKey, int]]] = {}
+    fold_seen: dict[str, set[tuple[WoKey, int]]] = {}
+
+    for r in rows:
+        target = route(r.res, resource_map)
+        if target in (FOLD, IGNORE):
+            continue
+        is_tracked = r.key in tracked
+        in_window = r.st == "F" and r.rel is not None and window[0] <= r.rel <= window[1]
+        if not is_tracked and not in_window:
+            continue
+        if target is None:
+            unmapped[r.res] = unmapped.get(r.res, 0) + 1
+            continue
+        if depts and target not in depts:
+            continue
+        if not is_tracked and golive is not None and r.rel is not None and r.rel < golive:
+            ignored.append(r)
+            continue
+        project = project_of(r.wo)
+        unit = unit_number(project, r.lot) if RX_PROJECT.match(project) else None
+        if unit is None:
+            k = f"{r.wo}/{r.lot}"
+            skipped_lots[k] = skipped_lots.get(k, 0) + 1
+            continue
+        key = f"{unit}|{r.res}|{r.dwg or NO_DRAWING}"
+        c = cands.get(key)
+        if c is None:
+            c = Candidate(
+                key=key,
+                project=project,
+                unit=unit,
+                lot=unit[len(project) :],
+                dept=target,
+                op=r.res,
+                drawing=r.dwg,
+                rev=r.rev,
+                title=r.desc,
+                legs=[],
+                folded=[],
+                partspecs=[],
+                rel=r.rel,
+                want=r.want,
+                bom=None,
+                statuses={},
+                base=r.wo,
+                in_window=False,
+            )
+            cands[key] = c
+            leg_seen[key] = set()
+            fold_seen[key] = set()
+        c.in_window = c.in_window or in_window
+        if (r.key, r.seq) not in leg_seen[key]:
+            leg_seen[key].add((r.key, r.seq))
+            c.legs.append(
+                Leg(key=r.key, seq=r.seq, op=r.res, qty=r.qty, tgt=r.tgt, dwg=r.dwg, rev=r.rev, partspecs=r.ps, st=r.st)
+            )
+        c.statuses[r.key] = r.st
+        if r.rel is not None and (c.rel is None or r.rel < c.rel):
+            c.rel = r.rel
+        if r.want is not None and (c.want is None or r.want > c.want):
+            c.want = r.want
+        if _rev_wins(r.rev, c.rev):
+            c.rev = r.rev
+        for o in by_wo_lot.get((r.wo, r.lot), []):
+            if route(o.res, resource_map) == FOLD and (o.key, o.seq) not in fold_seen[key]:
+                fold_seen[key].add((o.key, o.seq))
+                c.folded.append(Folded(key=o.key, seq=o.seq, op=o.res, tgt=o.tgt))
+            if o.ps and o.ps not in c.partspecs:
+                c.partspecs.append(o.ps)
+
+    if material_legs is not None:
+        idx = mats.index(materials)
+        known = set(material_legs)
+        for c in cands.values():
+            if any(lg.key in known for lg in c.legs):
+                c.bom = mats.bom_for(c.legs, idx)
+    return Built(candidates=cands, unmapped=unmapped, ignored=ignored, skipped_lots=skipped_lots)

server/packetstation/feed/changes.py

@@ -0,0 +1,382 @@
+"""Change orders opened by the feed (origin visual) and by the drawing job (origin drawing). The old visCoNeed,
+coAutoChanges and visDiff.
+
+At most one pending feed change order per packet, whatever its origin (visual or drawing): when one is pending
+it is re-aimed (reason, vis, waiting, staged, target, minor) instead of a second one being opened. A drawing
+re-release that meets a pending visual change order stages the new drawing on it and adds its line to the reason.
+The required departments are filled at approval (the API's co_touched), not here.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from datetime import UTC, date, datetime
+from typing import TYPE_CHECKING, Any
+
+from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
+
+from ..domain.pipeline import next_counter
+from ..text import LIMITS, clean_text
+from .fingerprint import bom_sig, drawing_key, leg_key, norm_decimal
+from .writes import SysWrite, packets_jsonable
+
+if TYPE_CHECKING:
+    from .candidates import Candidate
+
+FEED_ORIGINS = ("visual", "drawing")
+ECN = {"visual": "VISUAL", "drawing": "M-FILES"}
+
+
+@dataclass
+class CoResult:
+    co_id: int
+    co_no: int
+    fresh: bool
+    reaimed: bool
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat(timespec="seconds")
+
+
+def pending_visual(conn: Any, packet_id: int, *, lock: bool = False) -> dict[str, Any] | None:
+    """The single pending change order with origin visual or drawing, or None."""
+    sql = (
+        "SELECT *, origin::text AS origin, review::text AS review FROM change_orders"
+        " WHERE packet_id = %s AND origin IN ('visual', 'drawing') AND review = 'pending' ORDER BY co_no DESC LIMIT 1"
+    )
+    if lock:
+        sql += " FOR UPDATE"
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(sql, (packet_id,)).fetchone()
+    return dict(r) if r else None
+
+
+def last_feed_co(conn: Any, packet_id: int) -> dict[str, Any] | None:
+    """The newest change order with origin visual, any review."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(
+            "SELECT *, origin::text AS origin, review::text AS review FROM change_orders"
+            " WHERE packet_id = %s AND origin = 'visual' ORDER BY co_no DESC LIMIT 1",
+            (packet_id,),
+        ).fetchone()
+    return dict(r) if r else None
+
+
+def _j(value: Any) -> Any:
+    return None if value is None else Jsonb(packets_jsonable(value))
+
+
+def _same_target(old_vis: Mapping[str, Any] | None, vis: Mapping[str, Any]) -> bool:
+    o = old_vis or {}
+    return drawing_key(o.get("to_dwg")) == drawing_key(vis.get("to_dwg")) and str(o.get("to_rev") or "") == str(
+        vis.get("to_rev") or ""
+    )
+
+
+def open_or_reaim(
+    sw: SysWrite,
+    *,
+    origin: str,
+    reason: str,
+    vis: dict[str, Any],
+    waiting: dict[str, Any] | None,
+    staged: dict[str, Any] | None,
+    target_fp: str | None,
+    minor: bool,
+    required: list[str] | None = None,
+) -> CoResult:
+    """Open the packet's feed change order, or re-aim the pending one. Call it inside `system_write`."""
+    del required  # filled at approval by the API (co_touched)
+    if origin not in FEED_ORIGINS:
+        raise ValueError(f"origin must be visual or drawing, not {origin!r}")
+    conn = sw.conn
+    reason = clean_text(reason, LIMITS["reason"])
+    pend = pending_visual(conn, sw.id, lock=True)
+    if pend is not None:
+        old_vis = dict(pend.get("vis") or {})
+        moved = list(old_vis.get("moved") or [])
+        if origin == "drawing" and pend["origin"] == "visual":
+            # A drawing re-release meets a pending VISUAL change: stage the drawing on it, keep its content.
+            new_vis = old_vis
+            new_vis["moved"] = [*moved, {"at": _now(), "rev": vis.get("to_rev"), "dwg": vis.get("to_dwg")}]
+            old_reason = str(pend.get("reason") or "")
+            new_reason = (
+                old_reason if reason in old_reason else clean_text(f"{old_reason} · {reason}", LIMITS["reason"])
+            )
+            conn.execute(
+                "UPDATE change_orders SET reason = %s, vis = %s, staged = coalesce(%s, staged),"
+                " waiting = CASE WHEN %s::jsonb IS NULL THEN waiting ELSE NULL END, row_version = row_version + 1"
+                " WHERE id = %s",
+                (new_reason, _j(new_vis), _j(staged), _j(staged), pend["id"]),
+            )
+            label = f"Change order {pend['co_no']}: drawing from M-Files staged"
+        else:
+            new_vis = dict(vis)
+            new_vis["moved"] = [*moved, {"at": _now(), "rev": vis.get("to_rev"), "dwg": vis.get("to_dwg")}]
+            keep = staged is None and pend.get("staged") is not None and _same_target(old_vis, vis)
+            new_staged = staged if staged is not None else (pend.get("staged") if keep else None)
+            new_waiting = None if (keep or staged is not None) else waiting
+            conn.execute(
+                "UPDATE change_orders SET origin = %s::co_origin, reason = %s, vis = %s, waiting = %s, staged = %s,"
+                " target_fp = %s, minor = %s, ecn = %s, row_version = row_version + 1 WHERE id = %s",
+                (
+                    origin,
+                    reason,
+                    _j(new_vis),
+                    _j(new_waiting),
+                    _j(new_staged),
+                    target_fp,
+                    minor,
+                    ECN[origin],
+                    pend["id"],
+                ),
+            )
+            label = (
+                f"Change order {pend['co_no']} updated: VISUAL changed again before Clerical approved"
+                if origin == "visual"
+                else f"Change order {pend['co_no']} updated: new drawing from M-Files"
+            )
+        sw.log(label, reason)
+        sw.changed("change_orders")
+        sw.detail.update({"co_no": pend["co_no"], "co_id": pend["id"], "reaimed": True})
+        return CoResult(co_id=int(pend["id"]), co_no=int(pend["co_no"]), fresh=False, reaimed=True)
+
+    co_no = next_counter(conn, sw.id, "co")
+    row = conn.execute(
+        "INSERT INTO change_orders (packet_id, co_no, origin, review, minor, ecn, reason, issued_by, vis, waiting,"
+        " staged, target_fp) VALUES (%s, %s, %s::co_origin, 'pending', %s, %s, %s, NULL, %s, %s, %s, %s) RETURNING id",
+        (sw.id, co_no, origin, minor, ECN[origin], reason, _j(vis), _j(waiting), _j(staged), target_fp),
+    ).fetchone()
+    label = (
+        f"Change order {co_no}: change from VISUAL waiting on Clerical approval"
+        if origin == "visual"
+        else f"Change order {co_no}: new drawing from M-Files waiting on Clerical approval"
+    )
+    sw.log(label, reason)
+    sw.changed("change_orders")
+    sw.detail.update({"co_no": co_no, "co_id": int(row[0]), "fresh": True})
+    return CoResult(co_id=int(row[0]), co_no=co_no, fresh=True, reaimed=False)
+
+
+def withdraw(sw: SysWrite, co_id: int, reason: str) -> None:
+    """A pending feed change order VISUAL no longer asks for: review -> rejected by nobody, with the reason."""
+    reason = clean_text(reason, LIMITS["reason"])
+    r = sw.conn.execute(
+        "UPDATE change_orders SET review = 'rejected', rejected_at = now(), rejected_by = NULL, rejected_reason = %s,"
+        " row_version = row_version + 1 WHERE id = %s AND packet_id = %s AND review = 'pending' RETURNING co_no",
+        (reason, co_id, sw.id),
+    ).fetchone()
+    if r is None:
+        sw.skip()
+        return
+    sw.log(f"Change order {r[0]} withdrawn", reason)
+    sw.changed("change_orders")
+    sw.detail.update({"co_no": int(r[0]), "co_id": co_id, "withdrawn": True})
+
+
+# ---------------------------------------------------------------- the human diff (the old visDiff + matDiff)
+
+
+def packet_view(conn: Any, packet_id: int) -> dict[str, Any]:
+    """What the packet holds now, for `diff_text`: legs, rev, drawing, want, the VISUAL BOM and manual_bom."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        p = cur.execute(
+            "SELECT rev, drawing, visual_want, visual, status::text AS status FROM packets WHERE id = %s",
+            (packet_id,),
+        ).fetchone()
+        wos = cur.execute(
+            "SELECT wo_base, lot, split, sub, seq, qty, target_hrs FROM packet_work_orders WHERE packet_id = %s"
+            " ORDER BY sort, id",
+            (packet_id,),
+        ).fetchall()
+        bom = cur.execute(
+            "SELECT coalesce(cl_item, item::text) AS item, part_id, qty, src FROM bom_lines WHERE packet_id = %s"
+            " ORDER BY sort, item",
+            (packet_id,),
+        ).fetchall()
+    p = dict(p or {})
+    visual = p.get("visual") or {}
+    return {
+        "status": p.get("status"),
+        "rev": p.get("rev"),
+        "drawing": p.get("drawing"),
+        "want": p.get("visual_want"),
+        "legs": [dict(w) for w in wos],
+        "bom": [dict(b) for b in bom],
+        "manual_bom": bool(visual.get("manual_bom")),
+    }
+
+
+def _n(v: Any) -> str:
+    return norm_decimal(v) or "0"
+
+
+def _items(items: Sequence[str]) -> str:
+    head = ", ".join(items[:8])
+    return head + ("..." if len(items) > 8 else "")
+
+
+def mat_diff(old: Sequence[Mapping[str, Any]], new: Sequence[Any]) -> list[str]:
+    """The old matDiff: materials added, removed and changed, item by item."""
+    o = {str(b["item"]): b for b in old if b.get("src") == "visual"}
+    n = {b.item: b for b in new}
+    hand = sum(1 for b in old if b.get("src") != "visual")
+    if not o:
+        if not (hand or n):
+            return []
+        if hand:
+            return [f"BOM by hand ({hand} line{'' if hand == 1 else 's'}) replaced by VISUAL ({len(n)} lines)"]
+        return [f"materials: {len(n)} line{'' if len(n) == 1 else 's'} from VISUAL"]
+    add = [k for k in n if k not in o]
+    rem = [k for k in o if k not in n]
+    chg = [
+        k
+        for k in n
+        if k in o and (str(o[k].get("part_id") or "") != n[k].part_id or _n(o[k].get("qty")) != _n(n[k].qty))
+    ]
+    d: list[str] = []
+    if add:
+        d.append(f"materials added: item{'s' if len(add) > 1 else ''} {_items(add)}")
+    if rem:
+        d.append(f"materials removed in VISUAL: item{'s' if len(rem) > 1 else ''} {_items(rem)}")
+    for k in chg[:8]:
+        a, b = o[k], n[k]
+        if str(a.get("part_id") or "") != b.part_id:
+            d.append(f"item {k}: {a.get('part_id') or '-'} -> {b.part_id or '-'}")
+        else:
+            d.append(f"item {k}: qty {_n(a.get('qty'))} -> {_n(b.qty)}")
+    if len(chg) > 8:
+        more = len(chg) - 8
+        d.append(f"{more} more material change{'' if more == 1 else 's'}")
+    return d
+
+
+def diff_text(before: Mapping[str, Any], c: Candidate) -> list[str]:
+    """Human lines for what VISUAL changed on a packet (`before` is `packet_view`)."""
+    d: list[str] = []
+    old = {
+        leg_key(w["wo_base"], w["lot"], w["split"], w["sub"], w["seq"]): w
+        for w in before.get("legs") or []
+        if w.get("seq") is not None
+    }
+    new_keys: set[str] = set()
+    for lg in c.legs:
+        k = leg_key(lg.key.base, lg.key.lot, lg.key.split, lg.key.sub, lg.seq)
+        new_keys.add(k)
+        o = old.get(k)
+        if o is None:
+            d.append(f"work order {lg.key.base} added")
+            continue
+        if _n(o.get("target_hrs")) != _n(lg.tgt):
+            d.append(f"{lg.key.base} target {_n(o.get('target_hrs'))} h -> {_n(lg.tgt)} h")
+        if _n(o.get("qty")) != _n(lg.qty):
+            d.append(f"{lg.key.base} qty {_n(o.get('qty'))} -> {_n(lg.qty)}")
+    for k, o in old.items():
+        if k not in new_keys:
+            d.append(f"work order {o['wo_base']} no longer in VISUAL")
+    if drawing_key(before.get("drawing")) != drawing_key(c.drawing) and c.drawing:
+        d.append(f"drawing {before.get('drawing') or '-'} -> {c.drawing}")
+    if str(before.get("rev") or "") != str(c.rev or ""):
+        d.append(f"drawing rev {before.get('rev') or '-'} -> {c.rev or '-'}")
+    want_old = before.get("want")
+    want_old_s = want_old.isoformat() if isinstance(want_old, date) else str(want_old or "")
+    want_new_s = c.want.isoformat() if c.want else ""
+    if want_old_s != want_new_s:
+        d.append(f"want date {want_old_s or '-'} -> {want_new_s or '-'}")
+    hand = bool(before.get("manual_bom")) and (before.get("status") == "draft" or not c.bom)
+    if c.bom is not None and not hand and bom_sig(view_bom(before)) != bom_sig(c.bom):
+        d.extend(mat_diff(before.get("bom") or [], c.bom))
+    return d
+
+
+def view_bom(before: Mapping[str, Any]) -> list[dict[str, Any]]:
+    """The packet's BOM in matSig shape. Hand lines get a prefix so they always differ from VISUAL's."""
+    out = []
+    for b in before.get("bom") or []:
+        item = str(b.get("item"))
+        out.append(
+            {
+                "item": item if b.get("src") == "visual" else f"dwg:{item}",
+                "part_id": b.get("part_id"),
+                "qty": b.get("qty"),
+            }
+        )
+    return out
+
+
+def vis_for(
+    c: Candidate, fp_parts: Mapping[str, str], before: Mapping[str, Any], changes: list[str], target_fp: str
+) -> dict[str, Any]:
+    """The change order `vis` payload that approval applies."""
+    from_dwg = str(before.get("drawing") or "")
+    from_rev = str(before.get("rev") or "")
+    to_dwg = c.drawing or from_dwg
+    to_rev = c.rev or ""
+    dwg_change = (bool(to_rev) and to_rev != from_rev) or (
+        bool(c.drawing) and drawing_key(c.drawing) != drawing_key(from_dwg)
+    )
+    return {
+        "from_dwg": from_dwg,
+        "from_rev": from_rev,
+        "to_dwg": to_dwg,
+        "to_rev": to_rev,
+        "dwg_change": dwg_change,
+        "changes": [x for x in changes if not x.startswith("drawing rev ")],
+        "sig": target_fp,
+        "legs": legs_json(c),
+        "bom": [b.as_dict() for b in c.bom] if c.bom is not None else None,
+        "want": c.want.isoformat() if c.want else None,
+        "rel": c.rel.isoformat() if c.rel else None,
+        "folded": folded_json(c),
+        "partspecs": list(c.partspecs),
+        "fp_parts": dict(fp_parts),
+    }
+
+
+def co_reason(vis: Mapping[str, Any]) -> str:
+    """The change order reason, as the old coAutoChanges wrote it."""
+    if vis.get("dwg_change"):
+        fd, td = str(vis.get("from_dwg") or ""), str(vis.get("to_dwg") or "")
+        what = f"{fd} -> {td}" if fd and drawing_key(fd) != drawing_key(td) else td
+        return f"VISUAL changed the drawing: {what} Rev {vis.get('from_rev') or '?'} -> Rev {vis.get('to_rev')}"
+    other = list(vis.get("changes") or [])
+    more = f" (+{len(other) - 3} more)" if len(other) > 3 else ""
+    return f"VISUAL changed the packet: {'; '.join(other[:3])}{more}"
+
+
+def legs_json(c: Candidate) -> list[dict[str, Any]]:
+    return [
+        {
+            "wo_base": lg.key.base,
+            "lot": lg.key.lot,
+            "split": lg.key.split,
+            "sub": lg.key.sub,
+            "seq": lg.seq,
+            "op": lg.op,
+            "qty": str(lg.qty),
+            "target_hrs": str(lg.tgt),
+            "dwg": lg.dwg,
+            "rev": lg.rev,
+            "partspecs": lg.partspecs,
+            "st": lg.st,
+        }
+        for lg in c.legs
+    ]
+
+
+def folded_json(c: Candidate) -> list[dict[str, Any]]:
+    return [
+        {
+            "wo_base": f.key.base,
+            "lot": f.key.lot,
+            "split": f.key.split,
+            "sub": f.key.sub,
+            "seq": f.seq,
+            "op": f.op,
+            "target_hrs": str(f.tgt),
+        }
+        for f in c.folded
+    ]

server/packetstation/feed/config.py

@@ -0,0 +1,226 @@
+"""Feed settings: one `settings` row with key `feed`, a JSON object read by the worker on every fire.
+
+Unknown keys are ignored on load and missing keys take the default, so the row may be partial. A save
+validates, bumps the row version, writes an audit row and NOTIFYs `ps_settings` so the worker reschedules.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from datetime import date, datetime, time, timedelta
+from typing import Any
+
+from psycopg.types.json import Jsonb
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator
+
+from .. import audit
+
+KEY = "feed"
+SETTINGS_CHANNEL = "ps_settings"
+_HHMM = re.compile(r"^([01][0-9]|2[0-3]):[0-5][0-9]$")
+_DEPT = re.compile(r"^[A-Za-z0-9]{1,10}$")
+
+
+class SettingsError(ValueError):
+    """A patch the model refuses. `errors` holds one short line per problem."""
+
+    def __init__(self, errors: list[str]):
+        super().__init__("; ".join(errors))
+        self.errors = errors
+
+
+class FeedSettings(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    interval_shop_minutes: int = Field(default=5, ge=1, le=1440)
+    interval_off_minutes: int = Field(default=30, ge=1, le=1440)
+    shop_days: list[int] = Field(default_factory=lambda: [0, 1, 2, 3, 4])  # Monday = 0, site time
+    shop_start: str = "06:00"  # HH:MM site time
+    shop_end: str = "18:00"
+    window_back_days: int = Field(default=14, ge=0, le=120)
+    window_ahead_days: int = Field(default=30, ge=0, le=365)
+    golive_date: date | None = None  # work orders released before this date are ignored
+    departments: list[str] = Field(default_factory=list)  # shop codes; empty = every department the map routes to
+    drawing_recheck_hours: int = Field(default=4, ge=1, le=168)
+    drawing_budget: int = Field(default=60, ge=1, le=500)
+    drawing_interval_minutes: int = Field(default=30, ge=1, le=1440)
+    labor_tickets: bool = True  # read labor tickets (at most every 30 minutes)
+    auto_release: bool = True
+    auto_release_max: int = Field(default=25, ge=0, le=500)
+    paused: bool = False  # stops scheduled feed, drawing and auto-release runs; "run now" still works
+
+    @field_validator("shop_days")
+    @classmethod
+    def _days(cls, v: list[int]) -> list[int]:
+        if any(d < 0 or d > 6 for d in v):
+            raise ValueError("shop days are 0 (Monday) to 6 (Sunday)")
+        return sorted(set(v))
+
+    @field_validator("shop_start", "shop_end")
+    @classmethod
+    def _hhmm(cls, v: str) -> str:
+        v = v.strip()
+        if not _HHMM.match(v):
+            raise ValueError("a time is HH:MM, 00:00 to 23:59")
+        return v
+
+    @field_validator("departments")
+    @classmethod
+    def _depts(cls, v: list[str]) -> list[str]:
+        out: list[str] = []
+        for d in v:
+            d = str(d).strip()
+            if not _DEPT.match(d):
+                raise ValueError("a department is its short code, like 03")
+            if d not in out:
+                out.append(d)
+        return out
+
+    @model_validator(mode="after")
+    def _hours(self) -> FeedSettings:
+        if self.shop_start == self.shop_end:
+            raise ValueError("shop_start and shop_end must differ")
+        return self
+
+
+FIELDS = tuple(FeedSettings.model_fields)
+
+
+def _clock(hhmm: str) -> time:
+    h, m = hhmm.split(":")
+    return time(int(h), int(m))
+
+
+def _errors(e: ValidationError) -> list[str]:
+    out = []
+    for err in e.errors():
+        loc = ".".join(str(x) for x in err.get("loc", ()))
+        msg = str(err.get("msg", "invalid")).removeprefix("Value error, ")
+        out.append(f"{loc}: {msg}" if loc else msg)
+    return out
+
+
+def from_value(value: Any) -> FeedSettings:
+    """The model from a stored value. A bad stored value keeps its good fields; the rest take the default."""
+    if not isinstance(value, dict):
+        return FeedSettings()
+    try:
+        return FeedSettings.model_validate(value)
+    except ValidationError:
+        good: dict[str, Any] = {}
+        for k in FIELDS:
+            if k not in value:
+                continue
+            try:
+                FeedSettings.model_validate({**good, k: value[k]})
+            except ValidationError:
+                continue
+            good[k] = value[k]
+        return FeedSettings.model_validate(good)
+
+
+def load(conn: Any) -> FeedSettings:
+    """The current feed settings; the defaults when the row is missing."""
+    row = conn.execute("SELECT value FROM settings WHERE key = %s", (KEY,)).fetchone()
+    return from_value(row[0] if row else None)
+
+
+def load_versioned(conn: Any) -> tuple[FeedSettings, int, Any, Any]:
+    """(settings, version, updated_at, updated_by); version 0 when the row is missing."""
+    row = conn.execute("SELECT value, version, updated_at, updated_by FROM settings WHERE key = %s", (KEY,)).fetchone()
+    if row is None:
+        return FeedSettings(), 0, None, None
+    return from_value(row[0]), int(row[1]), row[2], row[3]
+
+
+def seed(conn: Any) -> bool:
+    """Write the defaults when the row is missing. True when it wrote."""
+    cur = conn.execute(
+        "INSERT INTO settings (key, value, version, updated_at) VALUES (%s, %s, 1, now()) ON CONFLICT (key) DO NOTHING",
+        (KEY, Jsonb(FeedSettings().model_dump(mode="json"))),
+    )
+    return bool(cur.rowcount)
+
+
+def validate_patch(current: FeedSettings, patch: dict[str, Any]) -> FeedSettings:
+    """The settings after a partial change. Raises SettingsError, also for a key that is not a setting."""
+    unknown = sorted(k for k in patch if k not in FIELDS)
+    if unknown:
+        raise SettingsError([f"{k}: not a feed setting" for k in unknown])
+    try:
+        return FeedSettings.model_validate({**current.model_dump(mode="json"), **patch})
+    except ValidationError as e:
+        raise SettingsError(_errors(e)) from None
+
+
+def save(conn: Any, *, who: Any, patch: dict[str, Any]) -> FeedSettings:
+    """Validate and store a partial change. Raises SettingsError. A patch that changes nothing writes nothing.
+
+    `who` has user_id, session_id, role_snapshot, ip and it_reason (domain.projects.Who).
+    """
+    with conn.transaction():
+        row = conn.execute("SELECT value FROM settings WHERE key = %s FOR UPDATE", (KEY,)).fetchone()
+        current = from_value(row[0] if row else None)
+        new = validate_patch(current, patch)
+        before, after = current.model_dump(mode="json"), new.model_dump(mode="json")
+        changed = {k: {"from": before[k], "to": after[k]} for k in FIELDS if before[k] != after[k]}
+        if not changed and row is not None:
+            return new
+        version = conn.execute(
+            "INSERT INTO settings (key, value, version, updated_at, updated_by) VALUES (%s, %s, 1, now(), %s)"
+            " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1,"
+            " updated_at = now(), updated_by = EXCLUDED.updated_by RETURNING version",
+            (KEY, Jsonb(after), who.user_id),
+        ).fetchone()[0]
+        audit.append(
+            conn,
+            action="feed.settings_changed",
+            user_id=who.user_id,
+            session_id=getattr(who, "session_id", None),
+            role_snapshot=getattr(who, "role_snapshot", None),
+            subject_type="settings",
+            subject_id=KEY,
+            detail={"changed": changed},
+            it_reason=getattr(who, "it_reason", None),
+            ip=getattr(who, "ip", None),
+        )
+        notify_settings(conn, int(version))
+    return new
+
+
+def notify_settings(conn: Any, version: int | None = None) -> None:
+    """Tell the worker the settings changed. Inside a transaction it is delivered on commit."""
+    payload = json.dumps({"key": KEY, "version": version}, separators=(",", ":"))
+    conn.execute("SELECT ps_notify(%s, %s)", (SETTINGS_CHANNEL, payload))
+
+
+# ---------------------------------------------------------------- derived values (pure)
+
+
+def window(settings: FeedSettings, today: date) -> tuple[date, date]:
+    """(since, until) for the released read, from the site-local date."""
+    return today - timedelta(days=settings.window_back_days), today + timedelta(days=settings.window_ahead_days)
+
+
+def in_shop_hours(settings: FeedSettings, now_site: datetime) -> bool:
+    """True on a shop day from shop_start up to shop_end, site time.
+
+    An end before the start spans midnight; the hours after midnight belong to the day the shift started.
+    """
+    start, end = _clock(settings.shop_start), _clock(settings.shop_end)
+    t = now_site.time().replace(tzinfo=None)
+    day = now_site.weekday()
+    if start < end:
+        return day in settings.shop_days and start <= t < end
+    if t >= start:
+        return day in settings.shop_days
+    if t < end:
+        return (day - 1) % 7 in settings.shop_days
+    return False
+
+
+def interval_seconds(settings: FeedSettings, now_site: datetime) -> int:
+    """The feed's cadence now: the shop interval inside shop hours, the off interval outside."""
+    minutes = settings.interval_shop_minutes if in_shop_hours(settings, now_site) else settings.interval_off_minutes
+    return minutes * 60

server/packetstation/feed/fingerprint.py

@@ -0,0 +1,210 @@
+"""The candidate fingerprint: what VISUAL says about a packet, hashed per part, to decide update, CO or nothing.
+
+Parts (each the sha256 hex of its canonical JSON):
+
+- major: `legs` (the sorted leg keys), `qty` (per leg), `rev`, `drawing` (upper-cased, spaces removed), `bom`
+  (sorted `[item, part_id, qty]`, the old matSig);
+- minor: `tgt` (target hours per leg), `want` (the want date).
+
+Never in it: rel, folded, partspecs, title, statuses, issued, whs, loc, orders, labor, dept, op. A candidate whose
+materials are unknown (`bom is None`) has no `bom` part; `classify` skips a part the new side lacks and `merge`
+carries the old one forward, so a failed materials read never looks like a BOM change.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import unicodedata
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import date, datetime
+from decimal import Decimal
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .candidates import Candidate
+
+MAJOR = ("legs", "qty", "rev", "drawing", "bom")
+MINOR = ("tgt", "want")
+PARTS = MAJOR + MINOR
+
+
+# ---------------------------------------------------------------- normalisation
+
+
+def norm_text(value: Any) -> str | None:
+    """NFC, CRLF and CR to LF, trimmed; '' and None both None."""
+    if value is None:
+        return None
+    s = value if isinstance(value, str) else str(value)
+    s = unicodedata.normalize("NFC", s.replace("\r\n", "\n").replace("\r", "\n")).strip()
+    return s or None
+
+
+def norm_id(value: Any) -> str | None:
+    """An ID (base, lot, split, sub, part, drawing): normalised text, upper-cased. Stays text."""
+    s = norm_text(value)
+    return s.upper() if s is not None else None
+
+
+def norm_decimal(value: Any) -> str | None:
+    """A number as text with no exponent and no trailing zeros: 2, 2.0 and Decimal('2.00') are all '2'."""
+    if value is None or value == "":
+        return None
+    d = value if isinstance(value, Decimal) else Decimal(str(value))
+    if d.is_nan() or d.is_infinite():
+        return None
+    if d == 0:
+        return "0"
+    s = format(d.normalize(), "f")
+    if "." in s:
+        s = s.rstrip("0").rstrip(".")
+    return s
+
+
+def normalise(value: Any) -> Any:
+    """The canonical form of a JSON-like value."""
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return norm_text(value)
+    if isinstance(value, int | float | Decimal):
+        return norm_decimal(value)
+    if isinstance(value, datetime):
+        return value.isoformat()
+    if isinstance(value, date):
+        return value.isoformat()
+    if isinstance(value, Mapping):
+        return {str(k): normalise(v) for k, v in value.items()}
+    if isinstance(value, list | tuple | set | frozenset):
+        return [normalise(v) for v in value]
+    return norm_text(str(value))
+
+
+def canonical(value: Any) -> bytes:
+    """json.dumps(sort_keys, compact separators, ensure_ascii False) of the normalised value, UTF-8."""
+    return json.dumps(normalise(value), sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
+
+
+def sha256_hex(value: Any) -> str:
+    return hashlib.sha256(canonical(value)).hexdigest()
+
+
+def drawing_key(dwg: Any) -> str | None:
+    """A drawing number for comparison: upper-cased, every whitespace removed."""
+    s = norm_text(dwg)
+    if s is None:
+        return None
+    s = "".join(s.split()).upper()
+    return s or None
+
+
+def leg_key(base: Any, lot: Any, split: Any, sub: Any, seq: Any) -> str:
+    """'BASE|LOT|SPLIT|SUB|SEQ' with the ID parts upper-cased (as text) and seq as an integer."""
+    parts = [norm_id(x) or "" for x in (base, lot, split, sub)]
+    return "|".join([*parts, str(int(seq)) if seq is not None and seq != "" else ""])
+
+
+# ---------------------------------------------------------------- the fingerprint
+
+
+@dataclass(frozen=True)
+class Fingerprint:
+    full: str
+    parts: dict[str, str]
+
+
+@dataclass(frozen=True)
+class Change:
+    major: frozenset[str]
+    minor: frozenset[str]
+
+    @property
+    def changed(self) -> bool:
+        return bool(self.major or self.minor)
+
+    @property
+    def is_minor(self) -> bool:
+        return bool(self.minor) and not self.major
+
+
+def part_values(c: Candidate) -> dict[str, Any]:
+    """The normalised value of every part, before hashing (tests and the diff text read these)."""
+    legs = sorted(
+        (
+            [
+                norm_id(lg.key.base) or "",
+                norm_id(lg.key.lot) or "",
+                norm_id(lg.key.split) or "",
+                norm_id(lg.key.sub) or "",
+                int(lg.seq),
+            ]
+            for lg in c.legs
+        ),
+        key=lambda x: (x[0], x[1], x[2], x[3], x[4]),
+    )
+    qty: dict[str, Any] = {}
+    tgt: dict[str, Any] = {}
+    for lg in c.legs:
+        k = leg_key(lg.key.base, lg.key.lot, lg.key.split, lg.key.sub, lg.seq)
+        qty[k] = norm_decimal(lg.qty)
+        tgt[k] = norm_decimal(lg.tgt)
+    values: dict[str, Any] = {
+        "legs": legs,
+        "qty": qty,
+        "rev": norm_text(c.rev),
+        "drawing": drawing_key(c.drawing),
+        "tgt": tgt,
+        "want": c.want.isoformat() if c.want else None,
+    }
+    if c.bom is not None:
+        values["bom"] = bom_sig(c.bom)
+    return values
+
+
+def bom_sig(bom: Any) -> list[list[Any]]:
+    """The old matSig: sorted [item, part_id, qty] of every line (BomLine objects or dicts)."""
+    out: list[list[Any]] = []
+    for b in bom or []:
+        get = b.get if isinstance(b, Mapping) else lambda k, _b=b: getattr(_b, k, None)
+        out.append([norm_text(get("item")) or "", norm_id(get("part_id")) or "", norm_decimal(get("qty"))])
+    out.sort(key=lambda x: (x[0], x[1], x[2] or ""))
+    return out
+
+
+def _full(parts: Mapping[str, str]) -> str:
+    return sha256_hex(dict(parts))
+
+
+def compute(c: Candidate) -> Fingerprint:
+    parts = {name: sha256_hex(v) for name, v in part_values(c).items()}
+    return Fingerprint(full=_full(parts), parts=parts)
+
+
+def merge(before: Mapping[str, str] | None, after: Fingerprint) -> Fingerprint:
+    """Carry forward parts the new side does not know (the BOM when materials are unknown this run)."""
+    if not before:
+        return after
+    parts = dict(after.parts)
+    for name in PARTS:
+        if name not in parts and name in before:
+            parts[name] = str(before[name])
+    if parts == after.parts:
+        return after
+    return Fingerprint(full=_full(parts), parts=parts)
+
+
+def classify(before: Mapping[str, str] | None, after: Mapping[str, str]) -> Change:
+    """Which parts changed. A part the new side lacks (unknown this run) is skipped. No `before`: all changed."""
+    major: set[str] = set()
+    minor: set[str] = set()
+    for name in PARTS:
+        if name not in after:
+            continue
+        if before is not None and before.get(name) == after[name]:
+            continue
+        (major if name in MAJOR else minor).add(name)
+    return Change(frozenset(major), frozenset(minor))

server/packetstation/feed/materials.py

@@ -0,0 +1,291 @@
+"""A packet's BOM from VISUAL's material lines: the old matBom, typed.
+
+Rules (Seth, 2026-10-08, as the old build):
+
+- a line belongs to a leg when its work order key and operation seq match the leg and its drawing is blank or
+  equals the leg's drawing ignoring spaces and case;
+- the same piece can come back more than once (one per header status): keep the best, R before F before U before
+  anything else;
+- lines without a part are not BOM lines; a part-less line whose piece is a multiple of 10 is the group header of
+  the next nine pieces of the same work order (`grp`);
+- the item is the piece number, suffixed `-<last dash part of the work order>` when the piece repeats;
+- WIP sub-assemblies (commodity WIP or warehouse ASY) are listed, marked sub and n/a (no heat needed);
+- `loc` is the warehouse (unless MAIN) and the location, distinct and non-empty, joined by a space.
+
+`bom_lines.item` is an int in the database while the BOM item label is text ("10", "10-5GA"). `db_items` gives
+the int per line: the piece number when the label is a plain number used once, else the next free number above
+every piece. The text label goes into `cl_item`. `insert_bom` writes lines that way; agent D's "use VISUAL's BOM
+again" calls `bom_for_packet` and `insert_bom` so both paths store the same thing.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from decimal import Decimal
+from typing import Any
+
+from psycopg.rows import dict_row
+
+from .types import Material, WoKey, integer, num, text
+
+STATUS_RANK = {"R": 0, "F": 1, "U": 2}
+
+MatIndex = dict[tuple[WoKey, int], list[Material]]
+
+
+@dataclass(slots=True)
+class BomLine:
+    item: str
+    qty: Decimal
+    part_id: str
+    desc: str
+    loc: str
+    whs: str
+    comm: str
+    cons: bool
+    sub: bool
+    na: bool
+    grp: str | None
+    vkey: str
+    issued: Decimal
+    istat: int | None
+
+    def as_dict(self) -> dict[str, Any]:
+        """JSON-ready (numbers as text) for change order `vis.bom`."""
+        return {
+            "item": self.item,
+            "qty": str(self.qty),
+            "part_id": self.part_id,
+            "desc": self.desc,
+            "loc": self.loc,
+            "whs": self.whs,
+            "comm": self.comm,
+            "cons": self.cons,
+            "sub": self.sub,
+            "na": self.na,
+            "grp": self.grp,
+            "vkey": self.vkey,
+            "issued": str(self.issued),
+            "istat": self.istat,
+        }
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]) -> BomLine:
+        return cls(
+            item=text(d.get("item")),
+            qty=num(d.get("qty")),
+            part_id=text(d.get("part_id")),
+            desc=text(d.get("desc")),
+            loc=text(d.get("loc")),
+            whs=text(d.get("whs")),
+            comm=text(d.get("comm")),
+            cons=bool(d.get("cons")),
+            sub=bool(d.get("sub")),
+            na=bool(d.get("na")),
+            grp=text(d.get("grp")) or None,
+            vkey=text(d.get("vkey")),
+            issued=num(d.get("issued")),
+            istat=None if d.get("istat") is None else integer(d.get("istat")),
+        )
+
+
+def dwg_norm(dwg: Any) -> str:
+    return "".join(text(dwg).split()).upper()
+
+
+def index(materials: Iterable[Material] | None) -> MatIndex:
+    """Material lines by (leg key, seq)."""
+    idx: MatIndex = {}
+    for m in materials or ():
+        idx.setdefault((m.key, m.seq), []).append(m)
+    return idx
+
+
+def _leg_parts(leg: Any) -> tuple[WoKey, int, str]:
+    """(key, seq, dwg) from a candidates.Leg, or from a packet_work_orders row dict (wo_base, lot, split, sub, seq,
+    dwg)."""
+    if isinstance(leg, Mapping):
+        key = WoKey.of(leg.get("wo_base"), leg.get("lot"), leg.get("split"), leg.get("sub"))
+        return key, integer(leg.get("seq")), text(leg.get("dwg"))
+    return leg.key, int(leg.seq), text(leg.dwg)
+
+
+def _rank(m: Material) -> int:
+    return STATUS_RANK.get(text(m.st).upper()[:1], 3)
+
+
+def bom_for(legs: Sequence[Any], idx: MatIndex) -> list[BomLine]:
+    """The BOM of these legs (candidates.Leg objects or packet_work_orders dicts) from the material index."""
+    lines: list[Material] = []
+    seen: set[tuple[WoKey, int]] = set()
+    for leg in legs:
+        key, seq, dwg = _leg_parts(leg)
+        k = (key, seq)
+        if k in seen:
+            continue
+        seen.add(k)
+        best: dict[int, Material] = {}
+        for m in idx.get(k, []):
+            if m.dwg and dwg and dwg_norm(m.dwg) != dwg_norm(dwg):
+                continue
+            o = best.get(m.piece)
+            if o is None or _rank(m) < _rank(o):
+                best[m.piece] = m
+        lines.extend(best.values())
+    lines.sort(key=lambda m: (m.wo, m.piece))  # stable: ties keep leg order
+    out: list[BomLine] = []
+    used: set[str] = set()
+    for m in lines:
+        if not m.part:
+            continue
+        grp = next(
+            (
+                h
+                for h in lines
+                if not h.part and h.wo == m.wo and h.piece % 10 == 0 and h.piece < m.piece < h.piece + 10
+            ),
+            None,
+        )
+        item = str(m.piece)
+        if item in used:
+            item = f"{item}-{m.wo.split('-')[-1]}"
+        used.add(item)
+        sub = m.comm.upper() == "WIP" or m.whs.upper() == "ASY"
+        locs: list[str] = []
+        for part in ("" if m.whs.upper() == "MAIN" else m.whs, m.loc):
+            if part and part not in locs:
+                locs.append(part)
+        out.append(
+            BomLine(
+                item=item,
+                qty=m.qty,
+                part_id=m.part,
+                desc=m.desc,
+                loc=" ".join(locs),
+                whs=m.whs,
+                comm=m.comm,
+                cons=m.cons.upper() == "Y",
+                sub=sub,
+                na=sub,
+                grp=str(grp.piece) if grp is not None else None,
+                vkey=f"{m.wo}|{m.lot}|{m.sub}|{m.seq}|{m.piece}",
+                issued=m.issued,
+                istat=m.istat,
+            )
+        )
+    return out
+
+
+def db_items(bom: Sequence[BomLine]) -> list[int]:
+    """The int `bom_lines.item` per line: the piece when the label is a plain number used once, else the next
+    free number above every piece number."""
+    plain = [b.item for b in bom if b.item.isdigit()]
+    taken: set[int] = set()
+    out: list[int | None] = []
+    for b in bom:
+        if b.item.isdigit() and plain.count(b.item) == 1 and int(b.item) not in taken:
+            n = int(b.item)
+            taken.add(n)
+            out.append(n)
+        else:
+            out.append(None)
+    nxt = max(taken, default=0)
+    for b in bom:
+        head = b.item.split("-", 1)[0]
+        if head.isdigit():
+            nxt = max(nxt, int(head))
+    result: list[int] = []
+    for slot in out:
+        if slot is not None:
+            result.append(slot)
+            continue
+        nxt += 1
+        while nxt in taken:
+            nxt += 1
+        taken.add(nxt)
+        result.append(nxt)
+    return result
+
+
+def insert_bom(conn: Any, packet_id: int, bom: Sequence[BomLine]) -> int:
+    """Insert VISUAL BOM lines (src 'visual'). cons is stored 'Y'/'N', sub 'Y' or NULL. Returns the count."""
+    items = db_items(bom)
+    for sort, (b, item) in enumerate(zip(bom, items, strict=True)):
+        conn.execute(
+            "INSERT INTO bom_lines (packet_id, item, cl_item, qty, part_id, description, loc, whs, comm, cons, sub,"
+            " grp, vkey, na, issued, src, sort) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,"
+            " 'visual', %s)",
+            (
+                packet_id,
+                item,
+                b.item,
+                b.qty,
+                b.part_id[:60],
+                b.desc[:240] or None,
+                b.loc[:120] or None,
+                b.whs or None,
+                b.comm or None,
+                "Y" if b.cons else "N",
+                "Y" if b.sub else None,
+                b.grp,
+                b.vkey,
+                b.na,
+                b.issued,
+                sort,
+            ),
+        )
+    return len(bom)
+
+
+# ---------------------------------------------------------------- from the cache
+
+
+def materials_for(conn: Any, keys: Iterable[WoKey]) -> list[Material]:
+    """Live cached material lines (gone_batch NULL) of these leg keys."""
+    ks = list(dict.fromkeys(keys))
+    if not ks:
+        return []
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute(
+            "SELECT m.* FROM visual_materials m"
+            " JOIN unnest(%s::text[], %s::text[], %s::text[], %s::text[]) AS k(b, l, sp, su)"
+            " ON m.wo_base = k.b AND m.lot = k.l AND m.split = k.sp AND m.sub = k.su"
+            " WHERE m.gone_batch IS NULL ORDER BY m.wo_base, m.lot, m.split, m.sub, m.seq, m.piece",
+            ([k.base for k in ks], [k.lot for k in ks], [k.split for k in ks], [k.sub for k in ks]),
+        ).fetchall()
+    return [material_from_row(r) for r in rows]
+
+
+def material_from_row(r: Mapping[str, Any]) -> Material:
+    return Material(
+        wo=r["wo_base"],
+        lot=r["lot"],
+        split=r["split"],
+        sub=r["sub"],
+        seq=int(r["seq"]),
+        piece=int(r["piece"]),
+        part=r["part"],
+        qty=num(r["qty"]),
+        issued=num(r["issued"]),
+        whs=r["whs"],
+        loc=r["loc"],
+        st=r["st"],
+        istat=r["istat"],
+        dwg=r["dwg"],
+        desc=r["descr"],
+        comm=r["comm"],
+        cons=r["cons"],
+    )
+
+
+def bom_for_packet(conn: Any, packet_id: int) -> list[BomLine]:
+    """VISUAL's BOM for a packet's legs, rebuilt from the cache (for "use VISUAL's BOM again")."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        legs = cur.execute(
+            "SELECT wo_base, lot, split, sub, seq, coalesce(dwg, '') AS dwg FROM packet_work_orders"
+            " WHERE packet_id = %s AND seq IS NOT NULL ORDER BY sort, id",
+            (packet_id,),
+        ).fetchall()
+    keys = [WoKey.of(r["wo_base"], r["lot"], r["split"], r["sub"]) for r in legs]
+    return bom_for(legs, index(materials_for(conn, keys)))

server/packetstation/feed/orders.py

@@ -0,0 +1,104 @@
+"""Which customer order a project or unit takes its customer, PO and dates from, when VISUAL has several.
+
+The one rule, in order:
+
+1. a Clerical pin that names one of the project's orders (`pin`);
+2. the base's own linked order, the first of `by_base[base]` (`base`);
+3. the project order whose top jobs name the unit, open orders first (`unit_top_job`);
+4. the project's first open order, status F, H or R (`project_open`);
+5. the project's first order, the newest (`project_newest`).
+
+`derive` maps the chosen order to the fields a project or unit keeps: customer, po, order_date, ship_date.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import date, datetime
+from typing import Any
+
+from .types import Order
+
+OPEN_STATUSES = ("F", "H", "R")
+VIA = ("pin", "base", "unit_top_job", "project_open", "project_newest")
+FIELDS = ("customer", "po", "order_date", "ship_date")
+
+
+@dataclass(frozen=True)
+class Chosen:
+    order: Order
+    via: str  # pin | base | unit_top_job | project_open | project_newest
+
+
+def _is_open(o: Order) -> bool:
+    return (o.status or "").strip().upper() in OPEN_STATUSES
+
+
+def choose(
+    *,
+    unit_number: str,
+    base: str,
+    by_base: Mapping[str, list[Order]],
+    by_project: Mapping[str, list[Order]],
+    project: str,
+    pin: str | None,
+) -> Chosen | None:
+    proj = list(by_project.get(project) or [])
+    own = list(by_base.get(base) or []) if base else []
+    if pin:
+        p = pin.strip().upper()
+        for o in [*proj, *own]:
+            if o.cust_order_id.strip().upper() == p:
+                return Chosen(o, "pin")
+    if own:
+        return Chosen(own[0], "base")
+    if unit_number:
+        u = unit_number.upper()
+        hits = [o for o in proj if any(t.upper() == u for t in o.top_jobs)]
+        hits.sort(key=lambda o: not _is_open(o))  # stable: open first, the list's own order within
+        if hits:
+            return Chosen(hits[0], "unit_top_job")
+    for o in proj:
+        if _is_open(o):
+            return Chosen(o, "project_open")
+    if proj:
+        return Chosen(proj[0], "project_newest")
+    return None
+
+
+def derive(order: Order) -> dict[str, Any]:
+    """customer <- customer_name, po <- customer_po_ref, order_date, ship_date <- promise or desired ship date."""
+    return {
+        "customer": order.customer_name,
+        "po": order.customer_po_ref,
+        "order_date": order.order_date,
+        "ship_date": order.promise_date or order.desired_ship_date,
+    }
+
+
+def _iso(v: date | None) -> str | None:
+    return v.isoformat() if v else None
+
+
+def order_ref(chosen: Chosen, at: datetime) -> dict[str, Any]:
+    """The jsonb kept on projects.order_ref and units.order_ref."""
+    o = chosen.order
+    return {
+        "cust_order_id": o.cust_order_id,
+        "via": chosen.via,
+        "status": o.status,
+        "customer_name": o.customer_name,
+        "customer_po_ref": o.customer_po_ref,
+        "order_date": _iso(o.order_date),
+        "desired_ship_date": _iso(o.desired_ship_date),
+        "promise_date": _iso(o.promise_date),
+        "chosen_at": at.isoformat(),
+    }
+
+
+def same_ref(a: Mapping[str, Any] | None, b: Mapping[str, Any] | None) -> bool:
+    """Two order_refs are the same when everything but chosen_at matches."""
+    if a is None or b is None:
+        return a is None and b is None
+    return {k: v for k, v in a.items() if k != "chosen_at"} == {k: v for k, v in b.items() if k != "chosen_at"}

server/packetstation/feed/source.py

@@ -0,0 +1,491 @@
+"""Where the feed reads VISUAL from: VisualKit (DEV unless the stack says otherwise) or a synthetic fixture file.
+
+`VisualKitSource` copies the read calls and output shapes of VisualKit's examples/packet_station_feed.py: the
+window read, the tracked keys by key, operations, operation text and parts for rows; for_bases, requirements,
+requirement text, parts and part sites for materials; DEMAND_SUPPLY_LINK orders by base and by project; labor
+tickets summed per operation. Read-only: VisualKit enforces it. No cost, price, amount or pay column is read.
+
+`FixtureSource` reads `<dir>/state` and `<dir>/feed.<state>.json` in the same contract shape (CI and tests).
+
+Every value goes through `types.parse_row` / `parse_material` / `parse_order`, so both sources give identical
+FeedData.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Iterable, Sequence
+from datetime import date, datetime
+from decimal import Decimal
+from pathlib import Path
+from typing import Any, Protocol
+
+from .types import (
+    FeedData,
+    LaborLeg,
+    Material,
+    Order,
+    Row,
+    WoKey,
+    parse_labor,
+    parse_material,
+    parse_order,
+    parse_orders,
+    parse_row,
+    status_code,
+    text,
+)
+
+STATUS_WORDS = {"F": "Firmed", "R": "Released", "U": "Unreleased", "C": "Closed", "X": "Cancelled"}
+MATERIAL_STATUSES = ("F", "R")
+OPEN_ORDER_STATUSES = ("F", "H", "R")
+QTY_EMPLOYEE = "9999"
+
+_SECRETISH = re.compile(r"(?i)(password|pwd|secret|token)\s*[=:]\s*\S+")
+
+
+def redact(message: object) -> str:
+    """An exception as one short line with anything password-like removed."""
+    s = _SECRETISH.sub(
+        r"\1=***", f"{type(message).__name__}: {message}" if isinstance(message, BaseException) else str(message)
+    )
+    return s.replace("\r", " ").replace("\n", " ")[:500]
+
+
+class FeedSource(Protocol):
+    def read(
+        self,
+        *,
+        site: str,
+        since: date,
+        until: date,
+        tracked: Sequence[WoKey],
+        with_materials: bool,
+        with_orders: bool,
+        with_labor: bool,
+        labor_since: date | None,
+    ) -> FeedData: ...
+
+    def describe(self) -> str: ...
+
+
+# ---------------------------------------------------------------- value helpers (as the VisualKit example)
+
+
+def _to_number(v: Any) -> Any:
+    if v is None or isinstance(v, bool):
+        return v
+    if isinstance(v, Decimal):
+        return v
+    return Decimal(str(v))
+
+
+def _fmt_date(v: date | datetime | None) -> str | None:
+    if v is None:
+        return None
+    if isinstance(v, datetime):
+        v = v.date()
+    return v.isoformat()
+
+
+def _clean(s: str | None) -> str:
+    if not s:
+        return ""
+    return s.replace("\r\n", "\n").replace("\r", "\n")
+
+
+def _istat(calc_qty: Any, issued_qty: Any, line_status: str | None) -> int:
+    """Job Pull's issue state: 3 over-issued, 0 line closed, 2 nothing issued, 1 partly."""
+    calc = Decimal(str(calc_qty or 0))
+    issued = Decimal(str(issued_qty or 0))
+    if issued > calc:
+        return 3
+    if (line_status or "").strip().upper() == "C":
+        return 0
+    if issued == 0:
+        return 2
+    return 1
+
+
+def _yes_no(flag: bool | None) -> str:
+    return "" if flag is None else ("Y" if flag else "N")
+
+
+def project_of(base: str) -> str:
+    """VisualKit's project_of: the base before its first dash, or the whole base."""
+    head = base.partition("-")[0]
+    return head or base
+
+
+# ---------------------------------------------------------------- VisualKit
+
+
+class VisualKitSource:
+    """The five reads through one `visualkit.Visual` (made by the job per run, closed after)."""
+
+    def __init__(self, visual: Any) -> None:
+        self.v = visual
+
+    def describe(self) -> str:
+        s = self.v.settings
+        return f"{s.server} / {s.database}"
+
+    def _key(self, k: WoKey) -> Any:
+        from visualkit import WorkOrderKey
+
+        return WorkOrderKey(k.base, k.lot, k.split, k.sub, "W")
+
+    def _load_parts(self, part_ids: Iterable[str | None], cache: dict[str, Any]) -> None:
+        missing = sorted({p for p in part_ids if p and p not in cache})
+        if missing:
+            cache.update(self.v.parts.get_many(missing))
+
+    def read(
+        self,
+        *,
+        site: str,
+        since: date,
+        until: date,
+        tracked: Sequence[WoKey],
+        with_materials: bool,
+        with_orders: bool,
+        with_labor: bool,
+        labor_since: date | None,
+    ) -> FeedData:
+        v = self.v
+        s = v.settings
+        meta: dict[str, Any] = {
+            "source": "visualkit",
+            "server": s.server,
+            "database": s.database,
+            "site": site,
+            "since": since.isoformat(),
+            "until": until.isoformat(),
+            "generated_at": datetime.now().isoformat(timespec="seconds"),
+        }
+        data = FeedData(meta=meta, rows=[], tracked=list(tracked), tracked_count=len(tracked))
+        parts: dict[str, Any] = {}
+
+        # 1. The window: firmed TYPE W work orders released in [since, until], paged.
+        wos: list[Any] = []
+        offset: int | None = 0
+        while offset is not None:
+            page = v.work_orders.released(site, since, until, status=("F",), offset=offset)
+            wos.extend(page.items)
+            offset = page.next_offset
+        data.window_count = len(wos)
+
+        # 2. The tracked keys, whatever their status or date. Keys VISUAL lacks are missing.
+        asked = {self._key(k): k for k in dict.fromkeys(tracked)}
+        found = v.work_orders.get_many(list(asked)) if asked else {}
+        data.tracked_missing = [k for vk, k in asked.items() if vk not in found]
+        have = {wo.key for wo in wos}
+        for wo in found.values():
+            if wo.key not in have:
+                wos.append(wo)
+                have.add(wo.key)
+
+        # 3. Rows for the union.
+        keys = [wo.key for wo in wos]
+        ops = v.work_orders.operations_for(keys) if keys else {}
+        specs = v.work_orders.operation_text_for(keys) if keys else {}
+        self._load_parts((wo.part_id for wo in wos), parts)
+        for wo in wos:
+            part = parts.get(wo.part_id) if wo.part_id else None
+            desc = _clean(part.description if part else None)
+            qty = wo.desired_qty or Decimal(0)
+            for op in ops.get(wo.key, []):
+                data.rows.append(
+                    parse_row(
+                        {
+                            "wo": wo.base_id,
+                            "lot": wo.lot_id,
+                            "split": wo.split_id,
+                            "sub": wo.sub_id,
+                            "seq": op.sequence_no,
+                            "res": op.resource_id or "",
+                            "dwg": wo.drawing_id or "",
+                            "rev": wo.drawing_rev_no or "",
+                            "ps": specs.get((wo.key, op.sequence_no), ""),
+                            "desc": desc,
+                            "qty": _to_number(qty),
+                            "tgt": _to_number((op.run or Decimal(0)) * qty),
+                            "rel": _fmt_date(wo.desired_rls_date),
+                            "want": _fmt_date(wo.desired_want_date),
+                            "st": STATUS_WORDS.get((wo.status or "").strip().upper(), wo.status or ""),
+                            "barcode": op.barcode(),
+                            "part": wo.part_id or "",
+                        }
+                    )
+                )
+
+        bases = sorted({wo.base_id for wo in wos})
+        if with_materials:
+            try:
+                data.materials, data.material_legs = self._materials(bases, site, parts)
+            except Exception as e:  # a failed materials read keeps the BOMs as they are
+                data.materials, data.material_legs = None, None
+                data.errors.append("materials: " + redact(e))
+        if with_orders:
+            try:
+                data.orders_by_base = self._orders_by_base(bases)
+                data.orders_by_project = self._orders_by_project({project_of(b) for b in bases})
+            except Exception as e:  # the apply keeps the last cached orders
+                data.orders_by_base, data.orders_by_project = None, None
+                data.errors.append("orders: " + redact(e))
+        if with_labor:
+            try:
+                data.labor = self._labor(list(asked) or keys)
+            except Exception as e:
+                data.labor = None
+                data.errors.append("labor: " + redact(e))
+        return data
+
+    def _materials(self, bases: Sequence[str], site: str, parts: dict[str, Any]) -> tuple[list[Material], set[WoKey]]:
+        v = self.v
+        if not bases:
+            return [], set()
+        legs = v.work_orders.for_bases(bases, type="W", status=MATERIAL_STATUSES, site_id=site)
+        keys = [wo.key for wo in legs]
+        reqs = v.work_orders.requirements_for(keys) if keys else {}
+        text_keys = [k for k in keys if any(not r.part_id for r in reqs.get(k, []))]
+        texts = v.work_orders.requirement_text_for(text_keys) if text_keys else {}
+        part_ids = {r.part_id for rs in reqs.values() for r in rs if r.part_id}
+        self._load_parts(part_ids, parts)
+        sites = v.parts.sites_for(sorted(part_ids), site) if part_ids else {}
+        out: list[Material] = []
+        covered: set[WoKey] = set()
+        for wo in legs:
+            covered.add(WoKey.of(wo.base_id, wo.lot_id, wo.split_id, wo.sub_id))
+            for r in reqs.get(wo.key, []):
+                part = parts.get(r.part_id) if r.part_id else None
+                ps = sites.get(r.part_id) if r.part_id else None
+                desc = (
+                    _clean(part.description if part else None)
+                    if r.part_id
+                    else _clean(texts.get((wo.key, r.operation_seq_no, r.piece_no)))
+                )
+                out.append(
+                    parse_material(
+                        {
+                            "wo": wo.base_id,
+                            "lot": wo.lot_id,
+                            "split": wo.split_id,
+                            "sub": wo.sub_id,
+                            "seq": r.operation_seq_no,
+                            "piece": r.piece_no,
+                            "part": r.part_id or None,
+                            "qty": _to_number(r.calc_qty or Decimal(0)),
+                            "issued": _to_number(r.issued_qty or Decimal(0)),
+                            "whs": (ps.primary_whs_id or "").upper() if ps else "",
+                            "loc": (ps.primary_loc_id or "").upper() if ps else "",
+                            "st": wo.status,
+                            "istat": _istat(r.calc_qty, r.issued_qty, r.status),
+                            "dwg": wo.drawing_id or "",
+                            "desc": desc,
+                            "comm": (part.commodity_code or "") if part else "",
+                            "cons": _yes_no(part.consumable) if part else "",
+                        }
+                    )
+                )
+        return out, covered
+
+    def _orders_by_base(self, bases: Sequence[str]) -> dict[str, list[Order]]:
+        v = self.v
+        if not bases:
+            return {}
+        pairs = v.links.orders_for_bases(list(bases))
+        order_ids = sorted({oid for ps in pairs.values() for oid, _ in ps})
+        orders = v.orders.get_many(order_ids) if order_ids else {}
+        customers = v.orders.customers(sorted({o.customer_id for o in orders.values()})) if orders else {}
+        out: dict[str, list[Order]] = {}
+        for base in bases:
+            lines_by_order: dict[str, list[int]] = {}
+            for oid, line_no in pairs.get(base, []):
+                lines_by_order.setdefault(oid, [])
+                if line_no is not None:
+                    lines_by_order[oid].append(line_no)
+            if not lines_by_order:
+                continue
+            out[base] = []
+            for oid, lines in sorted(lines_by_order.items()):
+                o = orders.get(oid)
+                cust = customers.get(o.customer_id) if o else None
+                out[base].append(
+                    parse_order(
+                        {
+                            "cust_order_id": oid,
+                            "lines": sorted(set(lines)),
+                            "status": o.status if o else None,
+                            "customer_id": o.customer_id if o else None,
+                            "customer_name": cust.name if cust else None,
+                            "customer_po_ref": o.customer_po_ref if o else None,
+                            "order_date": _fmt_date(getattr(o, "order_date", None)) if o else None,
+                            "desired_ship_date": _fmt_date(o.desired_ship_date) if o else None,
+                            "promise_date": _fmt_date(o.promise_date) if o else None,
+                        }
+                    )
+                )
+        return out
+
+    def _orders_by_project(self, projects: Iterable[str]) -> dict[str, list[Order]]:
+        v = self.v
+        wanted = sorted(set(projects))
+        if not wanted:
+            return {}
+        found = v.links.orders_for_projects(wanted)
+        cust_ids = sorted({po.customer_id for pos in found.values() for po in pos if po.customer_id})
+        customers = v.orders.customers(cust_ids) if cust_ids else {}
+        out: dict[str, list[Order]] = {}
+        for project in wanted:
+            entries: dict[str, dict[str, Any]] = {}
+            for po in found.get(project, []):
+                e = entries.get(po.cust_order_id)
+                if e is None:
+                    cust = customers.get(po.customer_id) if po.customer_id else None
+                    e = entries[po.cust_order_id] = {
+                        "cust_order_id": po.cust_order_id,
+                        "lines": [],
+                        "top_jobs": [],
+                        "status": po.status,
+                        "customer_id": po.customer_id,
+                        "customer_name": cust.name if cust else None,
+                        "customer_po_ref": po.customer_po_ref,
+                        "order_date": _fmt_date(po.order_date),
+                        "desired_ship_date": _fmt_date(po.desired_ship_date),
+                        "promise_date": _fmt_date(po.promise_date),
+                    }
+                if po.line_no is not None and po.line_no not in e["lines"]:
+                    e["lines"].append(po.line_no)
+                if po.supply_base_id not in e["top_jobs"]:
+                    e["top_jobs"].append(po.supply_base_id)
+            if entries:
+                for e in entries.values():
+                    e["lines"].sort()
+                    e["top_jobs"].sort()
+                ordered = sorted(entries.values(), key=lambda e: e["status"] not in OPEN_ORDER_STATUSES)
+                out[project] = [parse_order(e) for e in ordered]
+        return out
+
+    def _labor(self, keys: Sequence[Any]) -> list[LaborLeg]:
+        """Hours and ticket counts per (leg, seq). Employee 9999's quantity-only tickets and indirect tickets
+        are left out. Hours only: no cost or pay column is read. VisualKit has no date filter on this call, so
+        every ticket of the legs is read and summed (the totals need them all)."""
+        if not keys:
+            return []
+        tickets = self.v.labor.tickets_for_work_orders(list(keys))
+        sums: dict[tuple[WoKey, int], LaborLeg] = {}
+        for vk, ts in tickets.items():
+            k = WoKey.of(vk.base_id, vk.lot_id, vk.split_id, vk.sub_id)
+            for t in ts:
+                if t.operation_seq_no is None or t.is_quantity_only or t.is_indirect:
+                    continue
+                seq = int(t.operation_seq_no)
+                leg = sums.get((k, seq))
+                if leg is None:
+                    leg = sums[(k, seq)] = LaborLeg(k.base, k.lot, k.split, k.sub, seq, Decimal(0), 0, None)
+                leg.hours += Decimal(str(t.hours_worked or 0))
+                leg.tickets += 1
+                d = t.transaction_date.date() if isinstance(t.transaction_date, datetime) else t.transaction_date
+                if d is not None and (leg.last_ticket is None or d > leg.last_ticket):
+                    leg.last_ticket = d
+        return list(sums.values())
+
+
+# ---------------------------------------------------------------- fixtures
+
+
+class FixtureSource:
+    """Synthetic VISUAL: `<dir>/state` (default "1") picks `<dir>/feed.<state>.json`."""
+
+    def __init__(self, directory: Path) -> None:
+        self.dir = Path(directory)
+
+    def state(self) -> str:
+        try:
+            s = (self.dir / "state").read_text(encoding="utf-8").strip()
+        except OSError:
+            s = ""
+        return s or "1"
+
+    def describe(self) -> str:
+        return f"fixture state {self.state()}"
+
+    def read(
+        self,
+        *,
+        site: str,
+        since: date,
+        until: date,
+        tracked: Sequence[WoKey],
+        with_materials: bool,
+        with_orders: bool,
+        with_labor: bool,
+        labor_since: date | None,
+    ) -> FeedData:
+        state = self.state()
+        raw = json.loads((self.dir / f"feed.{state}.json").read_text(encoding="utf-8"))
+        all_rows: list[Row] = [parse_row(d) for d in raw.get("rows") or []]
+        tracked_set = set(tracked)
+        window_keys = {r.key for r in all_rows if r.st == "F" and r.rel is not None and since <= r.rel <= until}
+        keep = window_keys | (tracked_set & {r.key for r in all_rows})
+        rows = [r for r in all_rows if r.key in keep]
+        present = {r.key for r in all_rows}
+        meta = dict(raw.get("meta") or {})
+        meta.update(
+            {"source": "fixture", "state": state, "site": site, "since": since.isoformat(), "until": until.isoformat()}
+        )
+        data = FeedData(
+            meta=meta,
+            rows=rows,
+            tracked=list(tracked),
+            tracked_missing=[k for k in dict.fromkeys(tracked) if k not in present],
+            window_count=len(window_keys),
+            tracked_count=len(tracked),
+        )
+        bases = {r.wo for r in rows}
+        if with_materials:
+            if raw.get("materials") is None:
+                data.materials, data.material_legs = None, None
+            else:
+                lines = [parse_material(d) for d in raw.get("materials") or []]
+                data.materials = [m for m in lines if m.wo in bases]
+                data.material_legs = {m.key for m in data.materials if m.st in MATERIAL_STATUSES}
+                # A leg the file lists with status F or R counts as covered even when it has no lines.
+                for r in all_rows:
+                    if r.wo in bases and r.st in MATERIAL_STATUSES:
+                        data.material_legs.add(r.key)
+        if with_orders:
+            obb = parse_orders(raw.get("orders_by_base"))
+            obp = parse_orders(raw.get("orders_by_project"))
+            if obb is None or obp is None:
+                data.orders_by_base, data.orders_by_project = None, None
+            else:
+                projects = {project_of(b) for b in bases}
+                data.orders_by_base = {b: v for b, v in obb.items() if b in bases}
+                data.orders_by_project = {p: v for p, v in obp.items() if p in projects}
+        if with_labor and raw.get("labor") is not None:
+            data.labor = [parse_labor(d) for d in raw.get("labor") or []]
+        return data
+
+
+def make(settings: Any, visual: Any = None) -> FeedSource:
+    """The source the settings name. `visualkit` needs the open `visual` the job made."""
+    if settings.feed_source == "fixture":
+        return FixtureSource(Path(settings.fixture_dir))
+    if visual is None:
+        raise ValueError("the visualkit feed source needs an open visualkit.Visual")
+    return VisualKitSource(visual)
+
+
+__all__ = [
+    "FeedSource",
+    "FixtureSource",
+    "VisualKitSource",
+    "make",
+    "project_of",
+    "redact",
+    "status_code",
+    "text",
+]

server/packetstation/feed/types.py

@@ -0,0 +1,306 @@
+"""The feed's typed records: one VISUAL operation row, one material line, one customer order, labor per leg.
+
+Both feed sources (VisualKit and the fixture files) build these through `parse_row` and `parse_material`, so they
+produce identical data. Every ID part (work order base, lot, split, sub, part id, order id) is text and is never
+cast to a number: leading zeros stay and lot `7A` occurs. `seq` and `piece` are int. Quantities are Decimal.
+`None` and `""` are the same for text.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from datetime import date, datetime
+from decimal import Decimal, InvalidOperation
+from typing import Any
+
+STATUS_CODES = {
+    "FIRMED": "F",
+    "RELEASED": "R",
+    "UNRELEASED": "U",
+    "CLOSED": "C",
+    "CANCELLED": "X",
+    "CANCELED": "X",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class WoKey:
+    """A work order leg key, text exactly as VISUAL has it ('' never None)."""
+
+    base: str
+    lot: str
+    split: str
+    sub: str
+
+    def __str__(self) -> str:
+        return f"{self.base}/{self.lot}/{self.split}/{self.sub}"
+
+    def as_tuple(self) -> tuple[str, str, str, str]:
+        return (self.base, self.lot, self.split, self.sub)
+
+    @classmethod
+    def of(cls, base: Any, lot: Any, split: Any, sub: Any) -> WoKey:
+        return cls(text(base), text(lot), text(split), text(sub))
+
+
+@dataclass(slots=True)
+class Row:
+    """One TYPE W operation, as Job Release Information carries it."""
+
+    wo: str
+    lot: str
+    split: str
+    sub: str
+    seq: int
+    res: str
+    dwg: str
+    rev: str
+    ps: str
+    desc: str
+    qty: Decimal
+    tgt: Decimal
+    rel: date | None
+    want: date | None
+    st: str  # the status CODE: F R U C X, or ''
+    part: str = ""
+    barcode: str = ""
+
+    @property
+    def key(self) -> WoKey:
+        return WoKey(self.wo, self.lot, self.split, self.sub)
+
+
+@dataclass(slots=True)
+class Material:
+    """One requirement line, as Job Pull carries it. part is None for a line with no part (a group header)."""
+
+    wo: str
+    lot: str
+    split: str
+    sub: str
+    seq: int
+    piece: int
+    part: str | None
+    qty: Decimal
+    issued: Decimal
+    whs: str
+    loc: str
+    st: str
+    istat: int | None
+    dwg: str
+    desc: str
+    comm: str
+    cons: str
+
+    @property
+    def key(self) -> WoKey:
+        return WoKey(self.wo, self.lot, self.split, self.sub)
+
+
+@dataclass(slots=True)
+class Order:
+    cust_order_id: str
+    status: str | None
+    customer_id: str | None
+    customer_name: str | None
+    customer_po_ref: str | None
+    order_date: date | None
+    desired_ship_date: date | None
+    promise_date: date | None
+    lines: list[int] = field(default_factory=list)
+    top_jobs: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class LaborLeg:
+    wo: str
+    lot: str
+    split: str
+    sub: str
+    seq: int
+    hours: Decimal
+    tickets: int
+    last_ticket: date | None
+
+    @property
+    def key(self) -> WoKey:
+        return WoKey(self.wo, self.lot, self.split, self.sub)
+
+
+@dataclass(slots=True)
+class FeedData:
+    """One read of VISUAL (or of a fixture file). None on a section means that read failed or was not asked for."""
+
+    meta: dict[str, Any]
+    rows: list[Row]
+    materials: list[Material] | None = None
+    material_legs: set[WoKey] | None = None  # legs the materials read covered (header status F or R)
+    orders_by_base: dict[str, list[Order]] | None = None
+    orders_by_project: dict[str, list[Order]] | None = None
+    labor: list[LaborLeg] | None = None
+    tracked: list[WoKey] = field(default_factory=list)  # the tracked keys this read was asked for
+    tracked_missing: list[WoKey] = field(default_factory=list)
+    window_count: int = 0  # work orders the window read returned
+    tracked_count: int = 0
+    errors: list[str] = field(default_factory=list)
+
+
+# ---------------------------------------------------------------- value helpers
+
+
+def text(value: Any) -> str:
+    """Text as stored: None and '' alike become ''. CRLF and CR become LF. Never a number cast."""
+    if value is None:
+        return ""
+    s = value if isinstance(value, str) else str(value)
+    return s.replace("\r\n", "\n").replace("\r", "\n").strip()
+
+
+def opt_text(value: Any) -> str | None:
+    s = text(value)
+    return s or None
+
+
+def num(value: Any) -> Decimal:
+    """A quantity as Decimal, through str so 0.1 stays 0.1. None, '' and junk become 0."""
+    if value is None or value == "" or isinstance(value, bool):
+        return Decimal(0)
+    if isinstance(value, Decimal):
+        return value
+    try:
+        return Decimal(str(value))
+    except (InvalidOperation, ValueError):
+        return Decimal(0)
+
+
+def opt_num(value: Any) -> Decimal | None:
+    if value is None or value == "":
+        return None
+    return num(value)
+
+
+def day(value: Any) -> date | None:
+    """A date from a date, datetime or ISO text (the first 10 characters). Anything else is None."""
+    if value is None or value == "":
+        return None
+    if isinstance(value, datetime):
+        return value.date()
+    if isinstance(value, date):
+        return value
+    try:
+        return date.fromisoformat(str(value).strip()[:10])
+    except ValueError:
+        return None
+
+
+def integer(value: Any, default: int = 0) -> int:
+    """seq and piece: whole numbers. 10, 10.0, '10' and Decimal('10') are all 10."""
+    if value is None or value == "" or isinstance(value, bool):
+        return default
+    if isinstance(value, int):
+        return value
+    try:
+        return int(Decimal(str(value).strip()))
+    except (InvalidOperation, ValueError):
+        return default
+
+
+def opt_int(value: Any) -> int | None:
+    if value is None or value == "":
+        return None
+    return integer(value)
+
+
+def status_code(word_or_code: Any) -> str:
+    """'Firmed' or 'F' -> 'F', and so on. Unknown -> its first letter upper-cased. None and '' -> ''."""
+    s = text(word_or_code).upper()
+    if not s:
+        return ""
+    if s in STATUS_CODES:
+        return STATUS_CODES[s]
+    return s[0]
+
+
+# ---------------------------------------------------------------- parsing contract dicts
+
+
+def parse_row(d: Mapping[str, Any]) -> Row:
+    """A Row from a feed-contract dict (examples/packet_station_feed.py `rows`)."""
+    return Row(
+        wo=text(d.get("wo")),
+        lot=text(d.get("lot")),
+        split=text(d.get("split")),
+        sub=text(d.get("sub")),
+        seq=integer(d.get("seq")),
+        res=text(d.get("res")),
+        dwg=text(d.get("dwg")),
+        rev=text(d.get("rev")),
+        ps=text(d.get("ps")),
+        desc=text(d.get("desc")),
+        qty=num(d.get("qty")),
+        tgt=num(d.get("tgt")),
+        rel=day(d.get("rel")),
+        want=day(d.get("want")),
+        st=status_code(d.get("st")),
+        part=text(d.get("part")),
+        barcode=text(d.get("barcode")),
+    )
+
+
+def parse_material(d: Mapping[str, Any]) -> Material:
+    """A Material from a feed-contract dict (`materials`). A missing part stays None."""
+    return Material(
+        wo=text(d.get("wo")),
+        lot=text(d.get("lot")),
+        split=text(d.get("split")),
+        sub=text(d.get("sub")),
+        seq=integer(d.get("seq")),
+        piece=integer(d.get("piece")),
+        part=opt_text(d.get("part")),
+        qty=num(d.get("qty")),
+        issued=num(d.get("issued")),
+        whs=text(d.get("whs")),
+        loc=text(d.get("loc")),
+        st=status_code(d.get("st")),
+        istat=opt_int(d.get("istat")),
+        dwg=text(d.get("dwg")),
+        desc=text(d.get("desc")),
+        comm=text(d.get("comm")),
+        cons=text(d.get("cons")),
+    )
+
+
+def parse_order(d: Mapping[str, Any]) -> Order:
+    """An Order from an orders_by_base or orders_by_project entry."""
+    return Order(
+        cust_order_id=text(d.get("cust_order_id")),
+        status=opt_text(d.get("status")),
+        customer_id=opt_text(d.get("customer_id")),
+        customer_name=opt_text(d.get("customer_name")),
+        customer_po_ref=opt_text(d.get("customer_po_ref")),
+        order_date=day(d.get("order_date")),
+        desired_ship_date=day(d.get("desired_ship_date")),
+        promise_date=day(d.get("promise_date")),
+        lines=[integer(x) for x in (d.get("lines") or []) if x is not None and x != ""],
+        top_jobs=[text(x) for x in (d.get("top_jobs") or []) if text(x)],
+    )
+
+
+def parse_orders(m: Mapping[str, Any] | None) -> dict[str, list[Order]] | None:
+    if m is None:
+        return None
+    return {text(k): [parse_order(e) for e in (v or [])] for k, v in m.items() if text(k)}
+
+
+def parse_labor(d: Mapping[str, Any]) -> LaborLeg:
+    return LaborLeg(
+        wo=text(d.get("wo")),
+        lot=text(d.get("lot")),
+        split=text(d.get("split")),
+        sub=text(d.get("sub")),
+        seq=integer(d.get("seq")),
+        hours=num(d.get("hours")),
+        tickets=integer(d.get("tickets")),
+        last_ticket=day(d.get("last_ticket")),
+    )

server/packetstation/feed/writes.py

@@ -0,0 +1,216 @@
+"""Packet writes by the worker: the pipeline's packet_write for the system principal, with no session.
+
+`system_write` locks the packet row (FOR UPDATE SKIP LOCKED: a row a person is writing right now is skipped and
+retried next run, `Busy`), builds the policy's PacketState from the same child rows the API uses, asks the policy
+as `Principal(user_id=None, system="visual-feed")` (`Refused` when it says no), yields a `SysWrite`, then bumps
+the version, writes the packet_log row (by NULL, by_name 'VISUAL feed', role 'system'), the audit row
+(`packet.<action>`, role_snapshot 'system') and the NOTIFY. Everything runs in a savepoint (or a transaction when
+none is open), so a refusal or an error inside the body leaves nothing behind.
+
+VIS_AUTO_RELEASE carries the SEND_TO_WAREHOUSE side effects here, locally: status warehouse, status_at,
+wh_skipped false, wh_sent_at, a packet_wh incoming row (by NULL, auto true) and a packet_transitions row (by NULL,
+action 'vis_auto_release').
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from typing import Any
+
+from psycopg.rows import dict_row
+
+from .. import audit, events, policy
+from ..domain import packets
+from ..text import LIMITS, clean_opt
+
+A = policy.Action
+SYSTEM = policy.Principal(user_id=None, system="visual-feed")
+BY_NAME = "VISUAL feed"
+ROLE = "system"
+
+LABELS: dict[policy.Action, str] = {
+    A.VIS_UPDATE_DRAFT: "Updated from VISUAL",
+    A.VIS_OPEN_CO: "Change from VISUAL",
+    A.VIS_AUTO_RELEASE: "Sent to warehouse automatically",
+    A.VIS_RETIRE_DRAFT: "Retired: work orders moved to another packet",
+}
+
+_SELECT = (
+    "SELECT *, status::text AS status, source_kind::text AS source_kind, routed_to::text AS routed_to,"
+    " drawing_state::text AS drawing_state FROM packets WHERE id = %s FOR UPDATE SKIP LOCKED"
+)
+
+EMPTY: Mapping[str, Any] = {}
+
+
+class Busy(Exception):
+    """Someone else holds the packet row right now. Retry next run."""
+
+
+class Missing(LookupError):
+    """No such packet."""
+
+
+class Refused(Exception):
+    """The policy said no."""
+
+    def __init__(self, decision: policy.Decision) -> None:
+        super().__init__(f"{decision.code}: {decision.why}")
+        self.decision = decision
+
+
+@dataclass
+class SysWrite:
+    conn: Any
+    packet: dict[str, Any]  # the packets row, locked
+    state: policy.PacketState
+    action: policy.Action
+    label: str
+    note: str | None = None
+    detail: dict[str, Any] = field(default_factory=dict)
+    fields: list[str] = field(default_factory=list)
+    noop: bool = False
+    version: int = 0
+    transition_id: int | None = None
+
+    @property
+    def id(self) -> int:
+        return int(self.packet["id"])
+
+    def changed(self, *names: str) -> None:
+        for n in names:
+            if n not in self.fields:
+                self.fields.append(n)
+
+    def log(self, label: str, note: str | None = None) -> None:
+        """Override the packet_log label (and note)."""
+        self.label = label
+        if note is not None:
+            self.note = note
+
+    def skip(self) -> None:
+        """Nothing changed: no version bump, no log, audit or event."""
+        self.noop = True
+
+
+def lock(conn: Any, packet_id: int) -> dict[str, Any]:
+    """The packets row locked FOR UPDATE SKIP LOCKED. Busy when another transaction holds it."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        row = cur.execute(_SELECT, (packet_id,)).fetchone()
+    if row is not None:
+        return dict(row)
+    if conn.execute("SELECT 1 FROM packets WHERE id = %s", (packet_id,)).fetchone() is None:
+        raise Missing(packet_id)
+    raise Busy(packet_id)
+
+
+@contextmanager
+def system_write(
+    conn: Any,
+    packet_id: int,
+    action: policy.Action,
+    *,
+    label: str | None = None,
+    note: str | None = None,
+    detail: dict[str, Any] | None = None,
+    extra: Mapping[str, Any] = EMPTY,
+) -> Iterator[SysWrite]:
+    with conn.transaction():
+        row = lock(conn, packet_id)
+        ch = packets.load_children(conn, row, full=False)
+        state = packets.state_from(row, ch)
+        decision = policy.check(SYSTEM, state, action, extra=extra)
+        if not decision.ok:
+            raise Refused(decision)
+        sw = SysWrite(
+            conn=conn,
+            packet=row,
+            state=state,
+            action=action,
+            label=label or LABELS.get(action, str(action)),
+            note=note,
+            detail=dict(detail or {}),
+            version=int(row["version"]),
+        )
+        yield sw
+        _finish(sw)
+
+
+def _finish(sw: SysWrite) -> None:
+    if sw.noop:
+        return
+    conn = sw.conn
+    from_status = str(sw.packet["status"])
+    from_dept = str(sw.packet["dept_code"])
+    if sw.action is A.VIS_AUTO_RELEASE:
+        _auto_release(sw)
+    sw.version = int(
+        conn.execute("UPDATE packets SET version = version + 1 WHERE id = %s RETURNING version", (sw.id,)).fetchone()[0]
+    )
+    after = conn.execute("SELECT status::text, dept_code FROM packets WHERE id = %s", (sw.id,)).fetchone()
+    status, dept = str(after[0]), str(after[1])
+    note = clean_opt(sw.note, LIMITS["note"]) if sw.note else None
+    conn.execute(
+        "INSERT INTO packet_log (packet_id, by, by_name, role, action, note, transition_id)"
+        " VALUES (%s, NULL, %s, %s, %s, %s, %s)",
+        (sw.id, BY_NAME, ROLE, sw.label, note, sw.transition_id),
+    )
+    detail: dict[str, Any] = {"from": from_status, "to": status, "dept": dept}
+    if from_dept != dept:
+        detail["from_dept"] = from_dept
+    if sw.fields:
+        detail["fields"] = sw.fields
+    if note:
+        detail["note"] = note
+    detail.update(sw.detail)
+    audit.append(
+        conn,
+        action=f"packet.{sw.action}",
+        role_snapshot=ROLE,
+        subject_type="packet",
+        subject_id=sw.id,
+        packet_id=sw.id,
+        detail=packets_jsonable(detail),
+    )
+    events.publish(
+        conn, {"event": "changed", "packet_id": sw.id, "version": sw.version, "status": status, "dept": dept}
+    )
+
+
+def _auto_release(sw: SysWrite) -> None:
+    conn = sw.conn
+    conn.execute(
+        "UPDATE packets SET status = 'warehouse', status_at = now(), wh_skipped = false, wh_sent_at = now(),"
+        " return_note = NULL WHERE id = %s",
+        (sw.id,),
+    )
+    conn.execute("UPDATE packet_wh SET closed_at = now() WHERE packet_id = %s AND closed_at IS NULL", (sw.id,))
+    conn.execute(
+        "INSERT INTO packet_wh (packet_id, stage, by, note, auto) VALUES (%s, 'incoming', NULL, %s, true)",
+        (sw.id, "Sent automatically: no flags"),
+    )
+    dept = str(sw.packet["dept_code"])
+    tid = conn.execute(
+        "INSERT INTO packet_transitions (packet_id, by, action, from_status, to_status, from_dept, to_dept, reason)"
+        " VALUES (%s, NULL, 'vis_auto_release', %s::packet_status, 'warehouse', %s, %s, %s) RETURNING id",
+        (sw.id, str(sw.packet["status"]), dept, dept, sw.note),
+    ).fetchone()
+    sw.transition_id = int(tid[0])
+    sw.changed("status")
+
+
+def packets_jsonable(value: Any) -> Any:
+    """Audit detail as plain JSON (dates and Decimals as text)."""
+    from fastapi.encoders import jsonable_encoder
+
+    return jsonable_encoder(value)
+
+
+def log_only(conn: Any, packet_id: int, label: str, note: str | None = None) -> None:
+    """A packet_log row by the feed with no version bump (for observations that change nothing a user edits)."""
+    conn.execute(
+        "INSERT INTO packet_log (packet_id, by, by_name, role, action, note) VALUES (%s, NULL, %s, %s, %s, %s)",
+        (packet_id, BY_NAME, ROLE, label, clean_opt(note, LIMITS["note"]) if note else None),
+    )

server/packetstation/main.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import asyncio
 import contextlib
+import importlib
 import logging
 from collections.abc import AsyncIterator
 from pathlib import Path
@@ -93,13 +94,48 @@ def create_app(
     app.state.groups_cache = None
     api_errors.install(app)
 
-    from .api import account, admin, assets, auth, departments, packet_children, packets, projects, units
+    from .api import (
+        account,
+        admin,
+        assets,
+        auth,
+        departments,
+        feed_settings,
+        jobs,
+        packet_children,
+        packets,
+        projects,
+        units,
+    )
     from .api import events as events_api
 
     # Routers go in before the static catch-all below, which answers every other GET.
-    for module in (auth, account, admin, departments, events_api, projects, units, packets, packet_children, assets):
+    for module in (
+        auth,
+        account,
+        admin,
+        departments,
+        events_api,
+        projects,
+        units,
+        packets,
+        packet_children,
+        assets,
+        jobs,
+        feed_settings,
+    ):
         app.include_router(module.router)
     app.include_router(assets.blob_router)
+    for name in ("feed", "packet_visual"):
+        modname = f"{__package__}.api.{name}"
+        try:
+            extra = importlib.import_module(modname)
+        except ModuleNotFoundError as e:
+            if e.name != modname:
+                raise
+            log.warning("router api.%s not there yet; not loaded", name)
+            continue
+        app.include_router(extra.router)
 
     @app.get("/healthz")
     def healthz() -> dict[str, Any]:

server/packetstation/migrations/0004_feed.sql

@@ -0,0 +1,422 @@
+-- 0004_feed: the VISUAL and drawing caches, the go-live ignored list, the resource map, the worker's
+-- job tables and the order-derived columns on projects and units. Runs as ps_migrate; 0001's default
+-- privileges give ps_app SELECT, INSERT, UPDATE, DELETE on every table here. Every work order ID part
+-- (base, lot, split, sub) and every part ID is text: leading zeros are kept and lot 7A occurs.
+
+-- ---------------------------------------------------------------- VISUAL cache
+
+-- One row per feed read. suspect batches skip the "gone" marking.
+CREATE TABLE visual_batches (
+    id           bigserial PRIMARY KEY,
+    run_id       bigint,
+    started_at   timestamptz NOT NULL DEFAULT now(),
+    finished_at  timestamptz,
+    server       text,
+    database     text,
+    site         text,
+    since        date,
+    until        date,
+    window_rows  int,
+    tracked_keys int,
+    missing_keys int,
+    suspect      bool NOT NULL DEFAULT false,
+    counts       jsonb NOT NULL DEFAULT '{}'
+);
+
+-- One row per TYPE W operation as VISUAL last showed it. st is the status code (F R U C X) or ''.
+CREATE TABLE visual_rows (
+    id          bigserial PRIMARY KEY,
+    wo_base     text NOT NULL,
+    lot         text NOT NULL DEFAULT '',
+    split       text NOT NULL DEFAULT '',
+    sub         text NOT NULL DEFAULT '',
+    seq         int NOT NULL,
+    res         text NOT NULL DEFAULT '',
+    dwg         text NOT NULL DEFAULT '',
+    rev         text NOT NULL DEFAULT '',
+    ps          text NOT NULL DEFAULT '',
+    descr       text NOT NULL DEFAULT '',
+    qty         numeric,
+    tgt         numeric,
+    rel         date,
+    want        date,
+    st          text NOT NULL DEFAULT '',
+    part        text NOT NULL DEFAULT '',
+    barcode     text,
+    row_hash    bytea NOT NULL,
+    first_batch bigint NOT NULL REFERENCES visual_batches (id),
+    last_batch  bigint NOT NULL REFERENCES visual_batches (id),
+    gone_batch  bigint REFERENCES visual_batches (id),
+    UNIQUE (wo_base, lot, split, sub, seq)
+);
+
+CREATE INDEX visual_rows_wo ON visual_rows (wo_base, lot);
+CREATE INDEX visual_rows_dwg ON visual_rows (dwg);
+CREATE INDEX visual_rows_rel_live ON visual_rows (rel) WHERE gone_batch IS NULL;
+
+-- Material requirement lines. part is NULL on a line with no part (a group header).
+CREATE TABLE visual_materials (
+    id          bigserial PRIMARY KEY,
+    wo_base     text NOT NULL,
+    lot         text NOT NULL DEFAULT '',
+    split       text NOT NULL DEFAULT '',
+    sub         text NOT NULL DEFAULT '',
+    seq         int NOT NULL,
+    piece       int NOT NULL,
+    part        text,
+    qty         numeric,
+    issued      numeric,
+    whs         text NOT NULL DEFAULT '',
+    loc         text NOT NULL DEFAULT '',
+    st          text NOT NULL DEFAULT '',
+    istat       int,
+    dwg         text NOT NULL DEFAULT '',
+    descr       text NOT NULL DEFAULT '',
+    comm        text NOT NULL DEFAULT '',
+    cons        text NOT NULL DEFAULT '',
+    mat_hash    bytea NOT NULL,
+    first_batch bigint NOT NULL REFERENCES visual_batches (id),
+    last_batch  bigint NOT NULL REFERENCES visual_batches (id),
+    gone_batch  bigint REFERENCES visual_batches (id),
+    UNIQUE (wo_base, lot, split, sub, seq, piece)
+);
+
+CREATE INDEX visual_materials_wo ON visual_materials (wo_base, lot);
+
+CREATE TABLE visual_orders (
+    cust_order_id     text PRIMARY KEY,
+    status            text,
+    customer_id       text,
+    customer_name     text,
+    customer_po_ref   text,
+    order_date        date,
+    desired_ship_date date,
+    promise_date      date,
+    last_batch        bigint
+);
+
+-- kind base: key is a work order base. kind project: key is a project number.
+CREATE TABLE visual_order_links (
+    id            bigserial PRIMARY KEY,
+    kind          text NOT NULL CHECK (kind IN ('base', 'project')),
+    key           text NOT NULL,
+    cust_order_id text NOT NULL REFERENCES visual_orders (cust_order_id),
+    rank          int NOT NULL,
+    lines         int[] NOT NULL DEFAULT '{}',
+    top_jobs      text[] NOT NULL DEFAULT '{}',
+    last_batch    bigint,
+    UNIQUE (kind, key, cust_order_id)
+);
+
+CREATE INDEX visual_order_links_key ON visual_order_links (kind, key);
+
+-- Hours and ticket counts per operation. No cost columns, ever.
+CREATE TABLE visual_labor (
+    id          bigserial PRIMARY KEY,
+    wo_base     text NOT NULL,
+    lot         text NOT NULL DEFAULT '',
+    split       text NOT NULL DEFAULT '',
+    sub         text NOT NULL DEFAULT '',
+    seq         int NOT NULL,
+    hours       numeric NOT NULL DEFAULT 0,
+    tickets     int NOT NULL DEFAULT 0,
+    last_ticket date,
+    observed_at timestamptz NOT NULL DEFAULT now(),
+    UNIQUE (wo_base, lot, split, sub, seq)
+);
+
+-- What VISUAL changed, per batch, for the Admin view. Pruned at 90 days.
+CREATE TABLE visual_row_changes (
+    id       bigserial PRIMARY KEY,
+    batch_id bigint NOT NULL REFERENCES visual_batches (id),
+    wo_base  text NOT NULL,
+    lot      text NOT NULL DEFAULT '',
+    split    text NOT NULL DEFAULT '',
+    sub      text NOT NULL DEFAULT '',
+    seq      int,
+    kind     text NOT NULL CHECK (kind IN ('added', 'changed', 'gone', 'returned')),
+    before   jsonb,
+    after    jsonb,
+    at       timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE INDEX visual_row_changes_batch ON visual_row_changes (batch_id);
+CREATE INDEX visual_row_changes_at_brin ON visual_row_changes USING brin (at);
+
+CREATE TABLE visual_material_changes (
+    id       bigserial PRIMARY KEY,
+    batch_id bigint NOT NULL REFERENCES visual_batches (id),
+    wo_base  text NOT NULL,
+    lot      text NOT NULL DEFAULT '',
+    split    text NOT NULL DEFAULT '',
+    sub      text NOT NULL DEFAULT '',
+    seq      int,
+    piece    int,
+    kind     text NOT NULL CHECK (kind IN ('added', 'changed', 'gone', 'returned')),
+    before   jsonb,
+    after    jsonb,
+    at       timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE INDEX visual_material_changes_batch ON visual_material_changes (batch_id);
+CREATE INDEX visual_material_changes_at_brin ON visual_material_changes USING brin (at);
+
+-- Work orders released before the go-live date. A row pulled in by hand becomes tracked.
+CREATE TABLE visual_ignored_wos (
+    id               bigserial PRIMARY KEY,
+    wo_base          text NOT NULL,
+    lot              text NOT NULL DEFAULT '',
+    split            text NOT NULL DEFAULT '',
+    sub              text NOT NULL DEFAULT '',
+    rel              date,
+    project          text,
+    unit_number      text,
+    res              text,
+    dwg              text,
+    first_seen_batch bigint,
+    last_seen_at     timestamptz NOT NULL DEFAULT now(),
+    pulled_in_at     timestamptz,
+    pulled_in_by     uuid REFERENCES users (id),
+    pulled_in_reason text,
+    UNIQUE (wo_base, lot, split, sub)
+);
+
+-- VISUAL resource to shop department, or fold (an operation folded into the packet) or ignore.
+CREATE TABLE visual_resource_map (
+    resource   text PRIMARY KEY,
+    target     text NOT NULL CHECK (target = 'fold' OR target = 'ignore' OR target ~ '^[0-9]{2}$'),
+    confirmed  bool NOT NULL DEFAULT false,
+    updated_at timestamptz,
+    updated_by uuid
+);
+
+-- Seeded from the old build's sync/visual_map.json. "Electical" is spelled that way in VISUAL.
+INSERT INTO visual_resource_map (resource, target) VALUES
+    ('LAYOUT', '01'),
+    ('LAYOUT-BT', '01'),
+    ('LAYOUT-SM BAND', '01'),
+    ('LAYOUT-BEVELER', '01'),
+    ('LAYOUT-ATT', '01'),
+    ('LAYOUT-ROLL', '01'),
+    ('LAYOUT-PRESS', '01'),
+    ('LAYOUT-THREADER', '01'),
+    ('LAYOUT-HAND CUT', '01'),
+    ('LAYOUT-WEST', '02'),
+    ('LT-SS', '02'),
+    ('WELD-PART', '03'),
+    ('NOVARC_1', '03'),
+    ('WELD-PART-WEST', '04'),
+    ('PART-WELD-WEST', '04'),
+    ('WP-SS', '05'),
+    ('WELD-CODE', '06'),
+    ('WELD-CODE-BT', '07'),
+    ('SUBASSEMBLY', '08'),
+    ('WEST-SA', '08'),
+    ('HYDROTEST', '09'),
+    ('WELD-ASY', '10'),
+    ('FINAL ASY-NORTH', '11'),
+    ('PREP-NORTH', '11'),
+    ('FINAL ASSEMBLY', '11'),
+    ('FINAL ASY-SOUTH', '12'),
+    ('PREP-SOUTH', '12'),
+    ('WEST-FA', '12'),
+    ('PAINT', '13'),
+    ('SANDBLAST', '13'),
+    ('FINAL-ASY-ELEC', '14'),
+    ('Electical', '14'),
+    ('MFG', 'fold'),
+    ('INSPECTION', 'fold'),
+    ('REWORK', 'fold'),
+    ('REWORK-CUSTOMER', 'fold'),
+    ('OUTSIDE SERVICE', 'fold'),
+    ('PTU', 'fold'),
+    ('FORKLIFT', 'ignore'),
+    ('STEAM', 'ignore'),
+    ('WIPLOAD', 'ignore'),
+    ('FOREMAN', 'ignore'),
+    ('ENGINEERING', 'ignore'),
+    ('TRAINING', 'ignore'),
+    ('F.S. TECH.', 'ignore'),
+    ('F.S. TECH. ASST', 'ignore'),
+    ('F.S. STEAM', 'ignore'),
+    ('F.S. WELDER', 'ignore'),
+    ('F.S. SANDBLAST', 'ignore'),
+    ('F.S. PAINT', 'ignore'),
+    ('F.S. SAFETY', 'ignore'),
+    ('REP. TECH. A', 'ignore'),
+    ('REP. TECH. B', 'ignore'),
+    ('REP. WELDER A', 'ignore'),
+    ('REP. WELDER B', 'ignore');
+
+-- ---------------------------------------------------------------- drawing cache
+
+-- One row per drawing number as VISUAL writes it (trimmed). asset_id is the last fetched copy.
+CREATE TABLE drawing_lookups (
+    dwg             text PRIMARY KEY,
+    checked_at      timestamptz,
+    ok              bool NOT NULL DEFAULT true,
+    rev             int,
+    name            text,
+    obj_id          int,
+    obj_version     int,
+    file_id         int,
+    file_version    int,
+    file_size       bigint,
+    base            text,
+    draft_rev       int,
+    draft_where     text,
+    in_drafting     bool NOT NULL DEFAULT false,
+    note            text,
+    fail_count      int NOT NULL DEFAULT 0,
+    failing_since   timestamptz,
+    asset_id        uuid REFERENCES assets (id),
+    sha256          bytea,
+    visual_rev_seen text,
+    next_check_at   timestamptz
+);
+
+CREATE INDEX drawing_lookups_next_check ON drawing_lookups (next_check_at) WHERE next_check_at IS NOT NULL;
+
+-- ---------------------------------------------------------------- worker jobs
+
+-- "Run now": the app inserts a row and NOTIFYs ps_job_request; one run claims every pending row.
+CREATE TABLE job_requests (
+    id             bigserial PRIMARY KEY,
+    job            text NOT NULL,
+    requested_at   timestamptz NOT NULL DEFAULT now(),
+    requested_by   uuid REFERENCES users (id),
+    note           text,
+    claimed_run_id bigint,
+    claimed_at     timestamptz
+);
+
+CREATE INDEX job_requests_pending ON job_requests (job) WHERE claimed_at IS NULL;
+
+CREATE TABLE job_runs (
+    id            bigserial PRIMARY KEY,
+    job           text NOT NULL,
+    trigger       text NOT NULL CHECK (trigger IN ('schedule', 'request', 'chain', 'cli')),
+    requested_by  uuid[] NOT NULL DEFAULT '{}',
+    started_at    timestamptz NOT NULL DEFAULT now(),
+    finished_at   timestamptz,
+    status        text NOT NULL DEFAULT 'running'
+                  CHECK (status IN ('running', 'ok', 'degraded', 'failed', 'skipped', 'abandoned')),
+    counts        jsonb NOT NULL DEFAULT '{}',
+    errors        jsonb NOT NULL DEFAULT '[]',
+    build_version text,
+    visual_target text,
+    host          text
+);
+
+CREATE INDEX job_runs_job ON job_runs (job, id DESC);
+
+-- The next run per job as the worker last scheduled it, for the Admin and Clerical screens.
+CREATE TABLE job_schedule (
+    job              text PRIMARY KEY,
+    next_run_at      timestamptz,
+    interval_seconds int,
+    paused           bool NOT NULL DEFAULT false,
+    updated_at       timestamptz NOT NULL DEFAULT now()
+);
+
+INSERT INTO job_schedule (job) VALUES ('visual-feed'), ('drawing-lookup'), ('housekeeping');
+
+-- One row, written every 30 seconds by the running worker.
+CREATE TABLE worker_heartbeat (
+    id      int PRIMARY KEY CHECK (id = 1),
+    at      timestamptz NOT NULL,
+    build   text,
+    host    text,
+    pid     int,
+    running text[] NOT NULL DEFAULT '{}'
+);
+
+-- ---------------------------------------------------------------- column additions
+
+ALTER TABLE packets
+    ADD COLUMN visual_cancelled_at timestamptz,
+    ADD COLUMN wo_gone_at timestamptz;
+
+-- The leg's own drawing, rev and planner instructions as VISUAL carries them.
+ALTER TABLE packet_work_orders
+    ADD COLUMN partspecs text,
+    ADD COLUMN rev text,
+    ADD COLUMN dwg text;
+
+-- One row per leg: two operations of one work order may sit on one packet. Hand rows keep seq NULL,
+-- and NULLS NOT DISTINCT still allows only one hand row per key.
+DO $$
+DECLARE
+    c text;
+BEGIN
+    SELECT con.conname INTO c
+    FROM pg_constraint con
+    WHERE con.conrelid = 'packet_work_orders'::regclass
+      AND con.contype = 'u'
+      AND (
+          SELECT array_agg(a.attname::text ORDER BY a.attname)
+          FROM unnest(con.conkey) k
+          JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = k
+      ) = ARRAY['lot', 'packet_id', 'split', 'sub', 'wo_base'];
+    IF c IS NULL THEN
+        RAISE EXCEPTION 'packet_work_orders unique key (packet_id, wo_base, lot, split, sub) not found';
+    END IF;
+    EXECUTE format('ALTER TABLE packet_work_orders DROP CONSTRAINT %I', c);
+END;
+$$;
+
+ALTER TABLE packet_work_orders
+    ADD CONSTRAINT packet_work_orders_leg_key UNIQUE NULLS NOT DISTINCT (packet_id, wo_base, lot, split, sub, seq);
+
+-- One pending feed change order per packet, whether VISUAL or M-Files opened it.
+DROP INDEX change_orders_visual_pending;
+CREATE UNIQUE INDEX change_orders_feed_pending ON change_orders (packet_id)
+    WHERE origin IN ('visual', 'drawing') AND review = 'pending';
+
+-- order_ref is the order the feed chose; order_pin is one Clerical pinned (with a reason).
+ALTER TABLE projects
+    ADD COLUMN order_date_visual date,
+    ADD COLUMN ship_date_visual date,
+    ADD COLUMN order_ref jsonb,
+    ADD COLUMN order_pin text,
+    ADD COLUMN order_pin_by uuid REFERENCES users (id),
+    ADD COLUMN order_pin_at timestamptz,
+    ADD COLUMN order_pin_reason text;
+
+ALTER TABLE units
+    ADD COLUMN customer text,
+    ADD COLUMN po text,
+    ADD COLUMN customer_visual text,
+    ADD COLUMN po_visual text,
+    ADD COLUMN order_date_visual date,
+    ADD COLUMN ship_date_visual date,
+    ADD COLUMN order_ref jsonb,
+    ADD COLUMN order_pin text,
+    ADD COLUMN order_pin_by uuid REFERENCES users (id),
+    ADD COLUMN order_pin_at timestamptz,
+    ADD COLUMN order_pin_reason text;
+
+-- v_units expanded u.* when it was made, so it is rebuilt to carry the new unit columns.
+DROP VIEW v_units;
+CREATE VIEW v_units AS
+SELECT u.*,
+       p.number || u.lot AS number,
+       p.number AS project_number,
+       p.name AS project_name,
+       p.status AS project_status,
+       coalesce(u.due, p.due) AS due_effective
+FROM units u
+JOIN projects p ON p.id = u.project_id;
+
+-- Explicit, after the default privileges: the view is read-only to the app.
+REVOKE INSERT, UPDATE, DELETE, TRUNCATE ON v_units FROM ps_app;
+GRANT SELECT ON v_units TO ps_app;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ps_app;
+
+-- The feed COPYs each batch into temp tables before the MERGE. ps_migrate owns the database (initdb and
+-- bootstrap), so it may grant TEMP; stacks made before this migration get the grant here.
+DO $$
+BEGIN
+    EXECUTE format('GRANT TEMPORARY ON DATABASE %I TO ps_app', current_database());
+END;
+$$;

server/packetstation/settings.py

@@ -12,6 +12,8 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 
 ENV_FILE_VAR = "PS_ENV_FILE"
 Stack = Literal["prod", "test", "ci", "dev"]
+FeedSource = Literal["visualkit", "fixture"]
+DrawingSource = Literal["mfiles", "fixture"]
 
 
 class SecretFileError(Exception):
@@ -101,6 +103,11 @@ class Settings(BaseSettings):
     pin_max_digits: int = Field(default=12, ge=4)
     upload_max_bytes: int = Field(default=25 * 1024 * 1024, ge=1024)
 
+    # Where the worker reads VISUAL and M-Files from. `fixture` reads synthetic files under FIXTURE_DIR (CI and tests).
+    feed_source: FeedSource = "visualkit"
+    drawing_source: DrawingSource = "mfiles"
+    fixture_dir: Path = Path("/fixtures")
+
     @model_validator(mode="before")
     @classmethod
     def _blank_is_unset(cls, data: Any) -> Any:
@@ -110,7 +117,7 @@ class Settings(BaseSettings):
                 if key.lower() in _OPTIONAL_PATHS and data[key] in ("", None):
                     data[key] = None
             for key in list(data):
-                if key.lower() == "stack" and isinstance(data[key], str):
+                if key.lower() in ("stack", "feed_source", "drawing_source") and isinstance(data[key], str):
                     data[key] = data[key].strip().lower()
         return data
 
@@ -128,6 +135,12 @@ class Settings(BaseSettings):
             raise ValueError("LDAP_ALLOW_INSECURE=true is refused on STACK=prod")
         return self
 
+    @model_validator(mode="after")
+    def _fixture_sources(self) -> Settings:
+        if self.stack == "prod" and (self.feed_source == "fixture" or self.drawing_source == "fixture"):
+            raise ValueError("FEED_SOURCE=fixture and DRAWING_SOURCE=fixture are refused on STACK=prod")
+        return self
+
     @model_validator(mode="after")
     def _pin_digits(self) -> Settings:
         if self.pin_min_digits > self.pin_max_digits:

server/packetstation/text.py

@@ -46,6 +46,9 @@ LIMITS: dict[str, int] = {
     "doc_title": 160,
     "ecn": 40,
     "name": 160,
+    "order_pin": 40,
+    "file_name": 160,
+    "job_note": 200,
 }
 REASON_MIN = 8
 MAX_BODY_BYTES = 200_000

server/packetstation/worker.py

@@ -1,51 +0,0 @@
-"""Phase 0 worker stub: a heartbeat line every 30 s and a touched beat file; exits on SIGTERM."""
-
-from __future__ import annotations
-
-import logging
-import os
-import signal
-import threading
-import time
-from pathlib import Path
-
-log = logging.getLogger("packetstation.worker")
-
-BEAT_SECONDS = 30
-STALE_SECONDS = 90
-
-
-def touch(path: Path) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.touch(exist_ok=True)
-    now = time.time()
-    os.utime(path, (now, now))
-
-
-def run(beat_file: Path, build: str, stop: threading.Event | None = None, interval: float = BEAT_SECONDS) -> int:
-    stop = stop or threading.Event()
-
-    def _stop(signum: int, _frame: object) -> None:
-        log.info("worker stopping on signal %s", signum)
-        stop.set()
-
-    if threading.current_thread() is threading.main_thread():
-        signal.signal(signal.SIGTERM, _stop)
-        signal.signal(signal.SIGINT, _stop)
-    log.info("worker started", extra={"ps": {"build": build, "beat_file": str(beat_file)}})
-    while not stop.is_set():
-        try:
-            touch(beat_file)
-        except OSError as e:
-            log.error("can't touch beat file %s: %s", beat_file, e.strerror)
-        log.info("heartbeat", extra={"ps": {"build": build}})
-        stop.wait(interval)
-    log.info("worker stopped")
-    return 0
-
-
-def beat_age(beat_file: Path) -> float | None:
-    try:
-        return time.time() - beat_file.stat().st_mtime
-    except OSError:
-        return None

server/packetstation/worker/__init__.py

@@ -0,0 +1,63 @@
+"""The background worker: the job scheduler plus a heartbeat (beat file and worker_heartbeat row).
+
+`run(beat_file, build, stop=None, interval=BEAT_SECONDS, settings=None)` blocks until SIGTERM, SIGINT or `stop`.
+With settings it builds and starts a Scheduler; without (the CLI tests) it only beats the file.
+"""
+
+from __future__ import annotations
+
+import logging
+import signal
+import threading
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from .heartbeat import BEAT_SECONDS, STALE_SECONDS, beat_age, touch
+
+if TYPE_CHECKING:
+    from ..settings import Settings
+
+log = logging.getLogger("packetstation.worker")
+
+__all__ = ["BEAT_SECONDS", "STALE_SECONDS", "beat_age", "run", "touch"]
+
+
+def run(
+    beat_file: Path,
+    build: str,
+    stop: threading.Event | None = None,
+    interval: float = BEAT_SECONDS,
+    settings: Settings | None = None,
+) -> int:
+    stop = stop or threading.Event()
+
+    def _stop(signum: int, _frame: object) -> None:
+        log.info("worker stopping on signal %s", signum)
+        stop.set()
+
+    if threading.current_thread() is threading.main_thread():
+        signal.signal(signal.SIGTERM, _stop)
+        signal.signal(signal.SIGINT, _stop)
+    log.info("worker started", extra={"ps": {"build": build, "beat_file": str(beat_file)}})
+    sched = None
+    if settings is not None:
+        from .scheduler import Scheduler
+
+        sched = Scheduler(settings, build=build)
+        sched.start()
+    try:
+        if sched is not None:
+            sched.serve(beat_file, stop, interval)
+        else:
+            while not stop.is_set():
+                try:
+                    touch(beat_file)
+                except OSError as e:
+                    log.error("can't touch beat file %s: %s", beat_file, e.strerror)
+                log.info("heartbeat", extra={"ps": {"build": build}})
+                stop.wait(interval)
+    finally:
+        if sched is not None:
+            sched.shutdown(wait=True)
+    log.info("worker stopped")
+    return 0

server/packetstation/worker/heartbeat.py

@@ -0,0 +1,36 @@
+"""The worker's heartbeat: a touched beat file for the container healthcheck and a worker_heartbeat row."""
+
+from __future__ import annotations
+
+import os
+import time
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+BEAT_SECONDS = 30
+STALE_SECONDS = 90
+
+
+def touch(path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.touch(exist_ok=True)
+    now = time.time()
+    os.utime(path, (now, now))
+
+
+def beat_age(beat_file: Path) -> float | None:
+    try:
+        return time.time() - beat_file.stat().st_mtime
+    except OSError:
+        return None
+
+
+def write_row(conn: Any, *, build: str, host: str, pid: int, running: Iterable[str]) -> None:
+    """Upsert the single worker_heartbeat row."""
+    conn.execute(
+        "INSERT INTO worker_heartbeat (id, at, build, host, pid, running) VALUES (1, now(), %s, %s, %s, %s)"
+        " ON CONFLICT (id) DO UPDATE SET at = EXCLUDED.at, build = EXCLUDED.build, host = EXCLUDED.host,"
+        " pid = EXCLUDED.pid, running = EXCLUDED.running",
+        (build, host, pid, sorted(running)),
+    )

server/packetstation/worker/jobs/__init__.py

@@ -0,0 +1,146 @@
+"""The job registry and the shapes every job uses.
+
+Jobs import `from packetstation.worker.jobs import JobContext, JobSpec, register` and expose `SPEC`. The feed and
+drawing jobs are imported lazily by `load_jobs`, so the worker runs before they exist.
+"""
+
+from __future__ import annotations
+
+import importlib
+import logging
+import threading
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from datetime import UTC, date, datetime, time, timedelta
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from ...feed.config import FeedSettings
+from ...settings import Settings
+from ..redaction import MAX_ERRORS, clip, redact
+
+log = logging.getLogger(__name__)
+
+Counts = dict[str, Any]
+Cadence = Callable[[FeedSettings, datetime], int | None]
+TRIGGERS = ("schedule", "request", "chain", "cli")
+
+
+@dataclass
+class Stack:
+    """Per-process state shared by every run: the startup guard result and long-lived objects (the M-Files vault)."""
+
+    guard: Any = None
+    cache: dict[str, Any] = field(default_factory=dict)
+    lock: threading.Lock = field(default_factory=threading.Lock)
+
+    def cached(self, key: str, factory: Callable[[], Any]) -> Any:
+        """The object under `key`, made once by `factory`. A factory that raises leaves nothing cached."""
+        with self.lock:
+            if key not in self.cache:
+                self.cache[key] = factory()
+            return self.cache[key]
+
+    def drop(self, key: str) -> None:
+        with self.lock:
+            self.cache.pop(key, None)
+
+
+@dataclass
+class JobContext:
+    """What a run gets. `conn` is the run's own autocommit connection as the worker role."""
+
+    conn: Any
+    settings: Settings
+    feed_settings: FeedSettings
+    run_id: int
+    trigger: str
+    started_at: datetime
+    log: logging.Logger
+    stack: Stack
+    degraded: bool = False
+    errors: list[str] = field(default_factory=list)
+    visual_target: str | None = None
+
+    def error(self, text: str, *, degrade: bool = True) -> None:
+        """Record one error (redacted, clipped). It makes the run degraded unless degrade is False."""
+        if len(self.errors) < MAX_ERRORS:
+            self.errors.extend(clip([text]))
+        if degrade:
+            self.degraded = True
+
+    def degrade(self, reason: str | None = None) -> None:
+        self.degraded = True
+        if reason:
+            self.error(reason)
+
+    @property
+    def tz(self) -> ZoneInfo:
+        return ZoneInfo(self.settings.site_tz)
+
+    def now_site(self) -> datetime:
+        return datetime.now(self.tz)
+
+    def today_site(self) -> date:
+        return self.now_site().date()
+
+
+@dataclass(frozen=True)
+class JobSpec:
+    name: str
+    run: Callable[[JobContext], Counts]
+    cadence: Cadence  # seconds to the next run, or None for "no schedule"
+    pausable: bool  # the feed pause skips scheduled fires (requests still run)
+    chain: tuple[str, ...] = ()  # moved to now after a run that did not fail
+
+
+JOBS: dict[str, JobSpec] = {}
+_loaded = False
+_load_lock = threading.Lock()
+LAZY = ("visual_feed", "drawing_lookup")
+
+
+def register(spec: JobSpec) -> JobSpec:
+    JOBS[spec.name] = spec
+    return spec
+
+
+def load_jobs() -> dict[str, JobSpec]:
+    """Register every job once and return the registry. A job module that cannot import is logged and left out."""
+    global _loaded
+    with _load_lock:
+        if not _loaded:
+            from . import housekeeping
+
+            register(housekeeping.SPEC)
+            for mod in LAZY:
+                try:
+                    m = importlib.import_module(f"{__name__}.{mod}")
+                    register(m.SPEC)
+                except ImportError as e:
+                    log.warning("job module %s not loaded: %s", mod, redact(str(e)))
+            _loaded = True
+    return JOBS
+
+
+def seconds_until(now_site: datetime, at: time) -> int:
+    """Seconds from now_site to the next `at` (site wall clock), at least 1."""
+    target = datetime.combine(now_site.date(), at, tzinfo=now_site.tzinfo)
+    if target <= now_site:
+        target = datetime.combine(now_site.date() + timedelta(days=1), at, tzinfo=now_site.tzinfo)
+    # Aware datetimes sharing a tzinfo subtract as wall clock; through UTC a DST change counts right
+    return max(1, int((target.astimezone(UTC) - now_site.astimezone(UTC)).total_seconds()))
+
+
+__all__ = [
+    "JOBS",
+    "TRIGGERS",
+    "Cadence",
+    "Counts",
+    "JobContext",
+    "JobSpec",
+    "Stack",
+    "load_jobs",
+    "register",
+    "seconds_until",
+]

server/packetstation/worker/jobs/drawing_lookup.py

@@ -0,0 +1,536 @@
+"""The drawing-lookup job: look drawings up in M-Files (read-only), copy released PDFs into assets, and attach,
+flag, stage or open a change order on the packets that use them.
+
+Per run: the queue (drawings/queue.py) cut to `drawing_budget`; per drawing one `Lookup.status`, a fetch when a
+released file exists (drawings/fetch.py downloads only when its M-Files identity changed), the `drawing_lookups`
+row, then `decide` for every live packet that uses the drawing (its own drawing, or the target of its pending feed
+change order), applied through `feed.writes.system_write`. A busy packet is left for the next run.
+
+Never writes to M-Files.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+
+from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
+
+from ... import audit, policy
+from ...domain import pipeline
+from ...drawings import DRAWING_FLAGS, decide, queue
+from ...drawings import lookup as lookups
+from ...drawings.fetch import FetchError, FetchResult, fetch
+from ...drawings.lookup import Found, Lookup
+from ...feed import changes, writes
+from . import Counts, JobContext, JobSpec
+
+log = logging.getLogger(__name__)
+
+A = policy.Action
+D = decide
+
+FEED_ORIGINS = ("visual", "drawing")
+
+
+class _Stale(Exception):
+    """The packet changed between the read and the lock; the next run decides again."""
+
+
+def new_counts() -> Counts:
+    return {
+        "checked": 0,
+        "fetched": 0,
+        "unchanged": 0,
+        "attached": 0,
+        "flagged": {},
+        "lookup_failed": 0,
+        "cos_opened": 0,
+        "cos_reaimed": 0,
+        "staged": 0,
+        "budget_left": 0,
+        "fetch_failed": 0,
+        "busy": 0,
+        "refused": 0,
+        "no_file": 0,
+    }
+
+
+# ---------------------------------------------------------------- the run
+
+
+def run(ctx: JobContext) -> Counts:
+    counts = new_counts()
+    fs = ctx.feed_settings
+    budget = int(getattr(fs, "drawing_budget", 60))
+    try:
+        lk = lookups.make(ctx.settings, ctx.stack.cache)
+    except Exception as e:  # any failure to open M-Files skips the run
+        ctx.error(f"M-Files could not be opened: {type(e).__name__}: {e}")
+        counts["reason"] = "mfiles unavailable"
+        counts["budget_left"] = budget
+        return counts
+    now = datetime.now(UTC)
+    items = queue.queue(ctx.conn, fs, now, limit=budget)
+    counts["budget_left"] = max(0, budget - len(items))
+    counts["source"] = lk.describe()
+    for item in items:
+        try:
+            _drawing(ctx, lk, item, counts, now)
+        except Exception as e:  # one bad drawing must not stop the others
+            log.exception("drawing %s failed", item.dwg)
+            ctx.error(f"drawing {item.dwg}: {type(e).__name__}: {e}")
+    return counts
+
+
+def _drawing(ctx: JobContext, lk: Lookup, item: queue.QueueItem, counts: Counts, now: datetime) -> None:
+    conn = ctx.conn
+    try:
+        found = lk.status(item.dwg)
+    except Exception as e:  # a lookup error is a failed lookup, never a crash
+        found = lookups.failed(item.dwg, f"M-Files lookup failed: {type(e).__name__}: {e}")
+    counts["checked"] += 1
+    fetched: FetchResult | None = None
+    fetch_failed = False
+    if found.ok and found.rev is not None:
+        try:
+            fetched = fetch(conn, ctx.settings, lk, found, dwg=item.dwg)
+        except FetchError as e:
+            fetch_failed = True
+            counts["fetch_failed"] += 1
+            ctx.error(str(e))
+        else:
+            if fetched.downloaded:
+                counts["fetched"] += 1
+                audit.append(
+                    conn,
+                    action="drawing.fetched",
+                    role_snapshot="system",
+                    subject_type="asset",
+                    subject_id=str(fetched.asset_id),
+                    detail={
+                        "dwg": item.dwg,
+                        "rev": found.rev,
+                        "sha256": fetched.sha256.hex(),
+                        "reused": fetched.reused,
+                    },
+                )
+    if not found.ok:
+        counts["lookup_failed"] += 1
+        ctx.error(f"drawing {item.dwg}: {D.failed_note(found)}")
+    vrev_seen = item.visual_rev if item.co_id is None else own_rev(conn, item.dwg)
+    save_lookup(conn, item, found, fetched, now, fetch_failed=fetch_failed, visual_rev_seen=vrev_seen)
+    scope = [str(d) for d in (getattr(ctx.feed_settings, "departments", None) or [])]
+    for pid, relations in sharing(conn, item.dwg, scope):
+        _packet(ctx, item.dwg, found, fetched, pid, relations, counts)
+
+
+# ---------------------------------------------------------------- drawing_lookups
+
+
+def save_lookup(
+    conn: Any,
+    item: queue.QueueItem,
+    found: Found,
+    fetched: FetchResult | None,
+    now: datetime,
+    *,
+    fetch_failed: bool,
+    visual_rev_seen: str | None,
+) -> None:
+    """Upsert the drawing_lookups row for this lookup.
+
+    A failed lookup keeps everything known before (rev, file, asset) and backs off. A good one stores what M-Files
+    has; the file identity and the asset are stored together, and cleared when there is no released file or the
+    copy failed (so the next run downloads again).
+    """
+    with conn.transaction():
+        prev = conn.execute("SELECT fail_count FROM drawing_lookups WHERE dwg = %s FOR UPDATE", (item.dwg,)).fetchone()
+        if not found.ok:
+            fails = (int(prev[0]) if prev else 0) + 1
+            conn.execute(
+                "INSERT INTO drawing_lookups (dwg, checked_at, ok, note, fail_count, failing_since, next_check_at,"
+                " visual_rev_seen) VALUES (%(dwg)s, %(now)s, false, %(note)s, %(fails)s, %(now)s, %(next)s, %(vrev)s)"
+                " ON CONFLICT (dwg) DO UPDATE SET checked_at = EXCLUDED.checked_at, ok = false, note = EXCLUDED.note,"
+                " fail_count = EXCLUDED.fail_count,"
+                " failing_since = coalesce(drawing_lookups.failing_since, EXCLUDED.failing_since),"
+                " next_check_at = EXCLUDED.next_check_at, visual_rev_seen = EXCLUDED.visual_rev_seen",
+                {
+                    "dwg": item.dwg,
+                    "now": now,
+                    "note": D.failed_note(found),
+                    "fails": fails,
+                    "next": now + queue.backoff(fails),
+                    "vrev": visual_rev_seen,
+                },
+            )
+            return
+        has_file = fetched is not None
+        conn.execute(
+            "INSERT INTO drawing_lookups (dwg, checked_at, ok, rev, name, obj_id, obj_version, file_id, file_version,"
+            " file_size, base, draft_rev, draft_where, in_drafting, note, fail_count, failing_since, asset_id, sha256,"
+            " visual_rev_seen, next_check_at) VALUES (%(dwg)s, %(now)s, true, %(rev)s, %(name)s, %(obj_id)s,"
+            " %(obj_version)s, %(file_id)s, %(file_version)s, %(file_size)s, %(base)s, %(draft_rev)s,"
+            " %(draft_where)s, %(in_drafting)s, %(note)s, 0, NULL, %(asset_id)s, %(sha)s, %(vrev)s, %(next)s)"
+            " ON CONFLICT (dwg) DO UPDATE SET checked_at = EXCLUDED.checked_at, ok = true, rev = EXCLUDED.rev,"
+            " name = EXCLUDED.name, obj_id = EXCLUDED.obj_id, obj_version = EXCLUDED.obj_version,"
+            " file_id = EXCLUDED.file_id, file_version = EXCLUDED.file_version, file_size = EXCLUDED.file_size,"
+            " base = EXCLUDED.base, draft_rev = EXCLUDED.draft_rev, draft_where = EXCLUDED.draft_where,"
+            " in_drafting = EXCLUDED.in_drafting, note = EXCLUDED.note, fail_count = 0, failing_since = NULL,"
+            " asset_id = EXCLUDED.asset_id, sha256 = EXCLUDED.sha256,"
+            " visual_rev_seen = coalesce(EXCLUDED.visual_rev_seen, drawing_lookups.visual_rev_seen),"
+            " next_check_at = EXCLUDED.next_check_at",
+            {
+                "dwg": item.dwg,
+                "now": now,
+                "rev": found.rev,
+                "name": found.name,
+                "obj_id": found.obj_id if has_file else None,
+                "obj_version": found.obj_version if has_file else None,
+                "file_id": found.file_id if has_file else None,
+                "file_version": found.file_version if has_file else None,
+                "file_size": found.file_size,
+                "base": found.base,
+                "draft_rev": found.draft_rev,
+                "draft_where": found.draft_where or None,
+                "in_drafting": found.in_drafting,
+                "note": found.note or None,
+                "asset_id": fetched.asset_id if fetched else None,
+                "sha": fetched.sha256 if fetched else None,
+                "vrev": visual_rev_seen,
+                "next": now + queue.backoff(1) if fetch_failed else None,
+            },
+        )
+
+
+# ---------------------------------------------------------------- packets
+
+
+def own_rev(conn: Any, dwg: str) -> str | None:
+    """The rev a live VISUAL packet with this drawing carries (for visual_rev_seen when the item came from a CO)."""
+    row = conn.execute(
+        "SELECT coalesce(btrim(rev), '') FROM packets WHERE removed_at IS NULL AND source_kind = 'visual'"
+        " AND status <> 'filed' AND btrim(drawing) = %s ORDER BY id LIMIT 1",
+        (dwg,),
+    ).fetchone()
+    return None if row is None else str(row[0])
+
+
+SHARING_SQL = """
+SELECT p.id, 'own' AS rel FROM packets p
+WHERE p.removed_at IS NULL AND p.source_kind = 'visual' AND p.status <> 'filed' AND btrim(p.drawing) = %(dwg)s
+  AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[]))
+UNION
+SELECT p.id, 'co' AS rel FROM change_orders c JOIN packets p ON p.id = c.packet_id
+WHERE c.review = 'pending' AND c.origin IN ('visual', 'drawing') AND jsonb_typeof(c.waiting) = 'object'
+  AND btrim(c.waiting->>'dwg') = %(dwg)s AND p.removed_at IS NULL AND p.status NOT IN ('draft', 'filed')
+  AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[]))
+ORDER BY 1
+"""
+
+
+def sharing(conn: Any, dwg: str, depts: list[str]) -> list[tuple[int, set[str]]]:
+    """Live packets that use the drawing: as their own drawing ('own') or as a pending feed CO's target ('co')."""
+    out: dict[int, set[str]] = {}
+    for pid, rel in conn.execute(SHARING_SQL, {"dwg": dwg, "depts": depts}).fetchall():
+        out.setdefault(int(pid), set()).add(str(rel))
+    return sorted(out.items())
+
+
+@dataclass
+class PacketFacts:
+    row: dict[str, Any]
+    visual: dict[str, Any]
+    attached_rev: str | None
+    attached_shas: set[bytes]
+    co: dict[str, Any] | None
+    rejected_sha: bytes | None
+
+
+def _sha(value: Any) -> bytes | None:
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        return bytes.fromhex(value)
+    except ValueError:
+        return None
+
+
+def facts(conn: Any, pid: int) -> PacketFacts | None:
+    with conn.cursor(row_factory=dict_row) as cur:
+        row = cur.execute(
+            "SELECT id, status::text AS status, drawing, rev, drawing_state::text AS drawing_state, flags, visual,"
+            " version FROM packets WHERE id = %s AND removed_at IS NULL",
+            (pid,),
+        ).fetchone()
+        if row is None:
+            return None
+        pages = cur.execute(
+            "SELECT pg.rev, a.sha256 FROM packet_pages pg LEFT JOIN assets a ON a.id = pg.asset_id"
+            " WHERE pg.packet_id = %s AND pg.type = 'drawing' AND pg.void_at IS NULL ORDER BY pg.n",
+            (pid,),
+        ).fetchall()
+        co = cur.execute(
+            "SELECT id, co_no, origin::text AS origin, waiting, staged FROM change_orders"
+            " WHERE packet_id = %s AND review = 'pending' AND origin IN ('visual', 'drawing') ORDER BY id DESC LIMIT 1",
+            (pid,),
+        ).fetchone()
+        rej = cur.execute(
+            "SELECT staged FROM change_orders WHERE packet_id = %s AND origin = 'drawing' AND review = 'rejected'"
+            " ORDER BY id DESC LIMIT 1",
+            (pid,),
+        ).fetchone()
+    visual = row["visual"] if isinstance(row["visual"], dict) else {}
+    attached_rev = None
+    if pages:
+        attached_rev = str(pages[0]["rev"] or "").strip()
+    shas = {bytes(p["sha256"]) for p in pages if p["sha256"] is not None}
+    rejected = None
+    if rej is not None and isinstance(rej["staged"], dict):
+        rejected = _sha(rej["staged"].get("sha256"))
+    return PacketFacts(dict(row), dict(visual), attached_rev, shas, dict(co) if co else None, rejected)
+
+
+def _dwg_rev(value: Any) -> tuple[str, str] | None:
+    if isinstance(value, dict) and value.get("dwg") and value.get("rev") is not None:
+        return (str(value["dwg"]).strip(), str(value["rev"]).strip())
+    return None
+
+
+def target_for(dwg: str, pf: PacketFacts, *, own: bool) -> D.Target:
+    row, visual, co = pf.row, pf.visual, pf.co
+    waiting = co.get("waiting") if co else None
+    staged = co.get("staged") if co else None
+    co_waiting = _dwg_rev(waiting)
+    co_staged = _dwg_rev(staged)
+    packet_rev = str(row["rev"] or "").strip()
+    rev_ok = None
+    ro = visual.get("rev_ok")
+    if isinstance(ro, dict) and ro.get("rev") is not None and str(ro.get("vis_rev") or "").strip() == packet_rev:
+        rev_ok = str(ro["rev"]).strip()
+    visual_rev = packet_rev if own or co_waiting is None else co_waiting[1]
+    return D.Target(
+        dwg=dwg,
+        visual_rev=visual_rev,
+        attached_rev=pf.attached_rev,
+        attached_sha=next(iter(sorted(pf.attached_shas)), None),
+        status=str(row["status"]),
+        drawing_state=str(row["drawing_state"]),
+        has_hand_drawing=row["drawing_state"] == "hand" or bool(visual.get("manual_dwg")),
+        rev_ok=rev_ok,
+        co_waiting=co_waiting,
+        co_staged=co_staged,
+        co_pending=co is not None,
+        co_staged_sha=_sha(staged.get("sha256")) if isinstance(staged, dict) else None,
+        rejected_sha=pf.rejected_sha,
+    )
+
+
+def appr_of(t: D.Target, f: Found, now: datetime) -> dict[str, Any]:
+    return {
+        "ok": f.ok,
+        "rev": f.rev,
+        "vrev": t.visual_rev,
+        "name": f.name,
+        "note": f.note or "",
+        "base": f.base,
+        "draft_rev": f.draft_rev,
+        "draft_where": f.draft_where,
+        "in_drafting": f.in_drafting,
+        "at": now.isoformat(),
+    }
+
+
+def _same_appr(a: Any, b: dict[str, Any]) -> bool:
+    if not isinstance(a, dict):
+        return False
+    return {k: v for k, v in a.items() if k != "at"} == {k: v for k, v in b.items() if k != "at"}
+
+
+def new_flags(flags: list[str], d: D.Decision) -> list[str]:
+    """Every flag but the drawing ones, then the decision's flag."""
+    out = [f for f in flags if f not in DRAWING_FLAGS]
+    if d.flag and d.flag not in out:
+        out.append(d.flag)
+    return out
+
+
+def _packet(
+    ctx: JobContext,
+    dwg: str,
+    found: Found,
+    fetched: FetchResult | None,
+    pid: int,
+    relations: set[str],
+    counts: Counts,
+) -> None:
+    conn = ctx.conn
+    pf = facts(conn, pid)
+    if pf is None:
+        return
+    own = "own" in relations
+    t = target_for(dwg, pf, own=own)
+    same_file = fetched is not None and fetched.sha256 in pf.attached_shas
+    d = D.decide(t, found, same_file=same_file, fetched_sha=fetched.sha256 if fetched else None)
+    if not own and d.kind != D.STAGE_FOR_CO:
+        return  # only the CO's target drawing: nothing to do until it can be staged
+    if d.kind in (D.ATTACH, D.OPEN_CO, D.STAGE_FOR_CO) and fetched is None:
+        counts["no_file"] += 1  # the copy failed this run; try again next run
+        return
+    now = datetime.now(UTC)
+    row, visual = pf.row, pf.visual
+    flags = list(row["flags"] or [])
+    flags_after = new_flags(flags, d) if own else flags
+    patch: dict[str, Any] = {}
+    if own:
+        appr = appr_of(t, found, now)
+        if not _same_appr(visual.get("appr"), appr):
+            patch["appr"] = appr
+        if found.ok:
+            confirmed = D.rev_confirmed(t, found)
+            if bool(visual.get("rev_confirmed")) != confirmed:
+                patch["rev_confirmed"] = confirmed
+    state_after = d.drawing_state if own else str(row["drawing_state"])
+    material = (
+        d.kind in (D.ATTACH, D.OPEN_CO, D.STAGE_FOR_CO)
+        or flags_after != flags
+        or state_after != row["drawing_state"]
+        or bool(patch)
+    )
+    if not material:
+        counts["unchanged"] += 1
+        return
+    action = A.VIS_UPDATE_DRAFT if row["status"] == "draft" else A.VIS_OPEN_CO
+    label, note = "Drawing status from M-Files", d.note or None
+    if d.kind == D.ATTACH and fetched is not None:
+        label = "Drawing attached from M-Files"
+        note = "; ".join(x for x in (fetched.name or found.name or "", d.note) if x) or None
+    detail = {"dwg": dwg, "rev": found.rev, "kind": d.kind, "asset_id": str(fetched.asset_id) if fetched else None}
+    try:
+        with writes.system_write(conn, pid, action, label=label, note=note, detail=detail) as sw:
+            if int(sw.packet["version"]) != int(row["version"]):
+                raise _Stale
+            if d.kind == D.ATTACH and fetched is not None:
+                _attach(sw, pid, dwg, found, fetched, d, flags_after, patch, now)
+                counts["attached"] += 1
+            else:
+                _status(sw, pid, state_after, flags_after, patch)
+                if d.flag:
+                    fl = counts["flagged"]
+                    fl[d.kind] = fl.get(d.kind, 0) + 1
+                if d.kind == D.OPEN_CO and fetched is not None:
+                    _open_co(sw, t, dwg, found, fetched, d, counts)
+                elif d.kind == D.STAGE_FOR_CO and fetched is not None and pf.co is not None:
+                    _stage(sw, pf.co, dwg, found, fetched)
+                    counts["staged"] += 1
+                elif d.kind == D.UNCHANGED:
+                    counts["unchanged"] += 1
+    except (_Stale, writes.Busy, writes.Missing):
+        counts["busy"] += 1
+    except writes.Refused as e:
+        counts["refused"] += 1
+        ctx.error(f"packet {pid}: {d.kind} refused: {e}", degrade=False)
+
+
+def _staged(dwg: str, found: Found, fetched: FetchResult) -> dict[str, Any]:
+    return {
+        "asset_id": str(fetched.asset_id),
+        "name": fetched.name or found.name,
+        "rev": str(found.rev),
+        "dwg": dwg,
+        "sha256": fetched.sha256.hex(),
+        "pages": fetched.pages,
+    }
+
+
+def _attach(
+    sw: Any,
+    pid: int,
+    dwg: str,
+    found: Found,
+    fetched: FetchResult,
+    d: D.Decision,
+    flags: list[str],
+    patch: dict[str, Any],
+    now: datetime,
+) -> None:
+    conn = sw.conn
+    pages = max(1, int(fetched.pages))
+    conn.execute("DELETE FROM packet_pages WHERE packet_id = %s AND type = 'drawing'", (pid,))
+    n0 = pipeline.next_counter(conn, pid, "page", pages)
+    rev = str(found.rev)
+    with conn.cursor() as cur:
+        cur.executemany(
+            "INSERT INTO packet_pages (packet_id, n, type, asset_id, pn, dwg, rev) VALUES (%s, %s, 'drawing', %s, %s,"
+            " %s, %s)",
+            [(pid, n0 + i, fetched.asset_id, i + 1, dwg, rev) for i in range(pages)],
+        )
+    vault = {
+        "asset_id": str(fetched.asset_id),
+        "name": fetched.name or found.name,
+        "rev": found.rev,
+        "note": d.note,
+        "at": now.isoformat(),
+    }
+    merged = {**patch, "rev_confirmed": True, "vault": vault}
+    conn.execute(
+        "UPDATE packets SET drawing_state = 'attached', flags = %s, visual = (visual - 'manual_dwg') || %s"
+        " WHERE id = %s",
+        (flags, Jsonb(merged), pid),
+    )
+    sw.changed("pages", "drawing_state", "flags", "visual")
+
+
+def _status(sw: Any, pid: int, state: str, flags: list[str], patch: dict[str, Any]) -> None:
+    sw.conn.execute(
+        "UPDATE packets SET drawing_state = %s::drawing_state, flags = %s, visual = visual || %s WHERE id = %s",
+        (state, flags, Jsonb(patch), pid),
+    )
+    sw.changed("drawing_state", "flags", "visual")
+
+
+def _open_co(sw: Any, t: D.Target, dwg: str, found: Found, fetched: FetchResult, d: D.Decision, counts: Counts) -> None:
+    vis = {
+        "from_dwg": dwg,
+        "to_dwg": dwg,
+        "dwg": dwg,
+        "from_rev": t.attached_rev,
+        "to_rev": str(found.rev),
+        "dwg_change": True,
+        "changes": [d.note],
+    }
+    res = changes.open_or_reaim(
+        sw,
+        origin="drawing",
+        reason=d.note,
+        vis=vis,
+        waiting=None,
+        staged=_staged(dwg, found, fetched),
+        target_fp=None,
+        minor=False,
+    )
+    # open_or_reaim writes the packet_log label itself
+    counts["cos_opened" if res.fresh else "cos_reaimed"] += 1
+
+
+def _stage(sw: Any, co: dict[str, Any], dwg: str, found: Found, fetched: FetchResult) -> None:
+    staged = _staged(dwg, found, fetched)
+    sw.conn.execute(
+        "UPDATE change_orders SET staged = %s, waiting = NULL, row_version = row_version + 1"
+        " WHERE id = %s AND review = 'pending'",
+        (Jsonb(staged), co["id"]),
+    )
+    sw.changed("change_orders")
+    sw.log(f"Drawing for change order {co['co_no']} ready for Clerical", staged["name"])
+
+
+def _cadence(fs: Any, now: datetime) -> int | None:
+    return int(fs.drawing_interval_minutes) * 60
+
+
+SPEC = JobSpec(name="drawing-lookup", run=run, cadence=_cadence, pausable=True)
+
+__all__ = ["SPEC", "new_flags", "run", "save_lookup", "sharing", "target_for"]

server/packetstation/worker/jobs/housekeeping.py

@@ -0,0 +1,60 @@
+"""Housekeeping, nightly at 03:30 site time: prune the VISUAL change rows and gone cache rows, old job runs and
+old claimed run requests. It never touches packets, the audit log or anything append-only."""
+
+from __future__ import annotations
+
+from datetime import datetime, time
+
+from ...feed.config import FeedSettings
+from . import Counts, JobContext, JobSpec, seconds_until
+
+AT = time(3, 30)
+CACHE_DAYS = 90
+RUNS_DAYS = 180
+REQUESTS_DAYS = 30
+
+STEPS: tuple[tuple[str, str, int], ...] = (
+    ("row_changes", "DELETE FROM visual_row_changes WHERE at < now() - make_interval(days => %s)", CACHE_DAYS),
+    (
+        "material_changes",
+        "DELETE FROM visual_material_changes WHERE at < now() - make_interval(days => %s)",
+        CACHE_DAYS,
+    ),
+    (
+        "gone_rows",
+        "DELETE FROM visual_rows WHERE gone_batch IN"
+        " (SELECT id FROM visual_batches WHERE started_at < now() - make_interval(days => %s))",
+        CACHE_DAYS,
+    ),
+    (
+        "gone_materials",
+        "DELETE FROM visual_materials WHERE gone_batch IN"
+        " (SELECT id FROM visual_batches WHERE started_at < now() - make_interval(days => %s))",
+        CACHE_DAYS,
+    ),
+    (
+        "job_runs",
+        "DELETE FROM job_runs WHERE started_at < now() - make_interval(days => %s) AND status <> 'running'",
+        RUNS_DAYS,
+    ),
+    (
+        "job_requests",
+        "DELETE FROM job_requests WHERE claimed_at IS NOT NULL AND claimed_at < now() - make_interval(days => %s)",
+        REQUESTS_DAYS,
+    ),
+)
+
+
+def run(ctx: JobContext) -> Counts:
+    counts: Counts = {}
+    for name, sql, days in STEPS:
+        with ctx.conn.transaction():
+            counts[name] = ctx.conn.execute(sql, (days,)).rowcount
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    return seconds_until(now_site, AT)
+
+
+SPEC = JobSpec(name="housekeeping", run=run, cadence=cadence, pausable=False)

server/packetstation/worker/jobs/visual_feed.py

@@ -0,0 +1,141 @@
+"""The visual-feed job: read VISUAL (or the fixture files), record the read in the cache, build the candidates
+from the cache and apply them.
+
+Per run: the feed settings; the tracked keys (every work order of a live VISUAL packet that is not filed, plus the
+go-live rows Clerical pulled in); the window from today in site time; one read through the source (labor at most
+every 30 minutes); `cache.write_batch`; `cache.rows_for_apply`; `candidates.build`; `apply.apply`. The run is
+degraded when a read section failed or the batch is suspect.
+
+VisualKit: one `Visual.from_env()` per run, closed after, and the stack guard checked against its settings before
+the first read. Read-only: VisualKit enforces it.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from ... import guard
+from ...feed import apply as feed_apply
+from ...feed import cache, candidates, config
+from ...feed.config import FeedSettings
+from ...feed.source import FeedSource, FixtureSource, VisualKitSource
+from ...feed.types import FeedData, WoKey
+from . import Counts, JobContext, JobSpec
+
+LABOR_EVERY = timedelta(minutes=30)
+
+
+def tracked_keys(conn: Any) -> list[WoKey]:
+    """Work orders of live VISUAL packets that are not filed, plus pulled-in go-live rows."""
+    rows = conn.execute(
+        "SELECT DISTINCT w.wo_base, w.lot, w.split, w.sub FROM packet_work_orders w JOIN packets p"
+        " ON p.id = w.packet_id WHERE p.removed_at IS NULL AND p.status <> 'filed' AND p.source_kind = 'visual'"
+        " UNION SELECT wo_base, lot, split, sub FROM visual_ignored_wos WHERE pulled_in_at IS NOT NULL"
+    ).fetchall()
+    return sorted({WoKey(r[0], r[1], r[2], r[3]) for r in rows}, key=lambda k: k.as_tuple())
+
+
+def resource_map(conn: Any) -> dict[str, str]:
+    return {r[0]: r[1] for r in conn.execute("SELECT resource, target FROM visual_resource_map")}
+
+
+def _labor_due(conn: Any, fs: FeedSettings) -> bool:
+    if not fs.labor_tickets:
+        return False
+    last = cache.last_labor_read(conn)
+    return last is None or datetime.now(UTC) - last >= LABOR_EVERY
+
+
+def _read(source: FeedSource, ctx: JobContext, tracked: list[WoKey], window: tuple[Any, Any], labor: bool) -> FeedData:
+    return source.read(
+        site=ctx.settings.site_id,
+        since=window[0],
+        until=window[1],
+        tracked=tracked,
+        with_materials=True,
+        with_orders=True,
+        with_labor=labor,
+        labor_since=None,
+    )
+
+
+def run(ctx: JobContext) -> Counts:
+    conn = ctx.conn
+    fs = ctx.feed_settings
+    today = ctx.today_site()
+    window = config.window(fs, today)
+    tracked = tracked_keys(conn)
+    labor_due = _labor_due(conn, fs)
+
+    if ctx.settings.feed_source == "fixture":
+        fixture = FixtureSource(ctx.settings.fixture_dir)
+        describe = fixture.describe()
+        data = _read(fixture, ctx, tracked, window, labor_due)
+    else:
+        from visualkit import Visual
+
+        with Visual.from_env() as v:
+            # The stack guard against this run's own VisualKit settings, before the first read.
+            result = guard.check_stack(ctx.settings, v.settings)
+            ctx.visual_target = ctx.visual_target or result.target
+            vk = VisualKitSource(v)
+            describe = vk.describe()
+            data = _read(vk, ctx, tracked, window, labor_due)
+    for e in data.errors:
+        ctx.error(e)
+
+    batch = cache.write_batch(conn, ctx.run_id, data, describe, window)
+    if batch.suspect:
+        ctx.degrade(f"suspect read: {batch.reason}; nothing marked gone")
+
+    in_play = {r.key for r in data.rows} | set(tracked)
+    rows, mats, mat_legs = cache.rows_for_apply(conn, in_play)
+    if data.material_legs:
+        mat_legs |= data.material_legs
+    built = candidates.build(
+        rows,
+        mats,
+        mat_legs,
+        resource_map=resource_map(conn),
+        window=window,
+        golive=fs.golive_date,
+        departments=fs.departments,
+        tracked_keys=set(tracked),
+        today=today,
+    )
+    bases = {r.wo for r in rows}
+    projects = {candidates.project_of(b) for b in bases} | {c.project for c in built.candidates.values()}
+    orders = cache.orders(conn, bases=bases | projects, projects=projects)
+    counts = feed_apply.apply(
+        conn,
+        ctx=ctx,
+        batch=batch,
+        feed_settings=fs,
+        built=built,
+        orders=orders,
+        labor=cache.labor_hours(conn),
+        today=today,
+        seen={r.key: r.st for r in data.rows},
+        missing=() if batch.suspect else data.tracked_missing,
+    )
+    counts["wos_window"] = data.window_count
+    counts["wos_tracked"] = len(tracked)
+    counts["rows"] = len(data.rows)
+    counts["materials"] = len(data.materials) if data.materials is not None else 0
+    counts["orders"] = int(batch.counts.get("orders", 0))
+    counts["labor_legs"] = int(batch.counts.get("labor_legs", 0))
+    counts["batch_id"] = batch.id
+    if batch.suspect:
+        counts["suspect_reason"] = batch.reason
+    cache.finish_batch(conn, batch.id, counts)
+    return counts
+
+
+def _cadence(fs: FeedSettings, now_site: datetime) -> int | None:
+    return config.interval_seconds(fs, now_site)
+
+
+SPEC = JobSpec(name="visual-feed", run=run, cadence=_cadence, pausable=True, chain=("drawing-lookup",))
+
+__all__ = ["SPEC", "resource_map", "run", "tracked_keys"]

server/packetstation/worker/redaction.py

@@ -0,0 +1,62 @@
+"""Redaction for job errors and log lines: no password, key or token ever reaches job_runs or a log."""
+
+from __future__ import annotations
+
+import os
+import re
+
+MAX_ERRORS = 20
+MAX_ERROR_CHARS = 500
+
+# Environment variables whose values are secrets when set (MFilesKit reads MFILES_PASSWORD; VisualKit may read
+# VISUAL_PASSWORD). They are read only to strip them from text, never logged.
+SECRET_ENV = ("VISUAL_PASSWORD", "MFILES_PASSWORD", "PGPASSWORD", "LDAP_PASSWORD")
+SECRET_FILE_ENV = ("VISUAL_PASSWORD_FILE", "MFILES_PASSWORD_FILE")
+
+_PATTERNS = (
+    re.compile(r"(?i)\b(password|passwd|pwd|secret|token|api[_-]?key)\s*[=:]\s*(\"[^\"]*\"|'[^']*'|[^\s;,&]+)"),
+    re.compile(r"(?i)\bauthorization\s*:\s*[^\r\n]*"),
+    re.compile(r"(?i)\bx-authentication\s*:\s*[^\r\n]*"),
+    re.compile(r"(?i)(\w+://[^:/\s@]+:)([^@\s]+)(@)"),
+)
+
+
+def _secrets() -> list[str]:
+    out = []
+    for name in SECRET_ENV:
+        v = os.environ.get(name)
+        if v and len(v) >= 4:
+            out.append(v)
+    for name in SECRET_FILE_ENV:
+        path = os.environ.get(name)
+        if not path:
+            continue
+        try:
+            with open(path, encoding="utf-8") as f:
+                v = f.read().rstrip("\r\n")
+        except OSError:
+            continue
+        if v and len(v) >= 4:
+            out.append(v)
+    return out
+
+
+def redact(text: str) -> str:
+    """Strip anything that looks like a credential, plus the VISUAL and M-Files passwords if set."""
+    s = str(text)
+    for secret in _secrets():
+        s = s.replace(secret, "[redacted]")
+    s = _PATTERNS[0].sub(lambda m: f"{m.group(1)}=[redacted]", s)
+    s = _PATTERNS[1].sub("Authorization: [redacted]", s)
+    s = _PATTERNS[2].sub("X-Authentication: [redacted]", s)
+    s = _PATTERNS[3].sub(lambda m: f"{m.group(1)}[redacted]{m.group(3)}", s)
+    return s
+
+
+def clip(errors: list[str]) -> list[str]:
+    """At most MAX_ERRORS redacted strings of at most MAX_ERROR_CHARS each."""
+    out = []
+    for e in errors[:MAX_ERRORS]:
+        r = redact(e)
+        out.append(r if len(r) <= MAX_ERROR_CHARS else r[: MAX_ERROR_CHARS - 3] + "...")
+    return out

server/packetstation/worker/runs.py

@@ -0,0 +1,173 @@
+"""One run of one job: advisory lock, job_runs row, request claiming, the job itself, the result.
+
+`run_job` works with a started Scheduler (the worker) or an unstarted one (the CLI, the tests): it only needs
+`settings`, `jobs`, `stack`, `host`, `build`, `connect()`, `set_feed()`, `running_add()`, `running_discard()`
+and `after_run()`.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import traceback
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
+
+from .. import guard
+from ..feed import config
+from .jobs import JobContext, JobSpec
+from .redaction import clip, redact
+
+if TYPE_CHECKING:
+    from .scheduler import Scheduler
+
+log = logging.getLogger("packetstation.worker.runs")
+
+LOCK_SPACE = 7310
+FEED_JOB = "visual-feed"
+
+__all__ = ["LOCK_SPACE", "redact", "run_job"]
+
+
+def _jsonable(value: Any) -> Any:
+    return json.loads(json.dumps(value if value is not None else {}, default=str))
+
+
+def _row(conn: Any, run_id: int) -> dict[str, Any]:
+    with conn.cursor(row_factory=dict_row) as cur:
+        row = cur.execute("SELECT * FROM job_runs WHERE id = %s", (run_id,)).fetchone()
+    return dict(row) if row else {"id": run_id}
+
+
+def _insert(conn: Any, sch: Scheduler, name: str, trigger: str, *, status: str = "running", counts: Any = None) -> int:
+    rid = conn.execute(
+        "INSERT INTO job_runs (job, trigger, status, counts, build_version, host, finished_at)"
+        " VALUES (%s, %s, %s, %s, %s, %s, CASE WHEN %s THEN now() END) RETURNING id",
+        (name, trigger, status, Jsonb(_jsonable(counts)), sch.build, sch.host, status != "running"),
+    ).fetchone()[0]
+    return int(rid)
+
+
+def _finish(conn: Any, run_id: int, *, status: str, counts: Any, errors: list[str], visual_target: str | None) -> None:
+    conn.execute(
+        "UPDATE job_runs SET status = %s, counts = %s, errors = %s, visual_target = coalesce(%s, visual_target),"
+        " finished_at = now() WHERE id = %s",
+        (status, Jsonb(_jsonable(counts)), Jsonb(clip(errors)), visual_target, run_id),
+    )
+
+
+def _target(result: Any) -> str:
+    """ "<server> / <database>" from a guard result, without the (DEV) or (PRODUCTION) suffix."""
+    text = str(getattr(result, "target", "") or "")
+    return text.rsplit(" (", 1)[0] if text.endswith(")") else text
+
+
+def run_job(sch: Scheduler, name: str, trigger: str) -> dict[str, Any]:
+    """Run `name` once and return its job_runs row. Never raises for a job failure; raises when the database
+    cannot be reached at all (the caller logs it)."""
+    spec: JobSpec = sch.jobs[name]
+    s = sch.settings
+    conn = sch.connect()
+    locked = False
+    run_id: int | None = None
+    status, counts, errors = "failed", {}, []  # type: tuple[str, Any, list[str]]
+    visual_target: str | None = None
+    pending_after = False
+    sch.running_add(name)
+    try:
+        feed = config.load(conn)
+        sch.set_feed(feed)
+        locked = bool(conn.execute("SELECT pg_try_advisory_lock(%s, hashtext(%s))", (LOCK_SPACE, name)).fetchone()[0])
+        if not locked:
+            rid = _insert(conn, sch, name, trigger, status="skipped", counts={"reason": "already running"})
+            log.info("job %s skipped: already running", name)
+            return _row(conn, rid)
+
+        with conn.transaction():
+            run_id = _insert(conn, sch, name, trigger)
+            claimed = conn.execute(
+                "UPDATE job_requests SET claimed_run_id = %s, claimed_at = now()"
+                " WHERE job = %s AND claimed_at IS NULL RETURNING requested_by",
+                (run_id, name),
+            ).fetchall()
+            if claimed:
+                trigger = "request"
+                who = sorted({r[0] for r in claimed if r[0] is not None}, key=str)
+                conn.execute("UPDATE job_runs SET trigger = 'request', requested_by = %s WHERE id = %s", (who, run_id))
+
+        if trigger == "schedule" and spec.pausable and feed.paused:
+            status, counts = "skipped", {"reason": "paused"}
+        else:
+            ctx = JobContext(
+                conn=conn,
+                settings=s,
+                feed_settings=feed,
+                run_id=run_id,
+                trigger=trigger,
+                started_at=datetime.now(sch.tz),
+                log=logging.getLogger(f"packetstation.worker.jobs.{name}"),
+                stack=sch.stack,
+            )
+            log.info("job %s started", name, extra={"ps": {"run_id": run_id, "trigger": trigger}})
+            try:
+                if name == FEED_JOB and s.feed_source == "visualkit":
+                    # The stack guard again with fresh VisualKit settings: a test stack never reads production
+                    ctx.visual_target = _target(guard.check_stack(s))
+                counts = spec.run(ctx) or {}
+                status = "degraded" if ctx.degraded else "ok"
+            except Exception as e:
+                status = "failed"
+                ctx.errors.append(f"{type(e).__name__}: {e}")
+                log.error("job %s failed: %s", name, redact(traceback.format_exc()))
+            errors, visual_target = ctx.errors, ctx.visual_target
+        try:
+            _finish(conn, run_id, status=status, counts=counts, errors=errors, visual_target=visual_target)
+        except Exception:
+            # The run's connection is broken: write the result on a fresh one
+            with sch.connect() as c2:
+                _finish(c2, run_id, status=status, counts=counts, errors=errors, visual_target=visual_target)
+        try:
+            pending_after = bool(
+                conn.execute(
+                    "SELECT EXISTS (SELECT 1 FROM job_requests WHERE job = %s AND claimed_at IS NULL)", (name,)
+                ).fetchone()[0]
+            )
+            row = _row(conn, run_id)
+        except Exception:
+            with sch.connect() as c2:
+                row = _row(c2, run_id)
+        log.info(
+            "job %s finished: %s", name, status, extra={"ps": {"run_id": run_id, "status": status, "counts": counts}}
+        )
+        return row
+    except Exception as e:
+        status = "failed"
+        if run_id is not None:
+            try:
+                with sch.connect() as c2:
+                    _finish(
+                        c2,
+                        run_id,
+                        status="failed",
+                        counts=counts,
+                        errors=[f"{type(e).__name__}: {e}"],
+                        visual_target=visual_target,
+                    )
+            except Exception as e2:
+                log.error("job %s: can't record the failure: %s", name, type(e2).__name__)
+        raise
+    finally:
+        if locked:
+            try:
+                conn.execute("SELECT pg_advisory_unlock(%s, hashtext(%s))", (LOCK_SPACE, name))
+            except Exception as e:
+                log.warning("job %s: unlock failed (%s); closing the connection releases it", name, type(e).__name__)
+        try:
+            conn.close()
+        finally:
+            sch.running_discard(name)
+            if run_id is not None:
+                sch.after_run(name, status, pending_after)

server/packetstation/worker/scheduler.py

@@ -0,0 +1,395 @@
+"""The worker's scheduler: APScheduler 3 in site time, settings read on every fire, "run now" over NOTIFY.
+
+Each job has a SettingsTrigger whose next fire time comes from the job's cadence and the current feed settings,
+so a changed interval applies on the next fire. A save NOTIFYs `ps_settings` and the listener calls
+`reschedule()` so it applies at once. A "run now" NOTIFYs `ps_job_request` and the listener moves the job to now.
+After a run the job's next fire is measured from the end of the run.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import os
+import socket
+import threading
+from collections.abc import Callable, Iterable
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from apscheduler.executors.pool import ThreadPoolExecutor
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.base import BaseTrigger
+
+from .. import db
+from ..feed import config
+from ..feed.config import FeedSettings
+from ..settings import Settings
+from . import heartbeat, runs
+from .jobs import JobSpec, Stack, load_jobs
+from .redaction import redact
+
+log = logging.getLogger("packetstation.worker")
+
+CHANNELS = ("ps_settings", "ps_job_request")
+IDLE_RECHECK_SECONDS = 3600  # a job with no schedule wakes this often and does nothing
+BACKOFF_MAX = 30.0
+THREADS = 4
+MISFIRE_GRACE = 300
+
+
+class SettingsTrigger(BaseTrigger):
+    """Fires `seconds(now)` after now. `seconds` reads the current settings, so a change applies on the next fire.
+    None means "no schedule": the trigger wakes again after an hour and the fire does nothing."""
+
+    __slots__ = ("seconds",)
+
+    def __init__(self, seconds: Callable[[datetime], int | None]):
+        self.seconds = seconds
+
+    def get_next_fire_time(self, previous_fire_time: datetime | None, now: datetime) -> datetime:
+        secs = self.seconds(now)
+        return now + timedelta(seconds=max(1, IDLE_RECHECK_SECONDS if secs is None else int(secs)))
+
+    def __str__(self) -> str:
+        return "settings"
+
+
+def next_due(
+    spec: JobSpec, feed: FeedSettings, now_site: datetime, last_end_site: datetime | None = None
+) -> datetime | None:
+    """When the job is next due: its cadence measured from the end of its last run (or from now when it never ran),
+    never before now. None when the cadence says "no schedule"."""
+    base = last_end_site if last_end_site is not None and last_end_site <= now_site else now_site
+    secs = spec.cadence(feed, base)
+    if secs is None:
+        return None
+    due = base + timedelta(seconds=max(1, int(secs)))
+    return due if due > now_site else now_site
+
+
+class Scheduler:
+    def __init__(
+        self,
+        settings: Settings,
+        *,
+        build: str | None = None,
+        jobs: dict[str, JobSpec] | None = None,
+        connect: Callable[[], Any] | None = None,
+    ):
+        self.settings = settings
+        self.build = build or settings.ps_build
+        self.jobs = dict(jobs) if jobs is not None else dict(load_jobs())
+        self.tz = ZoneInfo(settings.site_tz)
+        self._connect = connect or (lambda: db.connect_as("worker", settings, autocommit=True))
+        self.stack = Stack()
+        self.host = socket.gethostname()
+        self.pid = os.getpid()
+        self.feed = FeedSettings()
+        self._lock = threading.Lock()
+        self._running: set[str] = set()
+        self._next_trigger: dict[str, str] = {}
+        self._again: dict[str, str] = {}
+        self._last_end: dict[str, datetime] = {}
+        self._db_ready = False
+        self._stop = threading.Event()
+        self._listener: threading.Thread | None = None
+        self.listening = False
+        self.aps = BackgroundScheduler(
+            executors={"default": ThreadPoolExecutor(THREADS)},
+            job_defaults={"coalesce": True, "max_instances": 1, "misfire_grace_time": MISFIRE_GRACE},
+            timezone=self.tz,
+        )
+        self.started = False
+
+    # ------------------------------------------------ small helpers used by runs.run_job
+
+    def connect(self) -> Any:
+        return self._connect()
+
+    def now(self) -> datetime:
+        return datetime.now(self.tz)
+
+    def set_feed(self, feed: FeedSettings) -> None:
+        self.feed = feed
+
+    def running(self) -> list[str]:
+        with self._lock:
+            return sorted(self._running)
+
+    def running_add(self, name: str) -> None:
+        with self._lock:
+            self._running.add(name)
+
+    def running_discard(self, name: str) -> None:
+        with self._lock:
+            self._running.discard(name)
+
+    # ------------------------------------------------ lifecycle
+
+    def start(self) -> None:
+        """Set up the database side when it is reachable, add the jobs and start the scheduler and listener."""
+        self._setup_db()
+        now = self.now()
+        for name, spec in self.jobs.items():
+            self.aps.add_job(
+                self._fire,
+                trigger=SettingsTrigger(self._seconds_for(spec)),
+                args=(name,),
+                id=name,
+                name=name,
+                next_run_time=self._next_time(spec, now),
+                replace_existing=True,
+            )
+        self.aps.start()
+        self.started = True
+        self._write_schedule(self.jobs)
+        self._stop.clear()
+        self._listener = threading.Thread(target=self._listen, name="ps-worker-listen", daemon=True)
+        self._listener.start()
+        log.info("scheduler started", extra={"ps": {"jobs": sorted(self.jobs), "host": self.host}})
+
+    def shutdown(self, wait: bool = True) -> None:
+        self._stop.set()
+        if self.started:
+            self.aps.shutdown(wait=wait)
+            self.started = False
+        if self._listener is not None:
+            self._listener.join(5.0)
+            self._listener = None
+
+    def _setup_db(self) -> bool:
+        """Seed the feed settings, mark this host's leftover runs abandoned, load the settings and the last run
+        ends. Safe to call again; returns False when the database is not reachable."""
+        if self._db_ready:
+            return True
+        try:
+            with self.connect() as conn:
+                with conn.transaction():
+                    if config.seed(conn):
+                        log.info("feed settings seeded with the defaults")
+                self._abandon(conn)
+                self.feed = config.load(conn)
+                for job, end in conn.execute(
+                    "SELECT job, max(finished_at) FROM job_runs WHERE finished_at IS NOT NULL"
+                    " AND status <> 'skipped' GROUP BY job"
+                ).fetchall():
+                    if end is not None:
+                        self._last_end[job] = end.astimezone(self.tz)
+        except Exception as e:
+            log.warning("worker database setup deferred: %s", redact(f"{type(e).__name__}: {e}"))
+            return False
+        self._db_ready = True
+        return True
+
+    def _abandon(self, conn: Any) -> None:
+        """Runs left `running` by a dead worker become `abandoned`: this host's always, another host's when no
+        session holds that job's advisory lock (a container recreated with a new host name)."""
+        rows = conn.execute("SELECT id, job, host FROM job_runs WHERE status = 'running'").fetchall()
+        for rid, job, host in rows:
+            if host != self.host:
+                got = conn.execute("SELECT pg_try_advisory_lock(%s, hashtext(%s))", (runs.LOCK_SPACE, job)).fetchone()[
+                    0
+                ]
+                if not got:
+                    continue
+                conn.execute("SELECT pg_advisory_unlock(%s, hashtext(%s))", (runs.LOCK_SPACE, job))
+            conn.execute(
+                "UPDATE job_runs SET status = 'abandoned', finished_at = now(),"
+                " errors = errors || '[\"The worker stopped during this run.\"]'::jsonb"
+                " WHERE id = %s AND status = 'running'",
+                (rid,),
+            )
+            log.warning("job run %s (%s) marked abandoned", rid, job)
+
+    # ------------------------------------------------ timing
+
+    def _seconds_for(self, spec: JobSpec) -> Callable[[datetime], int | None]:
+        def seconds(now: datetime) -> int | None:
+            return spec.cadence(self.feed, now.astimezone(self.tz))
+
+        return seconds
+
+    def _next_time(self, spec: JobSpec, now: datetime) -> datetime:
+        due = next_due(spec, self.feed, now, self._last_end.get(spec.name))
+        return due if due is not None else now + timedelta(seconds=IDLE_RECHECK_SECONDS)
+
+    def reschedule(self, names: Iterable[str] | None = None) -> None:
+        """Recompute the next run of every job (or of `names`) from the current settings."""
+        if not self.started:
+            return
+        now = self.now()
+        todo = list(names) if names is not None else list(self.jobs)
+        for name in todo:
+            spec = self.jobs.get(name)
+            if spec is None:
+                continue
+            with contextlib.suppress(Exception):  # JobLookupError during shutdown
+                self.aps.modify_job(name, next_run_time=self._next_time(spec, now))
+        self._write_schedule(todo)
+
+    def run_soon(self, name: str, trigger: str = "request") -> None:
+        """Move a job to now. A job that is running now runs again right after."""
+        if name not in self.jobs:
+            log.warning("ignoring a run request for unknown job %r", name[:60])
+            return
+        with self._lock:
+            if name in self._running:
+                self._again[name] = trigger
+                return
+            self._next_trigger[name] = trigger
+        if self.started:
+            with contextlib.suppress(Exception):
+                self.aps.modify_job(name, next_run_time=self.now())
+
+    def after_run(self, name: str, status: str, pending: bool) -> None:
+        """Called by run_job when a run ends: chain, run again for new requests, else time the next run from now."""
+        self._last_end[name] = self.now()
+        spec = self.jobs[name]
+        with self._lock:
+            again = self._again.pop(name, None)
+        if status in ("ok", "degraded"):
+            for c in spec.chain:
+                self.run_soon(c, "chain")
+        if pending or again:
+            self.run_soon(name, again or "request")
+            self._write_schedule([name])
+        else:
+            self.reschedule([name])
+
+    def _fire(self, name: str) -> None:
+        with self._lock:
+            trigger = self._next_trigger.pop(name, "schedule")
+        spec = self.jobs[name]
+        if not self._db_ready and not self._setup_db():
+            log.warning("job %s not run: database not reachable", name)
+            return
+        if trigger == "schedule" and spec.cadence(self.feed, self.now()) is None:
+            self._write_schedule([name])
+            return
+        try:
+            runs.run_job(self, name, trigger)
+        except Exception as e:
+            log.error("job %s could not run: %s", name, redact(f"{type(e).__name__}: {e}"))
+
+    def _write_schedule(self, names: Iterable[str]) -> None:
+        """job_schedule rows for the Admin and Clerical screens. Best effort."""
+        if not self.started:
+            return
+        rows = []
+        now = self.now()
+        for name in names:
+            spec = self.jobs.get(name)
+            if spec is None:
+                continue
+            job = self.aps.get_job(name)
+            nxt = job.next_run_time if job is not None else None
+            secs = spec.cadence(self.feed, now)
+            if secs is None:
+                nxt = None
+            rows.append((name, nxt, secs, bool(spec.pausable and self.feed.paused)))
+        if not rows:
+            return
+        try:
+            with self.connect() as conn:
+                for r in rows:
+                    conn.execute(
+                        "INSERT INTO job_schedule (job, next_run_at, interval_seconds, paused, updated_at)"
+                        " VALUES (%s, %s, %s, %s, now()) ON CONFLICT (job) DO UPDATE SET"
+                        " next_run_at = EXCLUDED.next_run_at, interval_seconds = EXCLUDED.interval_seconds,"
+                        " paused = EXCLUDED.paused, updated_at = now()",
+                        r,
+                    )
+        except Exception as e:
+            log.warning("job_schedule not written: %s", type(e).__name__)
+
+    # ------------------------------------------------ settings
+
+    def reload_settings(self, conn: Any | None = None) -> None:
+        try:
+            if conn is not None:
+                self.feed = config.load(conn)
+            else:
+                with self.connect() as c:
+                    self.feed = config.load(c)
+        except Exception as e:
+            log.warning("feed settings not reloaded: %s", type(e).__name__)
+            return
+        self.reschedule()
+
+    # ------------------------------------------------ listener
+
+    def _on_connect(self, conn: Any) -> None:
+        """After every (re)connect: catch up on what was missed while not listening."""
+        self._setup_db()
+        self.reload_settings(conn)
+        for (job,) in conn.execute("SELECT DISTINCT job FROM job_requests WHERE claimed_at IS NULL").fetchall():
+            self.run_soon(job, "request")
+
+    def _notified(self, channel: str, payload: str) -> None:
+        if channel == "ps_settings":
+            self.reload_settings()
+            return
+        try:
+            data = json.loads(payload)
+        except ValueError:
+            log.warning("ignoring a non-JSON ps_job_request payload")
+            return
+        job = data.get("job") if isinstance(data, dict) else None
+        if isinstance(job, str):
+            self.run_soon(job, "request")
+
+    def _listen(self) -> None:
+        delay = 1.0
+        while not self._stop.is_set():
+            conn = None
+            try:
+                conn = self.connect()
+                for ch in CHANNELS:
+                    conn.execute(f"LISTEN {ch}")
+                self.listening = True
+                delay = 1.0
+                log.info("worker listening on %s", ", ".join(CHANNELS))
+                self._on_connect(conn)
+                while not self._stop.is_set():
+                    for n in conn.notifies(timeout=1.0):
+                        self._notified(n.channel, n.payload)
+            except Exception as e:
+                if not self._stop.is_set():
+                    log.warning("worker listener: %s; retrying in %.0f s", type(e).__name__, delay)
+            finally:
+                self.listening = False
+                if conn is not None:
+                    with contextlib.suppress(Exception):
+                        conn.close()
+            if self._stop.wait(delay):
+                break
+            delay = min(delay * 2, BACKOFF_MAX)
+
+    # ------------------------------------------------ heartbeat loop (main thread)
+
+    def serve(self, beat_file: Path, stop: threading.Event, interval: float) -> None:
+        """Beat until `stop`: the beat file and the worker_heartbeat row."""
+        conn = None
+        while not stop.is_set():
+            try:
+                heartbeat.touch(beat_file)
+            except OSError as e:
+                log.error("can't touch beat file %s: %s", beat_file, e.strerror)
+            try:
+                if conn is None or conn.closed:
+                    conn = self.connect()
+                heartbeat.write_row(conn, build=self.build, host=self.host, pid=self.pid, running=self.running())
+            except Exception as e:
+                log.warning("heartbeat row not written: %s", type(e).__name__)
+                if conn is not None:
+                    with contextlib.suppress(Exception):
+                        conn.close()
+                conn = None
+            stop.wait(interval)
+        if conn is not None:
+            with contextlib.suppress(Exception):
+                conn.close()

server/requirements.txt

@@ -17,6 +17,8 @@ httpx==0.28.1
 tzdata==2026.5
 pypdf==6.21.0
 python-multipart==0.0.32
+pypdfium2==5.14.0
+openpyxl==3.1.5
 
 # needed by visualkit 0.3.0 and mfileskit 0.1.2
 pyodbc==5.3.0
@@ -32,6 +34,7 @@ cffi==2.1.1
 click==8.5.0
 colorama==0.4.6; platform_system == "Windows"
 cryptography==50.0.2; sys_platform == "linux"
+et-xmlfile==2.0.0
 h11==0.16.0
 httpcore==1.0.9
 httptools==0.9.0