Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/server/test_jobs_api.py

@@ -0,0 +1,166 @@
+"""The worker's job API (/api/jobs) and the Admin feed settings and resource map (/api/admin/feed)."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from fastapi.testclient import TestClient
+from pshelpers import app_conn, needs_pg, sign_in
+
+from packetstation import db
+from packetstation.settings import Settings
+
+pytestmark = needs_pg
+
+
+def listen(settings: Settings, channel: str) -> Any:
+    conn = db.connect_as("worker", settings, autocommit=True)
+    conn.execute(f"LISTEN {channel}")
+    return conn
+
+
+def notes(conn: Any) -> list[Any]:
+    return list(conn.notifies(timeout=3, stop_after=1))
+
+
+def test_jobs_list_and_run_now(pg_client: TestClient, pg_settings: Settings) -> None:
+    assert sign_in(pg_client, "clerk.clara").status_code == 200
+    r = pg_client.get("/api/jobs")
+    assert r.status_code == 200, r.text
+    body = r.json()
+    assert [j["job"] for j in body["jobs"]] == ["drawing-lookup", "housekeeping", "visual-feed"]
+    assert body["heartbeat"]["stale"] is True and body["heartbeat"]["at"] is None
+    assert all(j["last_run"] is None and j["pending"] == 0 for j in body["jobs"])
+
+    listener = listen(pg_settings, "ps_job_request")
+    try:
+        r = pg_client.post("/api/jobs/visual-feed/run", json={"note": "check"})
+        assert r.status_code == 200 and r.json() == {"requested": True, "pending": 1}, r.text
+        got = notes(listener)
+        assert got and json.loads(got[0].payload) == {"job": "visual-feed"}
+    finally:
+        listener.close()
+    r = pg_client.post("/api/jobs/visual-feed/run")
+    assert r.json()["pending"] == 2
+    with app_conn(pg_settings) as conn:
+        req = conn.execute(
+            "SELECT r.note, u.sam_account FROM job_requests r JOIN users u ON u.id = r.requested_by ORDER BY r.id"
+        ).fetchall()
+        assert req == [("check", "clerk.clara"), (None, "clerk.clara")]
+        audit = conn.execute(
+            "SELECT role_snapshot, subject_id FROM audit_events WHERE action = 'job.run_requested'"
+        ).fetchall()
+        assert len(audit) == 2 and audit[0][1] == "visual-feed"
+    assert [j["pending"] for j in pg_client.get("/api/jobs").json()["jobs"] if j["job"] == "visual-feed"] == [2]
+
+
+def test_jobs_unknown_and_refused(pg_client: TestClient) -> None:
+    assert sign_in(pg_client, "clerk.clara").status_code == 200
+    assert pg_client.post("/api/jobs/no-such/run").status_code == 404
+    assert pg_client.get("/api/jobs/no-such/runs").status_code == 404
+    assert pg_client.post("/api/jobs/DROP TABLE/run").status_code == 404
+    assert sign_in(pg_client, "wh.wade").status_code == 200
+    assert pg_client.get("/api/jobs").status_code == 403
+    assert pg_client.post("/api/jobs/visual-feed/run").status_code == 403
+
+
+def test_job_runs_listing(pg_client: TestClient, pg_settings: Settings) -> None:
+    with app_conn(pg_settings) as conn:
+        for i in range(12):
+            conn.execute(
+                "INSERT INTO job_runs (job, trigger, status, counts, finished_at) VALUES"
+                " ('visual-feed', 'schedule', 'ok', %s, now())",
+                (json.dumps({"new_drafts": i}),),
+            )
+    assert sign_in(pg_client, "admin.ann").status_code == 200
+    runs = pg_client.get("/api/jobs/visual-feed/runs").json()
+    assert len(runs) == 10 and runs[0]["counts"] == {"new_drafts": 11}
+    assert len(pg_client.get("/api/jobs/visual-feed/runs?limit=3").json()) == 3
+    last = next(j for j in pg_client.get("/api/jobs").json()["jobs"] if j["job"] == "visual-feed")["last_run"]
+    assert last["status"] == "ok" and last["counts"] == {"new_drafts": 11}
+
+
+def test_feed_settings_patch_is_audited_and_notifies(pg_client: TestClient, pg_settings: Settings) -> None:
+    assert sign_in(pg_client, "clerk.clara").status_code == 200
+    assert pg_client.get("/api/admin/feed/settings").status_code == 403
+    assert pg_client.patch("/api/admin/feed/settings", json={"paused": True}).status_code == 403
+
+    assert sign_in(pg_client, "admin.ann").status_code == 200
+    r = pg_client.get("/api/admin/feed/settings")
+    assert r.status_code == 200, r.text
+    s = r.json()
+    assert s["interval_shop_minutes"] == 5 and s["paused"] is False and s["auto_release_max"] == 25
+    assert "runs" in s and "next_run_at" in s
+
+    listener = listen(pg_settings, "ps_settings")
+    try:
+        r = pg_client.patch(
+            "/api/admin/feed/settings",
+            json={"interval_shop_minutes": 7, "golive_date": "2026-11-02", "departments": ["03", "06"]},
+        )
+        assert r.status_code == 200, r.text
+        assert r.json()["interval_shop_minutes"] == 7 and r.json()["golive_date"] == "2026-11-02"
+        assert notes(listener)
+    finally:
+        listener.close()
+    with app_conn(pg_settings) as conn:
+        detail, role = conn.execute(
+            "SELECT detail, role_snapshot FROM audit_events WHERE action = 'feed.settings_changed' ORDER BY id DESC"
+        ).fetchone()
+        assert detail["changed"]["interval_shop_minutes"] == {"from": 5, "to": 7}
+        assert detail["changed"]["departments"] == {"from": [], "to": ["03", "06"]}
+        assert "paused" not in detail["changed"] and "admin" in role
+        assert conn.execute("SELECT value->>'interval_shop_minutes' FROM settings WHERE key = 'feed'").fetchone() == (
+            "7",
+        )
+
+    for bad in (
+        {"interval_shop_minutes": 0},
+        {"shop_start": "6am"},
+        {"nope": 1},
+        {"departments": ["99"]},
+        {"window_back_days": 500},
+    ):
+        r = pg_client.patch("/api/admin/feed/settings", json=bad)
+        assert r.status_code == 422 and r.json()["code"] == "bad_settings", (bad, r.text)
+
+
+def test_resource_map(pg_client: TestClient, pg_settings: Settings) -> None:
+    assert sign_in(pg_client, "clerk.clara").status_code == 200
+    assert pg_client.get("/api/admin/feed/resource-map").status_code == 403
+    assert sign_in(pg_client, "admin.ann").status_code == 200
+    rows = {r["resource"]: r for r in pg_client.get("/api/admin/feed/resource-map").json()}
+    assert rows["WELD-PART"]["target"] == "03" and rows["WELD-PART"]["confirmed"] is False
+    assert rows["MFG"]["target"] == "fold" and rows["FORKLIFT"]["target"] == "ignore"
+    assert rows["Electical"]["target"] == "14"
+
+    r = pg_client.put("/api/admin/feed/resource-map/NEWRES", json={"target": "03"})
+    assert r.status_code == 200 and r.json()["target"] == "03" and r.json()["confirmed"] is True, r.text
+    r = pg_client.put("/api/admin/feed/resource-map/NEWRES", json={"target": "Fold"})
+    assert r.status_code == 200 and r.json()["target"] == "fold"
+    for bad in ("77", "WH", "", "fold; drop"):
+        r = pg_client.put("/api/admin/feed/resource-map/NEWRES", json={"target": bad})
+        assert r.status_code == 422, (bad, r.text)
+    assert pg_client.delete("/api/admin/feed/resource-map/NEWRES").status_code == 204
+    assert pg_client.delete("/api/admin/feed/resource-map/NEWRES").status_code == 404
+    with app_conn(pg_settings) as conn:
+        acts = conn.execute(
+            "SELECT action, detail FROM audit_events WHERE subject_type = 'visual_resource' ORDER BY id"
+        ).fetchall()
+        assert [a[0] for a in acts] == ["feed.resource_mapped", "feed.resource_mapped", "feed.resource_unmapped"]
+        assert acts[1][1]["before"]["target"] == "03" and acts[1][1]["after"]["target"] == "fold"
+        assert acts[2][1]["after"] is None
+
+
+def test_resource_map_lists_unmapped_from_the_last_batch(
+    pg_client: TestClient, pg_settings: Settings, tmp_path: Any
+) -> None:
+    from feedhelpers import feed_settings_over, run_feed
+
+    fx = feed_settings_over(pg_settings, tmp_path)
+    run_feed(fx, 1)
+    assert sign_in(pg_client, "admin.ann").status_code == 200
+    rows = {r["resource"]: r for r in pg_client.get("/api/admin/feed/resource-map").json()}
+    assert rows["NEWRES"]["target"] is None and rows["NEWRES"]["rows_last_run"] == 2
+    assert rows["WELD-PART"]["rows_last_run"] >= 1