Phase 3: VISUAL feed and M-Files drawings in the worker
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,308 @@
|
||||
"""The VISUAL feed for Clerical: the From VISUAL panel (status, runs, ignored work orders, unmapped resources,
|
||||
projects without names), Run now, pulling in a work order released before go-live, and what VISUAL changed.
|
||||
|
||||
Clerical (clerical or clerical_supervisor) or an admin. Pulling in a work order changes what the feed tracks, so an
|
||||
admin outside Clerical gives an IT reason for it. Screens read PostgreSQL only: nothing here calls VISUAL. Run now
|
||||
inserts a job request and NOTIFYs the worker, which does the reading.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .. import audit
|
||||
from ..domain import packets
|
||||
from ..domain.pipeline import jsonable
|
||||
from ..text import LIMITS, clean_opt, reason_ok
|
||||
from .deps import Db, Principal, current_principal, get_db, require_password_session
|
||||
from .errors import ApiError
|
||||
from .packets import WRITE
|
||||
|
||||
router = APIRouter(prefix="/api/feed", tags=["feed"])
|
||||
|
||||
CLERICAL = frozenset({"clerical", "clerical_supervisor"})
|
||||
JOBS = ("visual-feed", "drawing-lookup")
|
||||
IGNORED_MAX = 200
|
||||
UNMAPPED_MAX = 30
|
||||
STALE_DEFAULT = 90
|
||||
PULL_IN_REASON_MIN = 3
|
||||
PULL_IN_REASON_MAX = 300
|
||||
|
||||
|
||||
class RunBody(BaseModel):
|
||||
job: Literal["visual-feed", "drawing-lookup"] = "visual-feed"
|
||||
note: str | None = Field(default=None, max_length=1000)
|
||||
|
||||
|
||||
class PullInBody(BaseModel):
|
||||
reason: str | None = Field(default=None, max_length=2000)
|
||||
it_reason: str | None = Field(default=None, max_length=1000)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- access
|
||||
|
||||
|
||||
def require_clerical(p: Principal = Depends(current_principal)) -> Principal: # noqa: B008
|
||||
if CLERICAL & set(p.roles) or p.is_admin:
|
||||
return p
|
||||
raise ApiError(403, "forbidden", "Only Clerical can see the VISUAL feed.")
|
||||
|
||||
|
||||
def _it_reason(p: Principal, it_reason: str | None) -> str | None:
|
||||
"""None for Clerical. An admin outside Clerical needs a password session and an IT reason."""
|
||||
if CLERICAL & set(p.roles):
|
||||
return None
|
||||
require_password_session(p)
|
||||
why = reason_ok(it_reason)
|
||||
if why is None:
|
||||
raise ApiError(400, "it_reason_required", "Give an IT reason of at least 8 characters.")
|
||||
return why
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- shapes
|
||||
|
||||
|
||||
def _run_out(r: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": r["id"],
|
||||
"job": r["job"],
|
||||
"trigger": r["trigger"],
|
||||
"requested_by": [str(u) for u in r.get("requested_by") or []],
|
||||
"started_at": r["started_at"],
|
||||
"finished_at": r["finished_at"],
|
||||
"status": r["status"],
|
||||
"counts": r["counts"] or {},
|
||||
"errors": r["errors"] or [],
|
||||
"build_version": r.get("build_version"),
|
||||
"visual_target": r.get("visual_target"),
|
||||
"host": r.get("host"),
|
||||
}
|
||||
|
||||
|
||||
def _runs(conn: Any, job: str, limit: int) -> list[dict[str, Any]]:
|
||||
return [
|
||||
_run_out(r)
|
||||
for r in packets.rows(conn, "SELECT * FROM job_runs WHERE job = %s ORDER BY id DESC LIMIT %s", (job, limit))
|
||||
]
|
||||
|
||||
|
||||
def _ignored_out(r: dict[str, Any]) -> dict[str, Any]:
|
||||
keys = ("id", "wo_base", "lot", "split", "sub", "rel", "project", "unit_number", "res", "dwg", "last_seen_at")
|
||||
out = {k: r[k] for k in keys}
|
||||
out["pulled_in_at"] = r["pulled_in_at"]
|
||||
out["pulled_in_by"] = str(r["pulled_in_by"]) if r["pulled_in_by"] else None
|
||||
out["pulled_in_reason"] = r["pulled_in_reason"]
|
||||
return out
|
||||
|
||||
|
||||
def _stale_seconds() -> int:
|
||||
try:
|
||||
from ..worker import STALE_SECONDS
|
||||
except ImportError: # the worker package is agent A's; the panel still answers without it
|
||||
return STALE_DEFAULT
|
||||
return int(STALE_SECONDS)
|
||||
|
||||
|
||||
def _feed_settings(conn: Any) -> Any:
|
||||
from ..feed import config
|
||||
|
||||
return config.load(conn)
|
||||
|
||||
|
||||
def _request_run(conn: Any, p: Principal, job: str, note: str | None) -> int:
|
||||
"""Insert a job request and NOTIFY the worker in the same transaction. Returns the pending request count."""
|
||||
conn.execute(
|
||||
"INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)",
|
||||
(job, p.user_id, clean_opt(note, LIMITS["job_note"])),
|
||||
)
|
||||
conn.execute("SELECT ps_notify('ps_job_request', %s)", (json.dumps({"job": job}),))
|
||||
return int(
|
||||
conn.execute("SELECT count(*) FROM job_requests WHERE job = %s AND claimed_at IS NULL", (job,)).fetchone()[0]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- endpoints
|
||||
|
||||
|
||||
@router.get("/overview")
|
||||
def overview(
|
||||
p: Principal = Depends(require_clerical), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> dict[str, Any]:
|
||||
"""Everything the From VISUAL panel shows, from PostgreSQL only."""
|
||||
conn = d.conn
|
||||
fs = _feed_settings(conn)
|
||||
sched = packets.one(conn, "SELECT next_run_at, paused FROM job_schedule WHERE job = 'visual-feed'")
|
||||
beat = packets.one(
|
||||
conn,
|
||||
"SELECT at, build, running, now() - at > make_interval(secs => %s) AS stale FROM worker_heartbeat WHERE id = 1",
|
||||
(_stale_seconds(),),
|
||||
)
|
||||
runs = _runs(conn, "visual-feed", 10)
|
||||
last_run = runs[0] if runs else None
|
||||
finished = next((r for r in runs if r["status"] in ("ok", "degraded")), None)
|
||||
unmapped_raw = (finished or {}).get("counts", {}).get("unmapped") or {}
|
||||
unmapped: list[dict[str, Any]] = []
|
||||
if isinstance(unmapped_raw, dict):
|
||||
for k, v in unmapped_raw.items():
|
||||
if isinstance(v, int) or (isinstance(v, str) and v.isdigit()):
|
||||
unmapped.append({"resource": str(k), "rows": int(v)})
|
||||
unmapped.sort(key=lambda x: (-int(x["rows"]), str(x["resource"])))
|
||||
unmapped = unmapped[:UNMAPPED_MAX]
|
||||
ignored = packets.rows(
|
||||
conn,
|
||||
"SELECT * FROM visual_ignored_wos WHERE pulled_in_at IS NULL ORDER BY rel DESC NULLS LAST, wo_base, lot, id"
|
||||
" LIMIT %s",
|
||||
(IGNORED_MAX,),
|
||||
)
|
||||
nameless = packets.rows(
|
||||
conn,
|
||||
"SELECT p.id, p.number,"
|
||||
" (SELECT count(*) FROM units u WHERE u.project_id = p.id AND u.removed_at IS NULL) AS units,"
|
||||
" (SELECT count(*) FROM packets k JOIN units u ON u.id = k.unit_id"
|
||||
" WHERE u.project_id = p.id AND k.removed_at IS NULL) AS packets"
|
||||
" FROM projects p WHERE p.source = 'visual' AND p.needs_details AND p.name IS NULL AND p.removed_at IS NULL"
|
||||
" ORDER BY p.number LIMIT 200",
|
||||
)
|
||||
return {
|
||||
"status": {
|
||||
"paused": bool(fs.paused),
|
||||
"next_run_at": (sched or {}).get("next_run_at"),
|
||||
"worker": None
|
||||
if beat is None
|
||||
else {
|
||||
"at": beat["at"],
|
||||
"build": beat["build"],
|
||||
"running": list(beat["running"] or []),
|
||||
"stale": bool(beat["stale"]),
|
||||
},
|
||||
},
|
||||
"last_run": last_run,
|
||||
"runs": runs,
|
||||
"drawing_runs": _runs(conn, "drawing-lookup", 10),
|
||||
"pending_requests": int(
|
||||
conn.execute(
|
||||
"SELECT count(*) FROM job_requests WHERE job = 'visual-feed' AND claimed_at IS NULL"
|
||||
).fetchone()[0]
|
||||
),
|
||||
"ignored": [_ignored_out(r) for r in ignored],
|
||||
"unmapped": unmapped,
|
||||
"projects_without_names": [
|
||||
{"id": r["id"], "number": r["number"], "units": int(r["units"]), "packets": int(r["packets"])}
|
||||
for r in nameless
|
||||
],
|
||||
"golive_date": fs.golive_date,
|
||||
"departments": list(fs.departments),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/run", dependencies=WRITE)
|
||||
def run_now(
|
||||
body: RunBody | None = None,
|
||||
p: Principal = Depends(require_clerical), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> dict[str, Any]:
|
||||
"""Run now: ask the worker for a feed (or drawing) run. Requests made while one is pending share that run."""
|
||||
body = body or RunBody()
|
||||
conn = d.conn
|
||||
with conn.transaction():
|
||||
pending = _request_run(conn, p, body.job, body.note)
|
||||
audit.append(
|
||||
conn,
|
||||
action="feed.run_requested",
|
||||
user_id=p.user_id,
|
||||
session_id=p.session_id,
|
||||
role_snapshot=p.role_snapshot,
|
||||
subject_type="job",
|
||||
subject_id=body.job,
|
||||
detail={"job": body.job, "pending": pending},
|
||||
ip=p.ip,
|
||||
)
|
||||
return {"requested": True, "job": body.job, "pending": pending}
|
||||
|
||||
|
||||
@router.post("/ignored/{row_id}/pull-in", dependencies=WRITE)
|
||||
def pull_in(
|
||||
row_id: int,
|
||||
body: PullInBody,
|
||||
p: Principal = Depends(require_clerical), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> dict[str, Any]:
|
||||
"""Track a work order released before the go-live date after all. The next feed run makes its packet."""
|
||||
it_reason = _it_reason(p, body.it_reason)
|
||||
reason = clean_opt(body.reason, PULL_IN_REASON_MAX)
|
||||
if reason is None or len(reason) < PULL_IN_REASON_MIN:
|
||||
raise ApiError(400, "reason_required", "Say why this work order is pulled in (at least 3 characters).")
|
||||
conn = d.conn
|
||||
with conn.transaction():
|
||||
row = packets.one(conn, "SELECT * FROM visual_ignored_wos WHERE id = %s FOR UPDATE", (row_id,))
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "No such work order on the ignored list.")
|
||||
if row["pulled_in_at"] is not None:
|
||||
raise ApiError(
|
||||
409,
|
||||
"already_pulled_in",
|
||||
"This work order was already pulled in.",
|
||||
extra={"current": jsonable(_ignored_out(row))},
|
||||
)
|
||||
row = packets.one(
|
||||
conn,
|
||||
"UPDATE visual_ignored_wos SET pulled_in_at = now(), pulled_in_by = %s, pulled_in_reason = %s"
|
||||
" WHERE id = %s RETURNING *",
|
||||
(p.user_id, reason, row_id),
|
||||
)
|
||||
assert row is not None
|
||||
key = "/".join(str(row[k]) for k in ("wo_base", "lot", "split", "sub"))
|
||||
audit.append(
|
||||
conn,
|
||||
action="feed.pulled_in",
|
||||
user_id=p.user_id,
|
||||
session_id=p.session_id,
|
||||
role_snapshot=p.role_snapshot,
|
||||
subject_type="work_order",
|
||||
subject_id=key,
|
||||
detail={"id": row_id, "key": key, "rel": str(row["rel"]) if row["rel"] else None, "reason": reason},
|
||||
it_reason=it_reason,
|
||||
ip=p.ip,
|
||||
)
|
||||
_request_run(conn, p, "visual-feed", f"pull in {key}")
|
||||
return _ignored_out(row)
|
||||
|
||||
|
||||
@router.get("/batches")
|
||||
def batches(
|
||||
limit: int = Query(20, ge=1, le=200),
|
||||
p: Principal = Depends(require_clerical), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> list[dict[str, Any]]:
|
||||
return packets.rows(d.conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT %s", (limit,))
|
||||
|
||||
|
||||
@router.get("/changes")
|
||||
def changes(
|
||||
batch_id: int | None = Query(None),
|
||||
limit: int = Query(200, ge=1, le=2000),
|
||||
p: Principal = Depends(require_clerical), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> dict[str, Any]:
|
||||
"""What VISUAL changed in one batch (the latest when none is given): work order rows and material lines."""
|
||||
conn = d.conn
|
||||
if batch_id is None:
|
||||
batch = packets.one(conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT 1")
|
||||
else:
|
||||
batch = packets.one(conn, "SELECT * FROM visual_batches WHERE id = %s", (batch_id,))
|
||||
if batch is None:
|
||||
raise ApiError(404, "not_found", "No such feed batch.")
|
||||
if batch is None:
|
||||
return {"batch": None, "rows": [], "materials": []}
|
||||
rows = packets.rows(
|
||||
conn, "SELECT * FROM visual_row_changes WHERE batch_id = %s ORDER BY id LIMIT %s", (batch["id"], limit)
|
||||
)
|
||||
mats = packets.rows(
|
||||
conn, "SELECT * FROM visual_material_changes WHERE batch_id = %s ORDER BY id LIMIT %s", (batch["id"], limit)
|
||||
)
|
||||
return {"batch": batch, "rows": rows, "materials": mats}
|
||||
Reference in New Issue
Block a user