Phase 3: VISUAL feed and M-Files drawings in the worker
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,204 @@
|
||||
"""Admin: the feed settings and the VISUAL resource map. Every route needs the admin role and a password session.
|
||||
|
||||
A settings save NOTIFYs `ps_settings` so the worker reschedules at once. Resource map changes apply on the next
|
||||
feed run. Every change is audited with before and after.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, Response
|
||||
from psycopg.rows import dict_row
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .. import audit
|
||||
from ..domain.projects import Who
|
||||
from ..feed import config
|
||||
from .deps import Db, Principal, admin_principal, get_db
|
||||
from .errors import ApiError
|
||||
from .jobs import recent_runs
|
||||
from .packets import WRITE
|
||||
|
||||
router = APIRouter(prefix="/api/admin/feed", tags=["admin"])
|
||||
|
||||
FEED_JOB = "visual-feed"
|
||||
SPECIAL_TARGETS = ("fold", "ignore")
|
||||
RESOURCE_MAX = 100
|
||||
|
||||
|
||||
def _who(p: Principal) -> Who:
|
||||
return Who(user_id=p.user_id, session_id=p.session_id, role_snapshot=p.role_snapshot, ip=p.ip)
|
||||
|
||||
|
||||
def _shop_codes(conn: Any) -> set[str]:
|
||||
rows = conn.execute("SELECT code FROM departments WHERE kind = 'shop' AND active").fetchall()
|
||||
return {r[0] for r in rows}
|
||||
|
||||
|
||||
def _settings_out(conn: Any) -> dict[str, Any]:
|
||||
fs, version, updated_at, updated_by = config.load_versioned(conn)
|
||||
nxt = conn.execute("SELECT next_run_at FROM job_schedule WHERE job = %s", (FEED_JOB,)).fetchone()
|
||||
return {
|
||||
**fs.model_dump(mode="json"),
|
||||
"version": version,
|
||||
"updated_at": updated_at,
|
||||
"updated_by": str(updated_by) if updated_by else None,
|
||||
"next_run_at": nxt[0] if nxt else None,
|
||||
"runs": recent_runs(conn, FEED_JOB, 10),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/settings")
|
||||
def get_settings(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> dict[str, Any]: # noqa: B008
|
||||
return _settings_out(d.conn)
|
||||
|
||||
|
||||
@router.patch("/settings", dependencies=WRITE)
|
||||
def patch_settings(
|
||||
patch: dict[str, Any] = Body(...), # noqa: B008
|
||||
p: Principal = Depends(admin_principal), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> dict[str, Any]:
|
||||
conn = d.conn
|
||||
if "departments" in patch and isinstance(patch["departments"], list):
|
||||
shop = _shop_codes(conn)
|
||||
bad = sorted({str(x).strip() for x in patch["departments"]} - shop)
|
||||
if bad:
|
||||
raise ApiError(
|
||||
422,
|
||||
"bad_settings",
|
||||
f"Not an active shop department: {', '.join(bad)}.",
|
||||
extra={"errors": [f"departments: {b} is not an active shop department" for b in bad]},
|
||||
)
|
||||
try:
|
||||
config.save(conn, who=_who(p), patch=patch)
|
||||
except config.SettingsError as e:
|
||||
raise ApiError(422, "bad_settings", str(e), extra={"errors": e.errors}) from None
|
||||
return _settings_out(conn)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- resource map
|
||||
|
||||
|
||||
class MapBody(BaseModel):
|
||||
target: str = Field(max_length=20)
|
||||
confirmed: bool = True
|
||||
|
||||
|
||||
def _resource(resource: str) -> str:
|
||||
r = resource.strip()
|
||||
if not r or len(r) > RESOURCE_MAX:
|
||||
raise ApiError(400, "bad_resource", "Give the VISUAL resource name.")
|
||||
return r
|
||||
|
||||
|
||||
def _map_row(conn: Any, resource: str) -> dict[str, Any] | None:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
r = cur.execute(
|
||||
"SELECT resource, target, confirmed, updated_at, updated_by FROM visual_resource_map WHERE resource = %s",
|
||||
(resource,),
|
||||
).fetchone()
|
||||
return dict(r) if r else None
|
||||
|
||||
|
||||
def _row_out(r: dict[str, Any], rows_last_run: int | None) -> dict[str, Any]:
|
||||
return {
|
||||
"resource": r["resource"],
|
||||
"target": r.get("target"),
|
||||
"confirmed": bool(r.get("confirmed")),
|
||||
"rows_last_run": rows_last_run,
|
||||
"updated_at": r.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
def _last_counts(conn: Any) -> dict[str, int]:
|
||||
rows = conn.execute(
|
||||
"SELECT res, count(*) FROM visual_rows"
|
||||
" WHERE last_batch = (SELECT max(id) FROM visual_batches) AND res <> '' GROUP BY res"
|
||||
).fetchall()
|
||||
return {r[0]: int(r[1]) for r in rows}
|
||||
|
||||
|
||||
@router.get("/resource-map")
|
||||
def resource_map(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> list[dict[str, Any]]: # noqa: B008
|
||||
conn = d.conn
|
||||
counts = _last_counts(conn)
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
mapped = cur.execute("SELECT resource, target, confirmed, updated_at FROM visual_resource_map").fetchall()
|
||||
out = {r["resource"]: _row_out(r, counts.get(r["resource"])) for r in mapped}
|
||||
for res, n in counts.items():
|
||||
if res not in out:
|
||||
out[res] = {"resource": res, "target": None, "confirmed": False, "rows_last_run": n, "updated_at": None}
|
||||
return sorted(out.values(), key=lambda r: (r["target"] is not None, r["resource"].lower()))
|
||||
|
||||
|
||||
def _public(r: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
if r is None:
|
||||
return None
|
||||
return {"target": r["target"], "confirmed": r["confirmed"]}
|
||||
|
||||
|
||||
@router.put("/resource-map/{resource}", dependencies=WRITE)
|
||||
def map_resource(
|
||||
resource: str,
|
||||
body: MapBody,
|
||||
p: Principal = Depends(admin_principal), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> dict[str, Any]:
|
||||
conn = d.conn
|
||||
res = _resource(resource)
|
||||
target = body.target.strip()
|
||||
if target.lower() in SPECIAL_TARGETS:
|
||||
target = target.lower()
|
||||
elif target not in _shop_codes(conn):
|
||||
raise ApiError(422, "bad_target", "The target is Fold, Ignore or an active shop department.")
|
||||
with conn.transaction():
|
||||
before = _map_row(conn, res)
|
||||
conn.execute(
|
||||
"INSERT INTO visual_resource_map (resource, target, confirmed, updated_at, updated_by)"
|
||||
" VALUES (%s, %s, %s, now(), %s) ON CONFLICT (resource) DO UPDATE SET target = EXCLUDED.target,"
|
||||
" confirmed = EXCLUDED.confirmed, updated_at = now(), updated_by = EXCLUDED.updated_by",
|
||||
(res, target, body.confirmed, p.user_id),
|
||||
)
|
||||
after = _map_row(conn, res)
|
||||
audit.append(
|
||||
conn,
|
||||
action="feed.resource_mapped",
|
||||
user_id=p.user_id,
|
||||
session_id=p.session_id,
|
||||
role_snapshot=p.role_snapshot,
|
||||
subject_type="visual_resource",
|
||||
subject_id=res,
|
||||
detail={"resource": res, "before": _public(before), "after": _public(after)},
|
||||
ip=p.ip,
|
||||
)
|
||||
assert after is not None
|
||||
return _row_out(after, _last_counts(conn).get(res))
|
||||
|
||||
|
||||
@router.delete("/resource-map/{resource}", status_code=204, dependencies=WRITE)
|
||||
def unmap_resource(
|
||||
resource: str,
|
||||
p: Principal = Depends(admin_principal), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> Response:
|
||||
conn = d.conn
|
||||
res = _resource(resource)
|
||||
with conn.transaction():
|
||||
before = _map_row(conn, res)
|
||||
if before is None:
|
||||
raise ApiError(404, "not_found", "That resource is not mapped.")
|
||||
conn.execute("DELETE FROM visual_resource_map WHERE resource = %s", (res,))
|
||||
audit.append(
|
||||
conn,
|
||||
action="feed.resource_unmapped",
|
||||
user_id=p.user_id,
|
||||
session_id=p.session_id,
|
||||
role_snapshot=p.role_snapshot,
|
||||
subject_type="visual_resource",
|
||||
subject_id=res,
|
||||
detail={"resource": res, "before": _public(before), "after": None},
|
||||
ip=p.ip,
|
||||
)
|
||||
return Response(status_code=204)
|
||||
Reference in New Issue
Block a user