309 lines
12 KiB
Python
309 lines
12 KiB
Python
 
|
"""The VISUAL feed for Clerical: the From VISUAL panel (status, runs, ignored work orders, unmapped resources,
|
||
|
|
projects without names), Run now, pulling in a work order released before go-live, and what VISUAL changed.
|
||
|
|
|
||
|
|
Clerical (clerical or clerical_supervisor) or an admin. Pulling in a work order changes what the feed tracks, so an
|
||
|
|
admin outside Clerical gives an IT reason for it. Screens read PostgreSQL only: nothing here calls VISUAL. Run now
|
||
|
|
inserts a job request and NOTIFYs the worker, which does the reading.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
from typing import Any, Literal
|
||
|
|
|
||
|
|
from fastapi import APIRouter, Depends, Query
|
||
|
|
from pydantic import BaseModel, Field
|
||
|
|
|
||
|
|
from .. import audit
|
||
|
|
from ..domain import packets
|
||
|
|
from ..domain.pipeline import jsonable
|
||
|
|
from ..text import LIMITS, clean_opt, reason_ok
|
||
|
|
from .deps import Db, Principal, current_principal, get_db, require_password_session
|
||
|
|
from .errors import ApiError
|
||
|
|
from .packets import WRITE
|
||
|
|
|
||
|
|
router = APIRouter(prefix="/api/feed", tags=["feed"])
|
||
|
|
|
||
|
|
CLERICAL = frozenset({"clerical", "clerical_supervisor"})
|
||
|
|
JOBS = ("visual-feed", "drawing-lookup")
|
||
|
|
IGNORED_MAX = 200
|
||
|
|
UNMAPPED_MAX = 30
|
||
|
|
STALE_DEFAULT = 90
|
||
|
|
PULL_IN_REASON_MIN = 3
|
||
|
|
PULL_IN_REASON_MAX = 300
|
||
|
|
|
||
|
|
|
||
|
|
class RunBody(BaseModel):
|
||
|
|
job: Literal["visual-feed", "drawing-lookup"] = "visual-feed"
|
||
|
|
note: str | None = Field(default=None, max_length=1000)
|
||
|
|
|
||
|
|
|
||
|
|
class PullInBody(BaseModel):
|
||
|
|
reason: str | None = Field(default=None, max_length=2000)
|
||
|
|
it_reason: str | None = Field(default=None, max_length=1000)
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------- access
|
||
|
|
|
||
|
|
|
||
|
|
def require_clerical(p: Principal = Depends(current_principal)) -> Principal: # noqa: B008
|
||
|
|
if CLERICAL & set(p.roles) or p.is_admin:
|
||
|
|
return p
|
||
|
|
raise ApiError(403, "forbidden", "Only Clerical can see the VISUAL feed.")
|
||
|
|
|
||
|
|
|
||
|
|
def _it_reason(p: Principal, it_reason: str | None) -> str | None:
|
||
|
|
"""None for Clerical. An admin outside Clerical needs a password session and an IT reason."""
|
||
|
|
if CLERICAL & set(p.roles):
|
||
|
|
return None
|
||
|
|
require_password_session(p)
|
||
|
|
why = reason_ok(it_reason)
|
||
|
|
if why is None:
|
||
|
|
raise ApiError(400, "it_reason_required", "Give an IT reason of at least 8 characters.")
|
||
|
|
return why
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------- shapes
|
||
|
|
|
||
|
|
|
||
|
|
def _run_out(r: dict[str, Any]) -> dict[str, Any]:
|
||
|
|
return {
|
||
|
|
"id": r["id"],
|
||
|
|
"job": r["job"],
|
||
|
|
"trigger": r["trigger"],
|
||
|
|
"requested_by": [str(u) for u in r.get("requested_by") or []],
|
||
|
|
"started_at": r["started_at"],
|
||
|
|
"finished_at": r["finished_at"],
|
||
|
|
"status": r["status"],
|
||
|
|
"counts": r["counts"] or {},
|
||
|
|
"errors": r["errors"] or [],
|
||
|
|
"build_version": r.get("build_version"),
|
||
|
|
"visual_target": r.get("visual_target"),
|
||
|
|
"host": r.get("host"),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _runs(conn: Any, job: str, limit: int) -> list[dict[str, Any]]:
|
||
|
|
return [
|
||
|
|
_run_out(r)
|
||
|
|
for r in packets.rows(conn, "SELECT * FROM job_runs WHERE job = %s ORDER BY id DESC LIMIT %s", (job, limit))
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def _ignored_out(r: dict[str, Any]) -> dict[str, Any]:
|
||
|
|
keys = ("id", "wo_base", "lot", "split", "sub", "rel", "project", "unit_number", "res", "dwg", "last_seen_at")
|
||
|
|
out = {k: r[k] for k in keys}
|
||
|
|
out["pulled_in_at"] = r["pulled_in_at"]
|
||
|
|
out["pulled_in_by"] = str(r["pulled_in_by"]) if r["pulled_in_by"] else None
|
||
|
|
out["pulled_in_reason"] = r["pulled_in_reason"]
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def _stale_seconds() -> int:
|
||
|
|
try:
|
||
|
|
from ..worker import STALE_SECONDS
|
||
|
|
except ImportError: # the worker package is agent A's; the panel still answers without it
|
||
|
|
return STALE_DEFAULT
|
||
|
|
return int(STALE_SECONDS)
|
||
|
|
|
||
|
|
|
||
|
|
def _feed_settings(conn: Any) -> Any:
|
||
|
|
from ..feed import config
|
||
|
|
|
||
|
|
return config.load(conn)
|
||
|
|
|
||
|
|
|
||
|
|
def _request_run(conn: Any, p: Principal, job: str, note: str | None) -> int:
|
||
|
|
"""Insert a job request and NOTIFY the worker in the same transaction. Returns the pending request count."""
|
||
|
|
conn.execute(
|
||
|
|
"INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)",
|
||
|
|
(job, p.user_id, clean_opt(note, LIMITS["job_note"])),
|
||
|
|
)
|
||
|
|
conn.execute("SELECT ps_notify('ps_job_request', %s)", (json.dumps({"job": job}),))
|
||
|
|
return int(
|
||
|
|
conn.execute("SELECT count(*) FROM job_requests WHERE job = %s AND claimed_at IS NULL", (job,)).fetchone()[0]
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------- endpoints
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/overview")
|
||
|
|
def overview(
|
||
|
|
p: Principal = Depends(require_clerical), # noqa: B008
|
||
|
|
d: Db = Depends(get_db), # noqa: B008
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
"""Everything the From VISUAL panel shows, from PostgreSQL only."""
|
||
|
|
conn = d.conn
|
||
|
|
fs = _feed_settings(conn)
|
||
|
|
sched = packets.one(conn, "SELECT next_run_at, paused FROM job_schedule WHERE job = 'visual-feed'")
|
||
|
|
beat = packets.one(
|
||
|
|
conn,
|
||
|
|
"SELECT at, build, running, now() - at > make_interval(secs => %s) AS stale FROM worker_heartbeat WHERE id = 1",
|
||
|
|
(_stale_seconds(),),
|
||
|
|
)
|
||
|
|
runs = _runs(conn, "visual-feed", 10)
|
||
|
|
last_run = runs[0] if runs else None
|
||
|
|
finished = next((r for r in runs if r["status"] in ("ok", "degraded")), None)
|
||
|
|
unmapped_raw = (finished or {}).get("counts", {}).get("unmapped") or {}
|
||
|
|
unmapped: list[dict[str, Any]] = []
|
||
|
|
if isinstance(unmapped_raw, dict):
|
||
|
|
for k, v in unmapped_raw.items():
|
||
|
|
if isinstance(v, int) or (isinstance(v, str) and v.isdigit()):
|
||
|
|
unmapped.append({"resource": str(k), "rows": int(v)})
|
||
|
|
unmapped.sort(key=lambda x: (-int(x["rows"]), str(x["resource"])))
|
||
|
|
unmapped = unmapped[:UNMAPPED_MAX]
|
||
|
|
ignored = packets.rows(
|
||
|
|
conn,
|
||
|
|
"SELECT * FROM visual_ignored_wos WHERE pulled_in_at IS NULL ORDER BY rel DESC NULLS LAST, wo_base, lot, id"
|
||
|
|
" LIMIT %s",
|
||
|
|
(IGNORED_MAX,),
|
||
|
|
)
|
||
|
|
nameless = packets.rows(
|
||
|
|
conn,
|
||
|
|
"SELECT p.id, p.number,"
|
||
|
|
" (SELECT count(*) FROM units u WHERE u.project_id = p.id AND u.removed_at IS NULL) AS units,"
|
||
|
|
" (SELECT count(*) FROM packets k JOIN units u ON u.id = k.unit_id"
|
||
|
|
" WHERE u.project_id = p.id AND k.removed_at IS NULL) AS packets"
|
||
|
|
" FROM projects p WHERE p.source = 'visual' AND p.needs_details AND p.name IS NULL AND p.removed_at IS NULL"
|
||
|
|
" ORDER BY p.number LIMIT 200",
|
||
|
|
)
|
||
|
|
return {
|
||
|
|
"status": {
|
||
|
|
"paused": bool(fs.paused),
|
||
|
|
"next_run_at": (sched or {}).get("next_run_at"),
|
||
|
|
"worker": None
|
||
|
|
if beat is None
|
||
|
|
else {
|
||
|
|
"at": beat["at"],
|
||
|
|
"build": beat["build"],
|
||
|
|
"running": list(beat["running"] or []),
|
||
|
|
"stale": bool(beat["stale"]),
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"last_run": last_run,
|
||
|
|
"runs": runs,
|
||
|
|
"drawing_runs": _runs(conn, "drawing-lookup", 10),
|
||
|
|
"pending_requests": int(
|
||
|
|
conn.execute(
|
||
|
|
"SELECT count(*) FROM job_requests WHERE job = 'visual-feed' AND claimed_at IS NULL"
|
||
|
|
).fetchone()[0]
|
||
|
|
),
|
||
|
|
"ignored": [_ignored_out(r) for r in ignored],
|
||
|
|
"unmapped": unmapped,
|
||
|
|
"projects_without_names": [
|
||
|
|
{"id": r["id"], "number": r["number"], "units": int(r["units"]), "packets": int(r["packets"])}
|
||
|
|
for r in nameless
|
||
|
|
],
|
||
|
|
"golive_date": fs.golive_date,
|
||
|
|
"departments": list(fs.departments),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
@router.post("/run", dependencies=WRITE)
|
||
|
|
def run_now(
|
||
|
|
body: RunBody | None = None,
|
||
|
|
p: Principal = Depends(require_clerical), # noqa: B008
|
||
|
|
d: Db = Depends(get_db), # noqa: B008
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
"""Run now: ask the worker for a feed (or drawing) run. Requests made while one is pending share that run."""
|
||
|
|
body = body or RunBody()
|
||
|
|
conn = d.conn
|
||
|
|
with conn.transaction():
|
||
|
|
pending = _request_run(conn, p, body.job, body.note)
|
||
|
|
audit.append(
|
||
|
|
conn,
|
||
|
|
action="feed.run_requested",
|
||
|
|
user_id=p.user_id,
|
||
|
|
session_id=p.session_id,
|
||
|
|
role_snapshot=p.role_snapshot,
|
||
|
|
subject_type="job",
|
||
|
|
subject_id=body.job,
|
||
|
|
detail={"job": body.job, "pending": pending},
|
||
|
|
ip=p.ip,
|
||
|
|
)
|
||
|
|
return {"requested": True, "job": body.job, "pending": pending}
|
||
|
|
|
||
|
|
|
||
|
|
@router.post("/ignored/{row_id}/pull-in", dependencies=WRITE)
|
||
|
|
def pull_in(
|
||
|
|
row_id: int,
|
||
|
|
body: PullInBody,
|
||
|
|
p: Principal = Depends(require_clerical), # noqa: B008
|
||
|
|
d: Db = Depends(get_db), # noqa: B008
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
"""Track a work order released before the go-live date after all. The next feed run makes its packet."""
|
||
|
|
it_reason = _it_reason(p, body.it_reason)
|
||
|
|
reason = clean_opt(body.reason, PULL_IN_REASON_MAX)
|
||
|
|
if reason is None or len(reason) < PULL_IN_REASON_MIN:
|
||
|
|
raise ApiError(400, "reason_required", "Say why this work order is pulled in (at least 3 characters).")
|
||
|
|
conn = d.conn
|
||
|
|
with conn.transaction():
|
||
|
|
row = packets.one(conn, "SELECT * FROM visual_ignored_wos WHERE id = %s FOR UPDATE", (row_id,))
|
||
|
|
if row is None:
|
||
|
|
raise ApiError(404, "not_found", "No such work order on the ignored list.")
|
||
|
|
if row["pulled_in_at"] is not None:
|
||
|
|
raise ApiError(
|
||
|
|
409,
|
||
|
|
"already_pulled_in",
|
||
|
|
"This work order was already pulled in.",
|
||
|
|
extra={"current": jsonable(_ignored_out(row))},
|
||
|
|
)
|
||
|
|
row = packets.one(
|
||
|
|
conn,
|
||
|
|
"UPDATE visual_ignored_wos SET pulled_in_at = now(), pulled_in_by = %s, pulled_in_reason = %s"
|
||
|
|
" WHERE id = %s RETURNING *",
|
||
|
|
(p.user_id, reason, row_id),
|
||
|
|
)
|
||
|
|
assert row is not None
|
||
|
|
key = "/".join(str(row[k]) for k in ("wo_base", "lot", "split", "sub"))
|
||
|
|
audit.append(
|
||
|
|
conn,
|
||
|
|
action="feed.pulled_in",
|
||
|
|
user_id=p.user_id,
|
||
|
|
session_id=p.session_id,
|
||
|
|
role_snapshot=p.role_snapshot,
|
||
|
|
subject_type="work_order",
|
||
|
|
subject_id=key,
|
||
|
|
detail={"id": row_id, "key": key, "rel": str(row["rel"]) if row["rel"] else None, "reason": reason},
|
||
|
|
it_reason=it_reason,
|
||
|
|
ip=p.ip,
|
||
|
|
)
|
||
|
|
_request_run(conn, p, "visual-feed", f"pull in {key}")
|
||
|
|
return _ignored_out(row)
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/batches")
|
||
|
|
def batches(
|
||
|
|
limit: int = Query(20, ge=1, le=200),
|
||
|
|
p: Principal = Depends(require_clerical), # noqa: B008
|
||
|
|
d: Db = Depends(get_db), # noqa: B008
|
||
|
|
) -> list[dict[str, Any]]:
|
||
|
|
return packets.rows(d.conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT %s", (limit,))
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/changes")
|
||
|
|
def changes(
|
||
|
|
batch_id: int | None = Query(None),
|
||
|
|
limit: int = Query(200, ge=1, le=2000),
|
||
|
|
p: Principal = Depends(require_clerical), # noqa: B008
|
||
|
|
d: Db = Depends(get_db), # noqa: B008
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
"""What VISUAL changed in one batch (the latest when none is given): work order rows and material lines."""
|
||
|
|
conn = d.conn
|
||
|
|
if batch_id is None:
|
||
|
|
batch = packets.one(conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT 1")
|
||
|
|
else:
|
||
|
|
batch = packets.one(conn, "SELECT * FROM visual_batches WHERE id = %s", (batch_id,))
|
||
|
|
if batch is None:
|
||
|
|
raise ApiError(404, "not_found", "No such feed batch.")
|
||
|
|
if batch is None:
|
||
|
|
return {"batch": None, "rows": [], "materials": []}
|
||
|
|
rows = packets.rows(
|
||
|
|
conn, "SELECT * FROM visual_row_changes WHERE batch_id = %s ORDER BY id LIMIT %s", (batch["id"], limit)
|
||
|
|
)
|
||
|
|
mats = packets.rows(
|
||
|
|
conn, "SELECT * FROM visual_material_changes WHERE batch_id = %s ORDER BY id LIMIT %s", (batch["id"], limit)
|
||
|
|
)
|
||
|
|
return {"batch": batch, "rows": rows, "materials": mats}
|