spencerm/PacketStation
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 3: VISUAL feed and M-Files drawings in the worker
ci / frontend (push) Successful in 30s
Details
ci / server (push) Successful in 4m12s
Details
ci / image (push) Successful in 46s
Details
ci / e2e (push) Successful in 1m24s
Details

Browse Source 
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
spencerm
2026-10-16 14:45:38 -06:00
co-authored by Claude Fable 5.1
parent 16950055e3
commit 591fe90076
103 changed files with 16063 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/packetstation/api/packet_visual.py
+580
View File
@@ -0,0 +1,580 @@
"""VISUAL packets by hand: approve the M-Files rev, upload the drawing or the BOM on a flagged draft, and go back to
VISUAL's BOM.
Every write goes through domain.pipeline.packet_write with its policy action and a signature. The hand uploads are
for a VISUAL draft that is flagged (or already has a hand upload): Clerical checks it and sends it on; auto-release
never takes it because its flags are not empty. Every answer is the full read model.
"""
from __future__ import annotations
import hashlib
import uuid
from collections.abc import Mapping
from datetime import UTC, datetime
from typing import Any
from fastapi import APIRouter, Depends, Request
from psycopg.types.json import Jsonb
from pydantic import BaseModel, Field
from starlette.concurrency import run_in_threadpool
from starlette.datastructures import UploadFile
from .. import policy
from ..domain import assets as asset_store
from ..domain import handbom, packets
from ..domain.pipeline import SignBody, Write, next_counter, packet_write
from ..text import LIMITS, clean_opt, reason_ok
from .deps import Db, Principal, current_principal, get_db
from .errors import ApiError
from .packets import WRITE
router = APIRouter(prefix="/api/packets", tags=["packet-visual"])
A = policy.Action
# A hand BOM file is a small sheet. The multipart overhead on top is allowed before the body is refused unread.
BOM_MAX_BYTES = 5 * 1024 * 1024
FORM_SLACK = 64 * 1024
MSG_NOT_FLAGGED = (
"A drawing or BOM is uploaded by hand only on a flagged VISUAL draft. This one is not flagged: send it on as it is."
)
MSG_NOT_VISUAL = "This is not a packet made from VISUAL."
MSG_DRAFT_ONLY = "Only a draft takes a hand upload. Once it leaves Clerical, changes come in as change orders."
# ---------------------------------------------------------------- bodies
class RevApproveBody(BaseModel):
reason: str | None = Field(default=None, max_length=2000)
sign: SignBody | None = None
it_reason: str | None = Field(default=None, max_length=1000)
class HandDrawingBody(BaseModel):
asset_id: uuid.UUID
reason: str | None = Field(default=None, max_length=2000)
sign: SignBody | None = None
it_reason: str | None = Field(default=None, max_length=1000)
class HandBomOffBody(BaseModel):
sign: SignBody | None = None
it_reason: str | None = Field(default=None, max_length=1000)
# ---------------------------------------------------------------- shared helpers
def _pp(d: Db, p: Principal) -> policy.Principal:
return packets.policy_principal_for(d.conn, p)
def _read(d: Db, pp: policy.Principal, packet_id: int) -> dict[str, Any]:
return packets.read_packet(d.conn, packet_id, pp)
def now_iso() -> str:
return datetime.now(UTC).isoformat()
def who_of(w: Write) -> dict[str, Any]:
return {"by": str(w.user_id), "by_name": w.principal.user.get("display_name")}
def hand_flagged(row: Mapping[str, Any]) -> bool:
"""The hand-upload guard: a VISUAL draft that is flagged, still needs its drawing, or already has a hand upload."""
if row["source_kind"] != "visual" or row["status"] != "draft":
return False
visual = row["visual"] or {}
return bool(
list(row["flags"] or [])
or row["drawing_state"] in packets.DRAWING_STATES_FLAGGED
or visual.get("manual_dwg")
or visual.get("manual_bom")
)
def _hand_guard(w: Write) -> None:
row = w.packet
if row["source_kind"] != "visual":
raise ApiError(409, "not_visual", MSG_NOT_VISUAL)
if row["status"] != "draft":
raise ApiError(403, "wrong_status", MSG_DRAFT_ONLY)
if not hand_flagged(row):
raise ApiError(409, "not_flagged", MSG_NOT_FLAGGED)
def pdf_asset(conn: Any, asset_id: Any) -> dict[str, Any]:
asset = asset_store.get(conn, asset_id)
if asset is None:
raise ApiError(404, "not_found", "No such file.")
if asset["mime"] != asset_store.PDF or not asset["pages"]:
raise ApiError(422, "not_pdf", "Attach a PDF drawing.")
return asset
def flags_without(flags: Any, drop: tuple[str, ...], add: tuple[str, ...] = ()) -> list[str]:
out = [f for f in (flags or []) if f not in drop]
for f in add:
if f not in out:
out.append(f)
return out
def replace_drawing_pages(
conn: Any,
packet_id: int,
asset: Mapping[str, Any],
*,
dwg: str | None,
rev: str | None,
co_id: int | None = None,
void_by: Any = None,
) -> tuple[int, int, list[dict[str, Any]]]:
"""Put an asset's pages on the packet as its drawing.
Without `co_id` (a draft) the current drawing pages are deleted (the assets stay). With `co_id` (a change order
approval) they are voided and point at the first new page. Returns (first new page, page count, old pages).
"""
old = packets.rows(
conn,
"SELECT * FROM packet_pages WHERE packet_id = %s AND type = 'drawing' AND void_at IS NULL ORDER BY n",
(packet_id,),
)
count = int(asset["pages"] or 0)
if count < 1:
raise ApiError(422, "not_pdf", "That drawing has no pages.")
first = next_counter(conn, packet_id, "page", count)
if co_id is None:
conn.execute(
"DELETE FROM packet_pages WHERE packet_id = %s AND type = 'drawing' AND void_at IS NULL", (packet_id,)
)
elif old:
conn.execute(
"UPDATE packet_pages SET void_co_id = %s, void_at = now(), void_by = %s, void_new_n = %s,"
" row_version = row_version + 1 WHERE id = ANY(%s)",
(co_id, void_by, first, [pg["id"] for pg in old]),
)
for i in range(count):
conn.execute(
"INSERT INTO packet_pages (packet_id, n, type, asset_id, pn, dwg, rev, co_id)"
" VALUES (%s, %s, 'drawing', %s, %s, %s, %s, %s)",
(packet_id, first + i, asset["id"], i + 1, dwg, rev, co_id),
)
return first, count, old
def set_visual(conn: Any, packet_id: int, visual: Mapping[str, Any]) -> None:
conn.execute("UPDATE packets SET visual = %s WHERE id = %s", (Jsonb(dict(visual)), packet_id))
def _sha(asset: Mapping[str, Any]) -> str:
return bytes(asset["sha256"]).hex()
# ---------------------------------------------------------------- approve the M-Files rev
@router.post("/{packet_id}/rev-approve", dependencies=WRITE)
def rev_approve(
packet_id: int,
body: RevApproveBody,
request: Request,
p: Principal = Depends(current_principal), # noqa: B008
d: Db = Depends(get_db), # noqa: B008
) -> dict[str, Any]:
"""A Clerical Supervisor approves the rev M-Files has in place of the rev VISUAL calls, for this draft only."""
conn = d.conn
reason = reason_ok(body.reason)
pp = _pp(d, p)
row = packets.load_row(conn, packet_id)
staged = packets.staged_lookup(conn, row["drawing"]) if row else None
content = {
"action": str(A.REV_APPROVE),
"packet_id": packet_id,
"drawing": row["drawing"] if row else None,
"visual_rev": row["rev"] if row else None,
"rev": staged["rev"] if staged else None,
"sha256": bytes(staged["sha256"]).hex() if staged and staged["sha256"] else None,
"reason": reason,
}
with packet_write(
d,
p,
packet_id,
A.REV_APPROVE,
request=request,
extra={"reason": body.reason},
it_reason=body.it_reason,
sign=body.sign,
sign_content=content,
policy_principal=pp,
) as w:
if w.packet["source_kind"] != "visual":
raise ApiError(409, "not_visual", MSG_NOT_VISUAL)
lk = packets.staged_lookup(conn, w.packet["drawing"])
if lk is None:
raise ApiError(
409, "nothing_staged", "No M-Files copy of this drawing is ready yet. Try after the next run."
)
asset = asset_store.get(conn, lk["asset_id"])
if asset is None or not asset["pages"]:
raise ApiError(
409, "nothing_staged", "The M-Files copy of this drawing is missing. Try after the next run."
)
rev = str(lk["rev"])
vis_rev = w.packet["rev"] or ""
replace_drawing_pages(conn, packet_id, asset, dwg=w.packet["drawing"], rev=rev)
visual = dict(w.packet["visual"] or {})
visual["rev_ok"] = {"rev": rev, "vis_rev": vis_rev, **who_of(w), "at": now_iso(), "reason": reason}
visual["rev_confirmed"] = True
visual["vault"] = {
"asset_id": str(asset["id"]),
"name": lk["name"],
"rev": rev,
"note": lk["note"],
"at": now_iso(),
}
visual.pop("manual_dwg", None)
set_visual(conn, packet_id, visual)
conn.execute(
"UPDATE packets SET drawing_state = 'attached', flags = %s WHERE id = %s",
(flags_without(w.packet["flags"], (*packets.DRAWING_FLAGS, "hand_drawing")), packet_id),
)
w.changed("pages", "drawing_state", "flags", "visual")
w.detail.update(asset_id=str(asset["id"]), rev=rev, vis_rev=vis_rev)
w.log("Different rev approved by hand", f"rev {rev} in place of VISUAL rev {vis_rev or '-'} · {reason}")
return _read(d, pp, packet_id)
# ---------------------------------------------------------------- hand drawing
@router.post("/{packet_id}/hand/drawing", dependencies=WRITE)
def hand_drawing(
packet_id: int,
body: HandDrawingBody,
request: Request,
p: Principal = Depends(current_principal), # noqa: B008
d: Db = Depends(get_db), # noqa: B008
) -> dict[str, Any]:
"""Clerical uploads the drawing by hand on a flagged VISUAL draft. It replaces the drawing pages."""
conn = d.conn
reason = clean_opt(body.reason, LIMITS["reason"])
if reason is None:
raise ApiError(400, "reason_required", "Say why the drawing is uploaded by hand.")
asset = pdf_asset(conn, body.asset_id)
pp = _pp(d, p)
content = {
"action": "hand_drawing",
"packet_id": packet_id,
"asset_id": str(asset["id"]),
"sha256": _sha(asset),
"pages": asset["pages"],
"reason": reason,
}
with packet_write(
d,
p,
packet_id,
A.EDIT_DOCUMENT_SET,
request=request,
it_reason=body.it_reason,
sign=body.sign,
sign_content=content,
require_sign=True,
policy_principal=pp,
) as w:
_hand_guard(w)
vis_rev = w.packet["rev"] or ""
first, count, _ = replace_drawing_pages(conn, packet_id, asset, dwg=w.packet["drawing"], rev=w.packet["rev"])
name = asset["original_name"] or "drawing.pdf"
visual = dict(w.packet["visual"] or {})
visual["manual_dwg"] = {
"asset_id": str(asset["id"]),
"name": name,
"reason": reason,
**who_of(w),
"at": now_iso(),
"vis_rev": vis_rev,
}
visual["rev_confirmed"] = False
set_visual(conn, packet_id, visual)
conn.execute(
"UPDATE packets SET drawing_state = 'hand', flags = %s WHERE id = %s",
(flags_without(w.packet["flags"], packets.DRAWING_FLAGS, ("hand_drawing",)), packet_id),
)
w.changed("pages", "drawing_state", "flags", "visual")
w.detail.update(asset_id=str(asset["id"]), pages=count, first_page=first)
w.log("Drawing uploaded by hand", f"{name}, {count} page{'s' if count != 1 else ''} · {reason}")
return _read(d, pp, packet_id)
# ---------------------------------------------------------------- hand BOM
def _insert_hand_bom(conn: Any, packet_id: int, lines: list[dict[str, Any]]) -> None:
conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,))
for i, b in enumerate(lines):
conn.execute(
"INSERT INTO bom_lines (packet_id, item, qty, part_id, description, dwg_ref, cl_item, length, material,"
" src, sort) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'hand', %s)",
(
packet_id,
b["item"],
b["qty"],
b["part_id"],
b["description"],
b["dwg_ref"],
b["cl_item"],
b["length"],
b["material"],
i,
),
)
def _hand_bom_write(
d: Db,
p: Principal,
request: Request,
packet_id: int,
*,
data: bytes,
name: str | None,
reason: str,
sign: SignBody,
it_reason: str | None,
) -> dict[str, Any]:
conn = d.conn
try:
lines = handbom.parse(data, name)
except handbom.BadFile as e:
raise ApiError(422, "bad_file", str(e)) from None
pp = _pp(d, p)
sha = hashlib.sha256(data).hexdigest()
fname = clean_opt(name, LIMITS["file_name"]) or "bom"
content = {
"action": "hand_bom",
"packet_id": packet_id,
"file": fname,
"sha256": sha,
"lines": len(lines),
"reason": reason,
}
with packet_write(
d,
p,
packet_id,
A.EDIT_BOM,
request=request,
it_reason=it_reason,
sign=sign,
sign_content=content,
require_sign=True,
policy_principal=pp,
) as w:
_hand_guard(w)
_insert_hand_bom(conn, packet_id, lines)
visual = dict(w.packet["visual"] or {})
visual["manual_bom"] = {
"name": fname,
"reason": reason,
**who_of(w),
"at": now_iso(),
"lines": len(lines),
"sha256": sha,
}
set_visual(conn, packet_id, visual)
conn.execute(
"UPDATE packets SET flags = %s WHERE id = %s",
(flags_without(w.packet["flags"], (), ("hand_bom",)), packet_id),
)
w.changed("bom", "flags", "visual")
w.detail.update(file=fname, sha256=sha, lines=len(lines))
n = len(lines)
w.log("BOM uploaded by hand", f"{fname} · {n} line{'s' if n != 1 else ''} · {reason}")
return _read(d, pp, packet_id)
@router.post("/{packet_id}/hand/bom")
async def hand_bom(
packet_id: int,
request: Request,
p: Principal = Depends(current_principal), # noqa: B008
d: Db = Depends(get_db), # noqa: B008
) -> dict[str, Any]:
"""Clerical uploads the BOM by hand (multipart: file, reason, sign_factor, sign_secret, it_reason)."""
length = request.headers.get("content-length")
if length is not None and length.isdigit() and int(length) > BOM_MAX_BYTES + FORM_SLACK:
raise ApiError(413, "too_large", f"The BOM file is over the {BOM_MAX_BYTES // (1024 * 1024)} MB limit.")
form = await request.form(max_files=1, max_fields=8)
try:
file = form.get("file")
if not isinstance(file, UploadFile):
raise ApiError(422, "invalid_request", "Send the BOM file in the 'file' field.")
def field(name: str, cap: int) -> str | None:
v = form.get(name)
return v[:cap] if isinstance(v, str) else None
reason = clean_opt(field("reason", 2000), LIMITS["reason"])
if reason is None:
raise ApiError(400, "reason_required", "Say why the BOM is uploaded by hand.")
factor, secret = field("sign_factor", 20), field("sign_secret", 1024)
if not factor or secret is None:
raise ApiError(
400, "signature_required", "This step needs your signature.", extra={"meaning": "Edit the BOM"}
)
if factor not in ("pin", "password"):
raise ApiError(422, "invalid_request", "sign_factor is pin or password.")
sign = SignBody(factor=factor, secret=secret) # type: ignore[arg-type]
data = await file.read(BOM_MAX_BYTES + 1)
if len(data) > BOM_MAX_BYTES:
raise ApiError(413, "too_large", f"The BOM file is over the {BOM_MAX_BYTES // (1024 * 1024)} MB limit.")
name = file.filename
it_reason = field("it_reason", 1000)
finally:
await form.close()
return await run_in_threadpool(
lambda: _hand_bom_write(
d, p, request, packet_id, data=data, name=name, reason=reason, sign=sign, it_reason=it_reason
)
)
@router.delete("/{packet_id}/hand/bom", dependencies=WRITE)
def hand_bom_off(
packet_id: int,
request: Request,
body: HandBomOffBody | None = None,
p: Principal = Depends(current_principal), # noqa: B008
d: Db = Depends(get_db), # noqa: B008
) -> dict[str, Any]:
"""Drop the hand BOM and use VISUAL's BOM again, rebuilt from the cached materials."""
body = body or HandBomOffBody()
conn = d.conn
pp = _pp(d, p)
with packet_write(
d,
p,
packet_id,
A.EDIT_BOM,
request=request,
it_reason=body.it_reason,
sign=body.sign,
sign_content={"action": "hand_bom_off", "packet_id": packet_id},
require_sign=body.sign is not None,
policy_principal=pp,
) as w:
if w.packet["source_kind"] != "visual":
raise ApiError(409, "not_visual", MSG_NOT_VISUAL)
if w.packet["status"] != "draft":
raise ApiError(403, "wrong_status", MSG_DRAFT_ONLY)
visual = dict(w.packet["visual"] or {})
old = visual.pop("manual_bom", None)
if not old:
raise ApiError(409, "no_hand_bom", "This packet has no BOM uploaded by hand.")
from ..feed import materials as feed_materials # agent B's module, imported where it is used
lines = feed_materials.bom_for_packet(conn, packet_id)
conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,))
feed_materials.insert_bom(conn, packet_id, lines)
set_visual(conn, packet_id, visual)
conn.execute(
"UPDATE packets SET flags = %s WHERE id = %s",
(flags_without(w.packet["flags"], ("hand_bom",)), packet_id),
)
w.changed("bom", "flags", "visual")
n = len(lines)
name = old.get("name") if isinstance(old, dict) else None
note = f"{n} line{'s' if n != 1 else ''} from VISUAL" if n else "VISUAL has no materials for it yet"
w.log("Hand-uploaded BOM taken off", " · ".join(x for x in (name, note) if x))
return _read(d, pp, packet_id)
# ---------------------------------------------------------------- VISUAL BOM lines (change order approval)
def merge_visual_bom(conn: Any, packet_id: int, raw_lines: list[Any]) -> dict[str, int]:
"""Bring a packet's BOM to VISUAL's lines (change order `vis.bom`, dicts or BomLine objects) without losing work.
When nothing has been recorded on the lines (no pulled mark, heat, MTR or short) the BOM is replaced through
feed.materials.insert_bom, exactly as the feed writes it. Otherwise lines are matched on vkey and updated in
place, so pulled marks, heats and shorts stay. New lines are added with the same encoding (item numbers from
feed.materials.db_items, the label in cl_item, cons Y/N, sub Y or NULL). Lines VISUAL dropped are deleted, or
marked n/a when they were pulled or have a short.
"""
from ..feed.materials import BomLine, db_items, insert_bom
lines = [ln if isinstance(ln, BomLine) else BomLine.from_dict(ln) for ln in raw_lines]
counts = {"added": 0, "updated": 0, "removed": 0, "na": 0}
existing = packets.rows(conn, "SELECT * FROM bom_lines WHERE packet_id = %s ORDER BY item", (packet_id,))
worked = (
any(e["pulled"] or e["heat"] or e["mtr_asset_id"] for e in existing)
or conn.execute("SELECT 1 FROM bom_shorts WHERE packet_id = %s LIMIT 1", (packet_id,)).fetchone()
)
if not worked:
# Nothing recorded on the lines yet: write them exactly as the feed does.
conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,))
insert_bom(conn, packet_id, lines)
counts["added"] = len(lines)
counts["removed"] = len(existing)
return counts
by_vkey = {e["vkey"]: e for e in existing if e["vkey"]}
used = {int(e["item"]) for e in existing}
wanted = db_items(lines)
kept: set[int] = set()
for sort, (b, want_item) in enumerate(zip(lines, wanted, strict=True)):
values = (
b.qty,
b.part_id[:60] or None,
b.desc[:240] or None,
b.loc[:120] or None,
b.whs or None,
b.comm or None,
"Y" if b.cons else "N",
"Y" if b.sub else None,
b.na,
b.grp,
b.issued,
sort,
)
match = by_vkey.get(b.vkey) if b.vkey else None
if match is not None and int(match["id"]) not in kept:
kept.add(int(match["id"]))
conn.execute(
"UPDATE bom_lines SET qty = %s, part_id = %s, description = %s, loc = %s, whs = %s, comm = %s,"
" cons = %s, sub = %s, na = %s, grp = %s, issued = coalesce(%s, issued), sort = %s, cl_item = %s,"
" row_version = row_version + 1 WHERE id = %s",
(*values, b.item, match["id"]),
)
counts["updated"] += 1
continue
item = want_item
while item in used:
item += 1
used.add(item)
new_id = conn.execute(
"INSERT INTO bom_lines (packet_id, item, cl_item, qty, part_id, description, loc, whs, comm, cons, sub, na,"
" grp, issued, sort, vkey, src) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,"
" 'visual') RETURNING id",
(packet_id, item, b.item, *values, b.vkey),
).fetchone()[0]
kept.add(int(new_id))
counts["added"] += 1
for e in existing:
if int(e["id"]) in kept:
continue
busy = e["pulled"] or conn.execute("SELECT 1 FROM bom_shorts WHERE bom_line_id = %s", (e["id"],)).fetchone()
if busy:
if not e["na"]:
conn.execute("UPDATE bom_lines SET na = true, row_version = row_version + 1 WHERE id = %s", (e["id"],))
counts["na"] += 1
else:
conn.execute("DELETE FROM bom_lines WHERE id = %s", (e["id"],))
counts["removed"] += 1
return counts