spencermandClaude Fable 5.1
591fe90076
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
492 lines
20 KiB
Python
492 lines
20 KiB
Python
"""Where the feed reads VISUAL from: VisualKit (DEV unless the stack says otherwise) or a synthetic fixture file.
|
|
|
|
`VisualKitSource` copies the read calls and output shapes of VisualKit's examples/packet_station_feed.py: the
|
|
window read, the tracked keys by key, operations, operation text and parts for rows; for_bases, requirements,
|
|
requirement text, parts and part sites for materials; DEMAND_SUPPLY_LINK orders by base and by project; labor
|
|
tickets summed per operation. Read-only: VisualKit enforces it. No cost, price, amount or pay column is read.
|
|
|
|
`FixtureSource` reads `<dir>/state` and `<dir>/feed.<state>.json` in the same contract shape (CI and tests).
|
|
|
|
Every value goes through `types.parse_row` / `parse_material` / `parse_order`, so both sources give identical
|
|
FeedData.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from collections.abc import Iterable, Sequence
|
|
from datetime import date, datetime
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
from typing import Any, Protocol
|
|
|
|
from .types import (
|
|
FeedData,
|
|
LaborLeg,
|
|
Material,
|
|
Order,
|
|
Row,
|
|
WoKey,
|
|
parse_labor,
|
|
parse_material,
|
|
parse_order,
|
|
parse_orders,
|
|
parse_row,
|
|
status_code,
|
|
text,
|
|
)
|
|
|
|
STATUS_WORDS = {"F": "Firmed", "R": "Released", "U": "Unreleased", "C": "Closed", "X": "Cancelled"}
|
|
MATERIAL_STATUSES = ("F", "R")
|
|
OPEN_ORDER_STATUSES = ("F", "H", "R")
|
|
QTY_EMPLOYEE = "9999"
|
|
|
|
_SECRETISH = re.compile(r"(?i)(password|pwd|secret|token)\s*[=:]\s*\S+")
|
|
|
|
|
|
def redact(message: object) -> str:
|
|
"""An exception as one short line with anything password-like removed."""
|
|
s = _SECRETISH.sub(
|
|
r"\1=***", f"{type(message).__name__}: {message}" if isinstance(message, BaseException) else str(message)
|
|
)
|
|
return s.replace("\r", " ").replace("\n", " ")[:500]
|
|
|
|
|
|
class FeedSource(Protocol):
|
|
def read(
|
|
self,
|
|
*,
|
|
site: str,
|
|
since: date,
|
|
until: date,
|
|
tracked: Sequence[WoKey],
|
|
with_materials: bool,
|
|
with_orders: bool,
|
|
with_labor: bool,
|
|
labor_since: date | None,
|
|
) -> FeedData: ...
|
|
|
|
def describe(self) -> str: ...
|
|
|
|
|
|
# ---------------------------------------------------------------- value helpers (as the VisualKit example)
|
|
|
|
|
|
def _to_number(v: Any) -> Any:
|
|
if v is None or isinstance(v, bool):
|
|
return v
|
|
if isinstance(v, Decimal):
|
|
return v
|
|
return Decimal(str(v))
|
|
|
|
|
|
def _fmt_date(v: date | datetime | None) -> str | None:
|
|
if v is None:
|
|
return None
|
|
if isinstance(v, datetime):
|
|
v = v.date()
|
|
return v.isoformat()
|
|
|
|
|
|
def _clean(s: str | None) -> str:
|
|
if not s:
|
|
return ""
|
|
return s.replace("\r\n", "\n").replace("\r", "\n")
|
|
|
|
|
|
def _istat(calc_qty: Any, issued_qty: Any, line_status: str | None) -> int:
|
|
"""Job Pull's issue state: 3 over-issued, 0 line closed, 2 nothing issued, 1 partly."""
|
|
calc = Decimal(str(calc_qty or 0))
|
|
issued = Decimal(str(issued_qty or 0))
|
|
if issued > calc:
|
|
return 3
|
|
if (line_status or "").strip().upper() == "C":
|
|
return 0
|
|
if issued == 0:
|
|
return 2
|
|
return 1
|
|
|
|
|
|
def _yes_no(flag: bool | None) -> str:
|
|
return "" if flag is None else ("Y" if flag else "N")
|
|
|
|
|
|
def project_of(base: str) -> str:
|
|
"""VisualKit's project_of: the base before its first dash, or the whole base."""
|
|
head = base.partition("-")[0]
|
|
return head or base
|
|
|
|
|
|
# ---------------------------------------------------------------- VisualKit
|
|
|
|
|
|
class VisualKitSource:
|
|
"""The five reads through one `visualkit.Visual` (made by the job per run, closed after)."""
|
|
|
|
def __init__(self, visual: Any) -> None:
|
|
self.v = visual
|
|
|
|
def describe(self) -> str:
|
|
s = self.v.settings
|
|
return f"{s.server} / {s.database}"
|
|
|
|
def _key(self, k: WoKey) -> Any:
|
|
from visualkit import WorkOrderKey
|
|
|
|
return WorkOrderKey(k.base, k.lot, k.split, k.sub, "W")
|
|
|
|
def _load_parts(self, part_ids: Iterable[str | None], cache: dict[str, Any]) -> None:
|
|
missing = sorted({p for p in part_ids if p and p not in cache})
|
|
if missing:
|
|
cache.update(self.v.parts.get_many(missing))
|
|
|
|
def read(
|
|
self,
|
|
*,
|
|
site: str,
|
|
since: date,
|
|
until: date,
|
|
tracked: Sequence[WoKey],
|
|
with_materials: bool,
|
|
with_orders: bool,
|
|
with_labor: bool,
|
|
labor_since: date | None,
|
|
) -> FeedData:
|
|
v = self.v
|
|
s = v.settings
|
|
meta: dict[str, Any] = {
|
|
"source": "visualkit",
|
|
"server": s.server,
|
|
"database": s.database,
|
|
"site": site,
|
|
"since": since.isoformat(),
|
|
"until": until.isoformat(),
|
|
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
|
}
|
|
data = FeedData(meta=meta, rows=[], tracked=list(tracked), tracked_count=len(tracked))
|
|
parts: dict[str, Any] = {}
|
|
|
|
# 1. The window: firmed TYPE W work orders released in [since, until], paged.
|
|
wos: list[Any] = []
|
|
offset: int | None = 0
|
|
while offset is not None:
|
|
page = v.work_orders.released(site, since, until, status=("F",), offset=offset)
|
|
wos.extend(page.items)
|
|
offset = page.next_offset
|
|
data.window_count = len(wos)
|
|
|
|
# 2. The tracked keys, whatever their status or date. Keys VISUAL lacks are missing.
|
|
asked = {self._key(k): k for k in dict.fromkeys(tracked)}
|
|
found = v.work_orders.get_many(list(asked)) if asked else {}
|
|
data.tracked_missing = [k for vk, k in asked.items() if vk not in found]
|
|
have = {wo.key for wo in wos}
|
|
for wo in found.values():
|
|
if wo.key not in have:
|
|
wos.append(wo)
|
|
have.add(wo.key)
|
|
|
|
# 3. Rows for the union.
|
|
keys = [wo.key for wo in wos]
|
|
ops = v.work_orders.operations_for(keys) if keys else {}
|
|
specs = v.work_orders.operation_text_for(keys) if keys else {}
|
|
self._load_parts((wo.part_id for wo in wos), parts)
|
|
for wo in wos:
|
|
part = parts.get(wo.part_id) if wo.part_id else None
|
|
desc = _clean(part.description if part else None)
|
|
qty = wo.desired_qty or Decimal(0)
|
|
for op in ops.get(wo.key, []):
|
|
data.rows.append(
|
|
parse_row(
|
|
{
|
|
"wo": wo.base_id,
|
|
"lot": wo.lot_id,
|
|
"split": wo.split_id,
|
|
"sub": wo.sub_id,
|
|
"seq": op.sequence_no,
|
|
"res": op.resource_id or "",
|
|
"dwg": wo.drawing_id or "",
|
|
"rev": wo.drawing_rev_no or "",
|
|
"ps": specs.get((wo.key, op.sequence_no), ""),
|
|
"desc": desc,
|
|
"qty": _to_number(qty),
|
|
"tgt": _to_number((op.run or Decimal(0)) * qty),
|
|
"rel": _fmt_date(wo.desired_rls_date),
|
|
"want": _fmt_date(wo.desired_want_date),
|
|
"st": STATUS_WORDS.get((wo.status or "").strip().upper(), wo.status or ""),
|
|
"barcode": op.barcode(),
|
|
"part": wo.part_id or "",
|
|
}
|
|
)
|
|
)
|
|
|
|
bases = sorted({wo.base_id for wo in wos})
|
|
if with_materials:
|
|
try:
|
|
data.materials, data.material_legs = self._materials(bases, site, parts)
|
|
except Exception as e: # a failed materials read keeps the BOMs as they are
|
|
data.materials, data.material_legs = None, None
|
|
data.errors.append("materials: " + redact(e))
|
|
if with_orders:
|
|
try:
|
|
data.orders_by_base = self._orders_by_base(bases)
|
|
data.orders_by_project = self._orders_by_project({project_of(b) for b in bases})
|
|
except Exception as e: # the apply keeps the last cached orders
|
|
data.orders_by_base, data.orders_by_project = None, None
|
|
data.errors.append("orders: " + redact(e))
|
|
if with_labor:
|
|
try:
|
|
data.labor = self._labor(list(asked) or keys)
|
|
except Exception as e:
|
|
data.labor = None
|
|
data.errors.append("labor: " + redact(e))
|
|
return data
|
|
|
|
def _materials(self, bases: Sequence[str], site: str, parts: dict[str, Any]) -> tuple[list[Material], set[WoKey]]:
|
|
v = self.v
|
|
if not bases:
|
|
return [], set()
|
|
legs = v.work_orders.for_bases(bases, type="W", status=MATERIAL_STATUSES, site_id=site)
|
|
keys = [wo.key for wo in legs]
|
|
reqs = v.work_orders.requirements_for(keys) if keys else {}
|
|
text_keys = [k for k in keys if any(not r.part_id for r in reqs.get(k, []))]
|
|
texts = v.work_orders.requirement_text_for(text_keys) if text_keys else {}
|
|
part_ids = {r.part_id for rs in reqs.values() for r in rs if r.part_id}
|
|
self._load_parts(part_ids, parts)
|
|
sites = v.parts.sites_for(sorted(part_ids), site) if part_ids else {}
|
|
out: list[Material] = []
|
|
covered: set[WoKey] = set()
|
|
for wo in legs:
|
|
covered.add(WoKey.of(wo.base_id, wo.lot_id, wo.split_id, wo.sub_id))
|
|
for r in reqs.get(wo.key, []):
|
|
part = parts.get(r.part_id) if r.part_id else None
|
|
ps = sites.get(r.part_id) if r.part_id else None
|
|
desc = (
|
|
_clean(part.description if part else None)
|
|
if r.part_id
|
|
else _clean(texts.get((wo.key, r.operation_seq_no, r.piece_no)))
|
|
)
|
|
out.append(
|
|
parse_material(
|
|
{
|
|
"wo": wo.base_id,
|
|
"lot": wo.lot_id,
|
|
"split": wo.split_id,
|
|
"sub": wo.sub_id,
|
|
"seq": r.operation_seq_no,
|
|
"piece": r.piece_no,
|
|
"part": r.part_id or None,
|
|
"qty": _to_number(r.calc_qty or Decimal(0)),
|
|
"issued": _to_number(r.issued_qty or Decimal(0)),
|
|
"whs": (ps.primary_whs_id or "").upper() if ps else "",
|
|
"loc": (ps.primary_loc_id or "").upper() if ps else "",
|
|
"st": wo.status,
|
|
"istat": _istat(r.calc_qty, r.issued_qty, r.status),
|
|
"dwg": wo.drawing_id or "",
|
|
"desc": desc,
|
|
"comm": (part.commodity_code or "") if part else "",
|
|
"cons": _yes_no(part.consumable) if part else "",
|
|
}
|
|
)
|
|
)
|
|
return out, covered
|
|
|
|
def _orders_by_base(self, bases: Sequence[str]) -> dict[str, list[Order]]:
|
|
v = self.v
|
|
if not bases:
|
|
return {}
|
|
pairs = v.links.orders_for_bases(list(bases))
|
|
order_ids = sorted({oid for ps in pairs.values() for oid, _ in ps})
|
|
orders = v.orders.get_many(order_ids) if order_ids else {}
|
|
customers = v.orders.customers(sorted({o.customer_id for o in orders.values()})) if orders else {}
|
|
out: dict[str, list[Order]] = {}
|
|
for base in bases:
|
|
lines_by_order: dict[str, list[int]] = {}
|
|
for oid, line_no in pairs.get(base, []):
|
|
lines_by_order.setdefault(oid, [])
|
|
if line_no is not None:
|
|
lines_by_order[oid].append(line_no)
|
|
if not lines_by_order:
|
|
continue
|
|
out[base] = []
|
|
for oid, lines in sorted(lines_by_order.items()):
|
|
o = orders.get(oid)
|
|
cust = customers.get(o.customer_id) if o else None
|
|
out[base].append(
|
|
parse_order(
|
|
{
|
|
"cust_order_id": oid,
|
|
"lines": sorted(set(lines)),
|
|
"status": o.status if o else None,
|
|
"customer_id": o.customer_id if o else None,
|
|
"customer_name": cust.name if cust else None,
|
|
"customer_po_ref": o.customer_po_ref if o else None,
|
|
"order_date": _fmt_date(getattr(o, "order_date", None)) if o else None,
|
|
"desired_ship_date": _fmt_date(o.desired_ship_date) if o else None,
|
|
"promise_date": _fmt_date(o.promise_date) if o else None,
|
|
}
|
|
)
|
|
)
|
|
return out
|
|
|
|
def _orders_by_project(self, projects: Iterable[str]) -> dict[str, list[Order]]:
|
|
v = self.v
|
|
wanted = sorted(set(projects))
|
|
if not wanted:
|
|
return {}
|
|
found = v.links.orders_for_projects(wanted)
|
|
cust_ids = sorted({po.customer_id for pos in found.values() for po in pos if po.customer_id})
|
|
customers = v.orders.customers(cust_ids) if cust_ids else {}
|
|
out: dict[str, list[Order]] = {}
|
|
for project in wanted:
|
|
entries: dict[str, dict[str, Any]] = {}
|
|
for po in found.get(project, []):
|
|
e = entries.get(po.cust_order_id)
|
|
if e is None:
|
|
cust = customers.get(po.customer_id) if po.customer_id else None
|
|
e = entries[po.cust_order_id] = {
|
|
"cust_order_id": po.cust_order_id,
|
|
"lines": [],
|
|
"top_jobs": [],
|
|
"status": po.status,
|
|
"customer_id": po.customer_id,
|
|
"customer_name": cust.name if cust else None,
|
|
"customer_po_ref": po.customer_po_ref,
|
|
"order_date": _fmt_date(po.order_date),
|
|
"desired_ship_date": _fmt_date(po.desired_ship_date),
|
|
"promise_date": _fmt_date(po.promise_date),
|
|
}
|
|
if po.line_no is not None and po.line_no not in e["lines"]:
|
|
e["lines"].append(po.line_no)
|
|
if po.supply_base_id not in e["top_jobs"]:
|
|
e["top_jobs"].append(po.supply_base_id)
|
|
if entries:
|
|
for e in entries.values():
|
|
e["lines"].sort()
|
|
e["top_jobs"].sort()
|
|
ordered = sorted(entries.values(), key=lambda e: e["status"] not in OPEN_ORDER_STATUSES)
|
|
out[project] = [parse_order(e) for e in ordered]
|
|
return out
|
|
|
|
def _labor(self, keys: Sequence[Any]) -> list[LaborLeg]:
|
|
"""Hours and ticket counts per (leg, seq). Employee 9999's quantity-only tickets and indirect tickets
|
|
are left out. Hours only: no cost or pay column is read. VisualKit has no date filter on this call, so
|
|
every ticket of the legs is read and summed (the totals need them all)."""
|
|
if not keys:
|
|
return []
|
|
tickets = self.v.labor.tickets_for_work_orders(list(keys))
|
|
sums: dict[tuple[WoKey, int], LaborLeg] = {}
|
|
for vk, ts in tickets.items():
|
|
k = WoKey.of(vk.base_id, vk.lot_id, vk.split_id, vk.sub_id)
|
|
for t in ts:
|
|
if t.operation_seq_no is None or t.is_quantity_only or t.is_indirect:
|
|
continue
|
|
seq = int(t.operation_seq_no)
|
|
leg = sums.get((k, seq))
|
|
if leg is None:
|
|
leg = sums[(k, seq)] = LaborLeg(k.base, k.lot, k.split, k.sub, seq, Decimal(0), 0, None)
|
|
leg.hours += Decimal(str(t.hours_worked or 0))
|
|
leg.tickets += 1
|
|
d = t.transaction_date.date() if isinstance(t.transaction_date, datetime) else t.transaction_date
|
|
if d is not None and (leg.last_ticket is None or d > leg.last_ticket):
|
|
leg.last_ticket = d
|
|
return list(sums.values())
|
|
|
|
|
|
# ---------------------------------------------------------------- fixtures
|
|
|
|
|
|
class FixtureSource:
|
|
"""Synthetic VISUAL: `<dir>/state` (default "1") picks `<dir>/feed.<state>.json`."""
|
|
|
|
def __init__(self, directory: Path) -> None:
|
|
self.dir = Path(directory)
|
|
|
|
def state(self) -> str:
|
|
try:
|
|
s = (self.dir / "state").read_text(encoding="utf-8").strip()
|
|
except OSError:
|
|
s = ""
|
|
return s or "1"
|
|
|
|
def describe(self) -> str:
|
|
return f"fixture state {self.state()}"
|
|
|
|
def read(
|
|
self,
|
|
*,
|
|
site: str,
|
|
since: date,
|
|
until: date,
|
|
tracked: Sequence[WoKey],
|
|
with_materials: bool,
|
|
with_orders: bool,
|
|
with_labor: bool,
|
|
labor_since: date | None,
|
|
) -> FeedData:
|
|
state = self.state()
|
|
raw = json.loads((self.dir / f"feed.{state}.json").read_text(encoding="utf-8"))
|
|
all_rows: list[Row] = [parse_row(d) for d in raw.get("rows") or []]
|
|
tracked_set = set(tracked)
|
|
window_keys = {r.key for r in all_rows if r.st == "F" and r.rel is not None and since <= r.rel <= until}
|
|
keep = window_keys | (tracked_set & {r.key for r in all_rows})
|
|
rows = [r for r in all_rows if r.key in keep]
|
|
present = {r.key for r in all_rows}
|
|
meta = dict(raw.get("meta") or {})
|
|
meta.update(
|
|
{"source": "fixture", "state": state, "site": site, "since": since.isoformat(), "until": until.isoformat()}
|
|
)
|
|
data = FeedData(
|
|
meta=meta,
|
|
rows=rows,
|
|
tracked=list(tracked),
|
|
tracked_missing=[k for k in dict.fromkeys(tracked) if k not in present],
|
|
window_count=len(window_keys),
|
|
tracked_count=len(tracked),
|
|
)
|
|
bases = {r.wo for r in rows}
|
|
if with_materials:
|
|
if raw.get("materials") is None:
|
|
data.materials, data.material_legs = None, None
|
|
else:
|
|
lines = [parse_material(d) for d in raw.get("materials") or []]
|
|
data.materials = [m for m in lines if m.wo in bases]
|
|
data.material_legs = {m.key for m in data.materials if m.st in MATERIAL_STATUSES}
|
|
# A leg the file lists with status F or R counts as covered even when it has no lines.
|
|
for r in all_rows:
|
|
if r.wo in bases and r.st in MATERIAL_STATUSES:
|
|
data.material_legs.add(r.key)
|
|
if with_orders:
|
|
obb = parse_orders(raw.get("orders_by_base"))
|
|
obp = parse_orders(raw.get("orders_by_project"))
|
|
if obb is None or obp is None:
|
|
data.orders_by_base, data.orders_by_project = None, None
|
|
else:
|
|
projects = {project_of(b) for b in bases}
|
|
data.orders_by_base = {b: v for b, v in obb.items() if b in bases}
|
|
data.orders_by_project = {p: v for p, v in obp.items() if p in projects}
|
|
if with_labor and raw.get("labor") is not None:
|
|
data.labor = [parse_labor(d) for d in raw.get("labor") or []]
|
|
return data
|
|
|
|
|
|
def make(settings: Any, visual: Any = None) -> FeedSource:
|
|
"""The source the settings name. `visualkit` needs the open `visual` the job made."""
|
|
if settings.feed_source == "fixture":
|
|
return FixtureSource(Path(settings.fixture_dir))
|
|
if visual is None:
|
|
raise ValueError("the visualkit feed source needs an open visualkit.Visual")
|
|
return VisualKitSource(visual)
|
|
|
|
|
|
__all__ = [
|
|
"FeedSource",
|
|
"FixtureSource",
|
|
"VisualKitSource",
|
|
"make",
|
|
"project_of",
|
|
"redact",
|
|
"status_code",
|
|
"text",
|
|
]
|