Phase 3: VISUAL feed and M-Files drawings in the worker
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,491 @@
|
||||
"""Where the feed reads VISUAL from: VisualKit (DEV unless the stack says otherwise) or a synthetic fixture file.
|
||||
|
||||
`VisualKitSource` copies the read calls and output shapes of VisualKit's examples/packet_station_feed.py: the
|
||||
window read, the tracked keys by key, operations, operation text and parts for rows; for_bases, requirements,
|
||||
requirement text, parts and part sites for materials; DEMAND_SUPPLY_LINK orders by base and by project; labor
|
||||
tickets summed per operation. Read-only: VisualKit enforces it. No cost, price, amount or pay column is read.
|
||||
|
||||
`FixtureSource` reads `<dir>/state` and `<dir>/feed.<state>.json` in the same contract shape (CI and tests).
|
||||
|
||||
Every value goes through `types.parse_row` / `parse_material` / `parse_order`, so both sources give identical
|
||||
FeedData.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterable, Sequence
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .types import (
|
||||
FeedData,
|
||||
LaborLeg,
|
||||
Material,
|
||||
Order,
|
||||
Row,
|
||||
WoKey,
|
||||
parse_labor,
|
||||
parse_material,
|
||||
parse_order,
|
||||
parse_orders,
|
||||
parse_row,
|
||||
status_code,
|
||||
text,
|
||||
)
|
||||
|
||||
STATUS_WORDS = {"F": "Firmed", "R": "Released", "U": "Unreleased", "C": "Closed", "X": "Cancelled"}
|
||||
MATERIAL_STATUSES = ("F", "R")
|
||||
OPEN_ORDER_STATUSES = ("F", "H", "R")
|
||||
QTY_EMPLOYEE = "9999"
|
||||
|
||||
_SECRETISH = re.compile(r"(?i)(password|pwd|secret|token)\s*[=:]\s*\S+")
|
||||
|
||||
|
||||
def redact(message: object) -> str:
|
||||
"""An exception as one short line with anything password-like removed."""
|
||||
s = _SECRETISH.sub(
|
||||
r"\1=***", f"{type(message).__name__}: {message}" if isinstance(message, BaseException) else str(message)
|
||||
)
|
||||
return s.replace("\r", " ").replace("\n", " ")[:500]
|
||||
|
||||
|
||||
class FeedSource(Protocol):
|
||||
def read(
|
||||
self,
|
||||
*,
|
||||
site: str,
|
||||
since: date,
|
||||
until: date,
|
||||
tracked: Sequence[WoKey],
|
||||
with_materials: bool,
|
||||
with_orders: bool,
|
||||
with_labor: bool,
|
||||
labor_since: date | None,
|
||||
) -> FeedData: ...
|
||||
|
||||
def describe(self) -> str: ...
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- value helpers (as the VisualKit example)
|
||||
|
||||
|
||||
def _to_number(v: Any) -> Any:
|
||||
if v is None or isinstance(v, bool):
|
||||
return v
|
||||
if isinstance(v, Decimal):
|
||||
return v
|
||||
return Decimal(str(v))
|
||||
|
||||
|
||||
def _fmt_date(v: date | datetime | None) -> str | None:
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, datetime):
|
||||
v = v.date()
|
||||
return v.isoformat()
|
||||
|
||||
|
||||
def _clean(s: str | None) -> str:
|
||||
if not s:
|
||||
return ""
|
||||
return s.replace("\r\n", "\n").replace("\r", "\n")
|
||||
|
||||
|
||||
def _istat(calc_qty: Any, issued_qty: Any, line_status: str | None) -> int:
|
||||
"""Job Pull's issue state: 3 over-issued, 0 line closed, 2 nothing issued, 1 partly."""
|
||||
calc = Decimal(str(calc_qty or 0))
|
||||
issued = Decimal(str(issued_qty or 0))
|
||||
if issued > calc:
|
||||
return 3
|
||||
if (line_status or "").strip().upper() == "C":
|
||||
return 0
|
||||
if issued == 0:
|
||||
return 2
|
||||
return 1
|
||||
|
||||
|
||||
def _yes_no(flag: bool | None) -> str:
|
||||
return "" if flag is None else ("Y" if flag else "N")
|
||||
|
||||
|
||||
def project_of(base: str) -> str:
|
||||
"""VisualKit's project_of: the base before its first dash, or the whole base."""
|
||||
head = base.partition("-")[0]
|
||||
return head or base
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- VisualKit
|
||||
|
||||
|
||||
class VisualKitSource:
|
||||
"""The five reads through one `visualkit.Visual` (made by the job per run, closed after)."""
|
||||
|
||||
def __init__(self, visual: Any) -> None:
|
||||
self.v = visual
|
||||
|
||||
def describe(self) -> str:
|
||||
s = self.v.settings
|
||||
return f"{s.server} / {s.database}"
|
||||
|
||||
def _key(self, k: WoKey) -> Any:
|
||||
from visualkit import WorkOrderKey
|
||||
|
||||
return WorkOrderKey(k.base, k.lot, k.split, k.sub, "W")
|
||||
|
||||
def _load_parts(self, part_ids: Iterable[str | None], cache: dict[str, Any]) -> None:
|
||||
missing = sorted({p for p in part_ids if p and p not in cache})
|
||||
if missing:
|
||||
cache.update(self.v.parts.get_many(missing))
|
||||
|
||||
def read(
|
||||
self,
|
||||
*,
|
||||
site: str,
|
||||
since: date,
|
||||
until: date,
|
||||
tracked: Sequence[WoKey],
|
||||
with_materials: bool,
|
||||
with_orders: bool,
|
||||
with_labor: bool,
|
||||
labor_since: date | None,
|
||||
) -> FeedData:
|
||||
v = self.v
|
||||
s = v.settings
|
||||
meta: dict[str, Any] = {
|
||||
"source": "visualkit",
|
||||
"server": s.server,
|
||||
"database": s.database,
|
||||
"site": site,
|
||||
"since": since.isoformat(),
|
||||
"until": until.isoformat(),
|
||||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||||
}
|
||||
data = FeedData(meta=meta, rows=[], tracked=list(tracked), tracked_count=len(tracked))
|
||||
parts: dict[str, Any] = {}
|
||||
|
||||
# 1. The window: firmed TYPE W work orders released in [since, until], paged.
|
||||
wos: list[Any] = []
|
||||
offset: int | None = 0
|
||||
while offset is not None:
|
||||
page = v.work_orders.released(site, since, until, status=("F",), offset=offset)
|
||||
wos.extend(page.items)
|
||||
offset = page.next_offset
|
||||
data.window_count = len(wos)
|
||||
|
||||
# 2. The tracked keys, whatever their status or date. Keys VISUAL lacks are missing.
|
||||
asked = {self._key(k): k for k in dict.fromkeys(tracked)}
|
||||
found = v.work_orders.get_many(list(asked)) if asked else {}
|
||||
data.tracked_missing = [k for vk, k in asked.items() if vk not in found]
|
||||
have = {wo.key for wo in wos}
|
||||
for wo in found.values():
|
||||
if wo.key not in have:
|
||||
wos.append(wo)
|
||||
have.add(wo.key)
|
||||
|
||||
# 3. Rows for the union.
|
||||
keys = [wo.key for wo in wos]
|
||||
ops = v.work_orders.operations_for(keys) if keys else {}
|
||||
specs = v.work_orders.operation_text_for(keys) if keys else {}
|
||||
self._load_parts((wo.part_id for wo in wos), parts)
|
||||
for wo in wos:
|
||||
part = parts.get(wo.part_id) if wo.part_id else None
|
||||
desc = _clean(part.description if part else None)
|
||||
qty = wo.desired_qty or Decimal(0)
|
||||
for op in ops.get(wo.key, []):
|
||||
data.rows.append(
|
||||
parse_row(
|
||||
{
|
||||
"wo": wo.base_id,
|
||||
"lot": wo.lot_id,
|
||||
"split": wo.split_id,
|
||||
"sub": wo.sub_id,
|
||||
"seq": op.sequence_no,
|
||||
"res": op.resource_id or "",
|
||||
"dwg": wo.drawing_id or "",
|
||||
"rev": wo.drawing_rev_no or "",
|
||||
"ps": specs.get((wo.key, op.sequence_no), ""),
|
||||
"desc": desc,
|
||||
"qty": _to_number(qty),
|
||||
"tgt": _to_number((op.run or Decimal(0)) * qty),
|
||||
"rel": _fmt_date(wo.desired_rls_date),
|
||||
"want": _fmt_date(wo.desired_want_date),
|
||||
"st": STATUS_WORDS.get((wo.status or "").strip().upper(), wo.status or ""),
|
||||
"barcode": op.barcode(),
|
||||
"part": wo.part_id or "",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
bases = sorted({wo.base_id for wo in wos})
|
||||
if with_materials:
|
||||
try:
|
||||
data.materials, data.material_legs = self._materials(bases, site, parts)
|
||||
except Exception as e: # a failed materials read keeps the BOMs as they are
|
||||
data.materials, data.material_legs = None, None
|
||||
data.errors.append("materials: " + redact(e))
|
||||
if with_orders:
|
||||
try:
|
||||
data.orders_by_base = self._orders_by_base(bases)
|
||||
data.orders_by_project = self._orders_by_project({project_of(b) for b in bases})
|
||||
except Exception as e: # the apply keeps the last cached orders
|
||||
data.orders_by_base, data.orders_by_project = None, None
|
||||
data.errors.append("orders: " + redact(e))
|
||||
if with_labor:
|
||||
try:
|
||||
data.labor = self._labor(list(asked) or keys)
|
||||
except Exception as e:
|
||||
data.labor = None
|
||||
data.errors.append("labor: " + redact(e))
|
||||
return data
|
||||
|
||||
def _materials(self, bases: Sequence[str], site: str, parts: dict[str, Any]) -> tuple[list[Material], set[WoKey]]:
|
||||
v = self.v
|
||||
if not bases:
|
||||
return [], set()
|
||||
legs = v.work_orders.for_bases(bases, type="W", status=MATERIAL_STATUSES, site_id=site)
|
||||
keys = [wo.key for wo in legs]
|
||||
reqs = v.work_orders.requirements_for(keys) if keys else {}
|
||||
text_keys = [k for k in keys if any(not r.part_id for r in reqs.get(k, []))]
|
||||
texts = v.work_orders.requirement_text_for(text_keys) if text_keys else {}
|
||||
part_ids = {r.part_id for rs in reqs.values() for r in rs if r.part_id}
|
||||
self._load_parts(part_ids, parts)
|
||||
sites = v.parts.sites_for(sorted(part_ids), site) if part_ids else {}
|
||||
out: list[Material] = []
|
||||
covered: set[WoKey] = set()
|
||||
for wo in legs:
|
||||
covered.add(WoKey.of(wo.base_id, wo.lot_id, wo.split_id, wo.sub_id))
|
||||
for r in reqs.get(wo.key, []):
|
||||
part = parts.get(r.part_id) if r.part_id else None
|
||||
ps = sites.get(r.part_id) if r.part_id else None
|
||||
desc = (
|
||||
_clean(part.description if part else None)
|
||||
if r.part_id
|
||||
else _clean(texts.get((wo.key, r.operation_seq_no, r.piece_no)))
|
||||
)
|
||||
out.append(
|
||||
parse_material(
|
||||
{
|
||||
"wo": wo.base_id,
|
||||
"lot": wo.lot_id,
|
||||
"split": wo.split_id,
|
||||
"sub": wo.sub_id,
|
||||
"seq": r.operation_seq_no,
|
||||
"piece": r.piece_no,
|
||||
"part": r.part_id or None,
|
||||
"qty": _to_number(r.calc_qty or Decimal(0)),
|
||||
"issued": _to_number(r.issued_qty or Decimal(0)),
|
||||
"whs": (ps.primary_whs_id or "").upper() if ps else "",
|
||||
"loc": (ps.primary_loc_id or "").upper() if ps else "",
|
||||
"st": wo.status,
|
||||
"istat": _istat(r.calc_qty, r.issued_qty, r.status),
|
||||
"dwg": wo.drawing_id or "",
|
||||
"desc": desc,
|
||||
"comm": (part.commodity_code or "") if part else "",
|
||||
"cons": _yes_no(part.consumable) if part else "",
|
||||
}
|
||||
)
|
||||
)
|
||||
return out, covered
|
||||
|
||||
def _orders_by_base(self, bases: Sequence[str]) -> dict[str, list[Order]]:
|
||||
v = self.v
|
||||
if not bases:
|
||||
return {}
|
||||
pairs = v.links.orders_for_bases(list(bases))
|
||||
order_ids = sorted({oid for ps in pairs.values() for oid, _ in ps})
|
||||
orders = v.orders.get_many(order_ids) if order_ids else {}
|
||||
customers = v.orders.customers(sorted({o.customer_id for o in orders.values()})) if orders else {}
|
||||
out: dict[str, list[Order]] = {}
|
||||
for base in bases:
|
||||
lines_by_order: dict[str, list[int]] = {}
|
||||
for oid, line_no in pairs.get(base, []):
|
||||
lines_by_order.setdefault(oid, [])
|
||||
if line_no is not None:
|
||||
lines_by_order[oid].append(line_no)
|
||||
if not lines_by_order:
|
||||
continue
|
||||
out[base] = []
|
||||
for oid, lines in sorted(lines_by_order.items()):
|
||||
o = orders.get(oid)
|
||||
cust = customers.get(o.customer_id) if o else None
|
||||
out[base].append(
|
||||
parse_order(
|
||||
{
|
||||
"cust_order_id": oid,
|
||||
"lines": sorted(set(lines)),
|
||||
"status": o.status if o else None,
|
||||
"customer_id": o.customer_id if o else None,
|
||||
"customer_name": cust.name if cust else None,
|
||||
"customer_po_ref": o.customer_po_ref if o else None,
|
||||
"order_date": _fmt_date(getattr(o, "order_date", None)) if o else None,
|
||||
"desired_ship_date": _fmt_date(o.desired_ship_date) if o else None,
|
||||
"promise_date": _fmt_date(o.promise_date) if o else None,
|
||||
}
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
def _orders_by_project(self, projects: Iterable[str]) -> dict[str, list[Order]]:
|
||||
v = self.v
|
||||
wanted = sorted(set(projects))
|
||||
if not wanted:
|
||||
return {}
|
||||
found = v.links.orders_for_projects(wanted)
|
||||
cust_ids = sorted({po.customer_id for pos in found.values() for po in pos if po.customer_id})
|
||||
customers = v.orders.customers(cust_ids) if cust_ids else {}
|
||||
out: dict[str, list[Order]] = {}
|
||||
for project in wanted:
|
||||
entries: dict[str, dict[str, Any]] = {}
|
||||
for po in found.get(project, []):
|
||||
e = entries.get(po.cust_order_id)
|
||||
if e is None:
|
||||
cust = customers.get(po.customer_id) if po.customer_id else None
|
||||
e = entries[po.cust_order_id] = {
|
||||
"cust_order_id": po.cust_order_id,
|
||||
"lines": [],
|
||||
"top_jobs": [],
|
||||
"status": po.status,
|
||||
"customer_id": po.customer_id,
|
||||
"customer_name": cust.name if cust else None,
|
||||
"customer_po_ref": po.customer_po_ref,
|
||||
"order_date": _fmt_date(po.order_date),
|
||||
"desired_ship_date": _fmt_date(po.desired_ship_date),
|
||||
"promise_date": _fmt_date(po.promise_date),
|
||||
}
|
||||
if po.line_no is not None and po.line_no not in e["lines"]:
|
||||
e["lines"].append(po.line_no)
|
||||
if po.supply_base_id not in e["top_jobs"]:
|
||||
e["top_jobs"].append(po.supply_base_id)
|
||||
if entries:
|
||||
for e in entries.values():
|
||||
e["lines"].sort()
|
||||
e["top_jobs"].sort()
|
||||
ordered = sorted(entries.values(), key=lambda e: e["status"] not in OPEN_ORDER_STATUSES)
|
||||
out[project] = [parse_order(e) for e in ordered]
|
||||
return out
|
||||
|
||||
def _labor(self, keys: Sequence[Any]) -> list[LaborLeg]:
|
||||
"""Hours and ticket counts per (leg, seq). Employee 9999's quantity-only tickets and indirect tickets
|
||||
are left out. Hours only: no cost or pay column is read. VisualKit has no date filter on this call, so
|
||||
every ticket of the legs is read and summed (the totals need them all)."""
|
||||
if not keys:
|
||||
return []
|
||||
tickets = self.v.labor.tickets_for_work_orders(list(keys))
|
||||
sums: dict[tuple[WoKey, int], LaborLeg] = {}
|
||||
for vk, ts in tickets.items():
|
||||
k = WoKey.of(vk.base_id, vk.lot_id, vk.split_id, vk.sub_id)
|
||||
for t in ts:
|
||||
if t.operation_seq_no is None or t.is_quantity_only or t.is_indirect:
|
||||
continue
|
||||
seq = int(t.operation_seq_no)
|
||||
leg = sums.get((k, seq))
|
||||
if leg is None:
|
||||
leg = sums[(k, seq)] = LaborLeg(k.base, k.lot, k.split, k.sub, seq, Decimal(0), 0, None)
|
||||
leg.hours += Decimal(str(t.hours_worked or 0))
|
||||
leg.tickets += 1
|
||||
d = t.transaction_date.date() if isinstance(t.transaction_date, datetime) else t.transaction_date
|
||||
if d is not None and (leg.last_ticket is None or d > leg.last_ticket):
|
||||
leg.last_ticket = d
|
||||
return list(sums.values())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- fixtures
|
||||
|
||||
|
||||
class FixtureSource:
|
||||
"""Synthetic VISUAL: `<dir>/state` (default "1") picks `<dir>/feed.<state>.json`."""
|
||||
|
||||
def __init__(self, directory: Path) -> None:
|
||||
self.dir = Path(directory)
|
||||
|
||||
def state(self) -> str:
|
||||
try:
|
||||
s = (self.dir / "state").read_text(encoding="utf-8").strip()
|
||||
except OSError:
|
||||
s = ""
|
||||
return s or "1"
|
||||
|
||||
def describe(self) -> str:
|
||||
return f"fixture state {self.state()}"
|
||||
|
||||
def read(
|
||||
self,
|
||||
*,
|
||||
site: str,
|
||||
since: date,
|
||||
until: date,
|
||||
tracked: Sequence[WoKey],
|
||||
with_materials: bool,
|
||||
with_orders: bool,
|
||||
with_labor: bool,
|
||||
labor_since: date | None,
|
||||
) -> FeedData:
|
||||
state = self.state()
|
||||
raw = json.loads((self.dir / f"feed.{state}.json").read_text(encoding="utf-8"))
|
||||
all_rows: list[Row] = [parse_row(d) for d in raw.get("rows") or []]
|
||||
tracked_set = set(tracked)
|
||||
window_keys = {r.key for r in all_rows if r.st == "F" and r.rel is not None and since <= r.rel <= until}
|
||||
keep = window_keys | (tracked_set & {r.key for r in all_rows})
|
||||
rows = [r for r in all_rows if r.key in keep]
|
||||
present = {r.key for r in all_rows}
|
||||
meta = dict(raw.get("meta") or {})
|
||||
meta.update(
|
||||
{"source": "fixture", "state": state, "site": site, "since": since.isoformat(), "until": until.isoformat()}
|
||||
)
|
||||
data = FeedData(
|
||||
meta=meta,
|
||||
rows=rows,
|
||||
tracked=list(tracked),
|
||||
tracked_missing=[k for k in dict.fromkeys(tracked) if k not in present],
|
||||
window_count=len(window_keys),
|
||||
tracked_count=len(tracked),
|
||||
)
|
||||
bases = {r.wo for r in rows}
|
||||
if with_materials:
|
||||
if raw.get("materials") is None:
|
||||
data.materials, data.material_legs = None, None
|
||||
else:
|
||||
lines = [parse_material(d) for d in raw.get("materials") or []]
|
||||
data.materials = [m for m in lines if m.wo in bases]
|
||||
data.material_legs = {m.key for m in data.materials if m.st in MATERIAL_STATUSES}
|
||||
# A leg the file lists with status F or R counts as covered even when it has no lines.
|
||||
for r in all_rows:
|
||||
if r.wo in bases and r.st in MATERIAL_STATUSES:
|
||||
data.material_legs.add(r.key)
|
||||
if with_orders:
|
||||
obb = parse_orders(raw.get("orders_by_base"))
|
||||
obp = parse_orders(raw.get("orders_by_project"))
|
||||
if obb is None or obp is None:
|
||||
data.orders_by_base, data.orders_by_project = None, None
|
||||
else:
|
||||
projects = {project_of(b) for b in bases}
|
||||
data.orders_by_base = {b: v for b, v in obb.items() if b in bases}
|
||||
data.orders_by_project = {p: v for p, v in obp.items() if p in projects}
|
||||
if with_labor and raw.get("labor") is not None:
|
||||
data.labor = [parse_labor(d) for d in raw.get("labor") or []]
|
||||
return data
|
||||
|
||||
|
||||
def make(settings: Any, visual: Any = None) -> FeedSource:
|
||||
"""The source the settings name. `visualkit` needs the open `visual` the job made."""
|
||||
if settings.feed_source == "fixture":
|
||||
return FixtureSource(Path(settings.fixture_dir))
|
||||
if visual is None:
|
||||
raise ValueError("the visualkit feed source needs an open visualkit.Visual")
|
||||
return VisualKitSource(visual)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FeedSource",
|
||||
"FixtureSource",
|
||||
"VisualKitSource",
|
||||
"make",
|
||||
"project_of",
|
||||
"redact",
|
||||
"status_code",
|
||||
"text",
|
||||
]
|
||||
Reference in New Issue
Block a user