"""The VISUAL cache in PostgreSQL: one batch per read, the rows and material lines as VISUAL last showed them, the orders and their links, labor per operation, and a change row for everything VISUAL added, changed, dropped or brought back. `write_batch` runs in one transaction. Rows and materials go in through COPY into a temp table, then an upsert keyed on the typed key (wo_base, lot, split, sub, seq[, piece]), all text but seq and piece. Rows in play that this read did not return get `gone_batch`, unless the batch is suspect: no window rows where the previous batch had some, or more than 5 tracked keys missing and more than 10 % of them. The apply reads back from here (`rows_for_apply`, `orders`, `labor_hours`), so it works on one consistent snapshot and keeps the last good orders when this run's order read failed. """ from __future__ import annotations import hashlib from collections.abc import Collection, Iterable, Mapping from dataclasses import dataclass, field from datetime import date from decimal import Decimal from typing import Any from psycopg.rows import dict_row from psycopg.types.json import Jsonb from .fingerprint import canonical, norm_decimal from .materials import material_from_row from .types import FeedData, Material, Order, Row, WoKey, num ROW_COLS = ( "wo_base", "lot", "split", "sub", "seq", "res", "dwg", "rev", "ps", "descr", "qty", "tgt", "rel", "want", "st", "part", "barcode", ) MAT_COLS = ( "wo_base", "lot", "split", "sub", "seq", "piece", "part", "qty", "issued", "whs", "loc", "st", "istat", "dwg", "descr", "comm", "cons", ) SUSPECT_MISSING_MIN = 5 SUSPECT_MISSING_SHARE = Decimal("0.10") @dataclass class Batch: id: int suspect: bool counts: dict[str, Any] = field(default_factory=dict) reason: str = "" # ---------------------------------------------------------------- row dicts and hashes def _d(v: Any) -> str | None: return v.isoformat() if isinstance(v, date) else None def row_doc(r: Row) -> dict[str, Any]: """The row as a change-row dict: text values, numbers as text, dates ISO.""" return { "wo": r.wo, "lot": r.lot, "split": r.split, "sub": r.sub, "seq": r.seq, "res": r.res, "dwg": r.dwg, "rev": r.rev, "ps": r.ps, "desc": r.desc, "qty": norm_decimal(r.qty), "tgt": norm_decimal(r.tgt), "rel": _d(r.rel), "want": _d(r.want), "st": r.st, "part": r.part, "barcode": r.barcode, } def mat_doc(m: Material) -> dict[str, Any]: return { "wo": m.wo, "lot": m.lot, "split": m.split, "sub": m.sub, "seq": m.seq, "piece": m.piece, "part": m.part, "qty": norm_decimal(m.qty), "issued": norm_decimal(m.issued), "whs": m.whs, "loc": m.loc, "st": m.st, "istat": m.istat, "dwg": m.dwg, "desc": m.desc, "comm": m.comm, "cons": m.cons, } def doc_hash(doc: Mapping[str, Any]) -> bytes: return hashlib.sha256(canonical(dict(doc))).digest() # SQL that rebuilds the same dict from a cached row, for the change rows' `before`. _ROW_BEFORE = ( "jsonb_build_object('wo', v.wo_base, 'lot', v.lot, 'split', v.split, 'sub', v.sub, 'seq', v.seq, 'res', v.res," " 'dwg', v.dwg, 'rev', v.rev, 'ps', v.ps, 'desc', v.descr, 'qty', v.qty::text, 'tgt', v.tgt::text," " 'rel', v.rel, 'want', v.want, 'st', v.st, 'part', v.part, 'barcode', v.barcode)" ) _MAT_BEFORE = ( "jsonb_build_object('wo', v.wo_base, 'lot', v.lot, 'split', v.split, 'sub', v.sub, 'seq', v.seq," " 'piece', v.piece, 'part', v.part, 'qty', v.qty::text, 'issued', v.issued::text, 'whs', v.whs, 'loc', v.loc," " 'st', v.st, 'istat', v.istat, 'dwg', v.dwg, 'desc', v.descr, 'comm', v.comm, 'cons', v.cons)" ) def _key_arrays(keys: Iterable[WoKey]) -> tuple[list[str], list[str], list[str], list[str]]: ks = list(dict.fromkeys(keys)) return [k.base for k in ks], [k.lot for k in ks], [k.split for k in ks], [k.sub for k in ks] _KEYS_UNNEST = "unnest(%s::text[], %s::text[], %s::text[], %s::text[]) AS k(b, l, sp, su)" # ---------------------------------------------------------------- write def is_suspect(window_rows: int, prev_window_rows: int | None, missing: int, tracked: int) -> str: """The reason a batch is suspect, or ''.""" if window_rows == 0 and (prev_window_rows or 0) > 0: return f"the window read returned nothing (the previous read had {prev_window_rows})" if missing > SUSPECT_MISSING_MIN and Decimal(missing) > SUSPECT_MISSING_SHARE * tracked: return f"{missing} of {tracked} tracked work orders missing" return "" def write_batch(conn: Any, run_id: int | None, data: FeedData, source_desc: str, window: tuple[date, date]) -> Batch: """Record one read: the batch row, rows, materials, orders, labor and the change rows. One transaction.""" since, until = window with conn.transaction(): prev = conn.execute("SELECT window_rows FROM visual_batches ORDER BY id DESC LIMIT 1").fetchone() missing = len(set(data.tracked_missing)) tracked_n = len(set(data.tracked)) or data.tracked_count reason = is_suspect(data.window_count, prev[0] if prev else None, missing, tracked_n) suspect = bool(reason) server, _, database = source_desc.partition(" / ") bid = int( conn.execute( "INSERT INTO visual_batches (run_id, server, database, site, since, until, window_rows, tracked_keys," " missing_keys, suspect) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id", ( run_id, str(data.meta.get("server") or server), str(data.meta.get("database") or database) or None, data.meta.get("site"), since, until, data.window_count, tracked_n, missing, suspect, ), ).fetchone()[0] ) counts: dict[str, Any] = {"rows": len(data.rows), "suspect": int(suspect)} if reason: counts["suspect_reason"] = reason counts.update(_write_rows(conn, bid, data, window, suspect)) if data.materials is not None: counts.update(_write_materials(conn, bid, data, suspect)) counts["materials"] = len(data.materials) if data.orders_by_base is not None and data.orders_by_project is not None: counts["orders"] = _write_orders(conn, bid, data) if data.labor is not None: counts["labor_legs"] = _write_labor(conn, data) counts["labor_read"] = 1 conn.execute("UPDATE visual_batches SET counts = %s WHERE id = %s", (Jsonb(counts), bid)) return Batch(id=bid, suspect=suspect, counts=counts, reason=reason) def _write_rows(conn: Any, bid: int, data: FeedData, window: tuple[date, date], suspect: bool) -> dict[str, int]: uniq: dict[tuple[WoKey, int], Row] = {} for r in data.rows: uniq[(r.key, r.seq)] = r conn.execute( "CREATE TEMP TABLE t_rows (wo_base text, lot text, split text, sub text, seq int, res text, dwg text," " rev text, ps text, descr text, qty numeric, tgt numeric, rel date, want date, st text, part text," " barcode text, row_hash bytea, doc jsonb) ON COMMIT DROP" ) with ( conn.cursor() as cur, cur.copy( "COPY t_rows (wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st, part," " barcode, row_hash, doc) FROM STDIN" ) as cp, ): for r in uniq.values(): doc = row_doc(r) cp.write_row( ( r.wo, r.lot, r.split, r.sub, r.seq, r.res, r.dwg, r.rev, r.ps, r.desc, r.qty, r.tgt, r.rel, r.want, r.st, r.part, r.barcode or None, doc_hash(doc), Jsonb(doc), ) ) out: dict[str, int] = {} join = ( "visual_rows v ON v.wo_base = t.wo_base AND v.lot = t.lot AND v.split = t.split AND v.sub = t.sub" " AND v.seq = t.seq" ) out["rows_added"] = conn.execute( "INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, after)" # noqa: S608 f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'added', t.doc FROM t_rows t LEFT JOIN {join}" " WHERE v.id IS NULL", (bid,), ).rowcount out["rows_returned"] = conn.execute( "INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before, after)" # noqa: S608 f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'returned', {_ROW_BEFORE}, t.doc FROM t_rows t" f" JOIN {join} WHERE v.gone_batch IS NOT NULL", (bid,), ).rowcount out["rows_changed"] = conn.execute( "INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before, after)" # noqa: S608 f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'changed', {_ROW_BEFORE}, t.doc FROM t_rows t" f" JOIN {join} WHERE v.gone_batch IS NULL AND v.row_hash <> t.row_hash", (bid,), ).rowcount conn.execute( "INSERT INTO visual_rows (wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st," " part, barcode, row_hash, first_batch, last_batch)" " SELECT wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st, part, barcode," " row_hash, %s, %s FROM t_rows" " ON CONFLICT (wo_base, lot, split, sub, seq) DO UPDATE SET res = EXCLUDED.res, dwg = EXCLUDED.dwg," " rev = EXCLUDED.rev, ps = EXCLUDED.ps, descr = EXCLUDED.descr, qty = EXCLUDED.qty, tgt = EXCLUDED.tgt," " rel = EXCLUDED.rel, want = EXCLUDED.want, st = EXCLUDED.st, part = EXCLUDED.part," " barcode = EXCLUDED.barcode, row_hash = EXCLUDED.row_hash, last_batch = EXCLUDED.last_batch," " gone_batch = NULL", (bid, bid), ) out["rows_gone"] = 0 if not suspect: b, lot, sp, su = _key_arrays(data.tracked) out["rows_gone"] = conn.execute( "WITH gone AS (UPDATE visual_rows v SET gone_batch = %s" # noqa: S608 " WHERE v.gone_batch IS NULL AND v.last_batch <> %s" f" AND (EXISTS (SELECT 1 FROM {_KEYS_UNNEST} WHERE k.b = v.wo_base AND k.l = v.lot AND k.sp = v.split" " AND k.su = v.sub) OR (v.st = 'F' AND v.rel BETWEEN %s AND %s)) RETURNING v.*)" " INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before)" f" SELECT %s, v.wo_base, v.lot, v.split, v.sub, v.seq, 'gone', {_ROW_BEFORE} FROM gone v", (bid, bid, b, lot, sp, su, window[0], window[1], bid), ).rowcount return out def _write_materials(conn: Any, bid: int, data: FeedData, suspect: bool) -> dict[str, int]: uniq: dict[tuple[WoKey, int, int], Material] = {} for m in data.materials or []: uniq[(m.key, m.seq, m.piece)] = m conn.execute( "CREATE TEMP TABLE t_mats (wo_base text, lot text, split text, sub text, seq int, piece int, part text," " qty numeric, issued numeric, whs text, loc text, st text, istat int, dwg text, descr text, comm text," " cons text, mat_hash bytea, doc jsonb) ON COMMIT DROP" ) with ( conn.cursor() as cur, cur.copy( "COPY t_mats (wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat, dwg, descr," " comm, cons, mat_hash, doc) FROM STDIN" ) as cp, ): for m in uniq.values(): doc = mat_doc(m) cp.write_row( ( m.wo, m.lot, m.split, m.sub, m.seq, m.piece, m.part, m.qty, m.issued, m.whs, m.loc, m.st, m.istat, m.dwg, m.desc, m.comm, m.cons, doc_hash(doc), Jsonb(doc), ) ) join = ( "visual_materials v ON v.wo_base = t.wo_base AND v.lot = t.lot AND v.split = t.split AND v.sub = t.sub" " AND v.seq = t.seq AND v.piece = t.piece" ) cols = "batch_id, wo_base, lot, split, sub, seq, piece, kind" out: dict[str, int] = {} out["mats_added"] = conn.execute( f"INSERT INTO visual_material_changes ({cols}, after)" # noqa: S608 f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'added', t.doc FROM t_mats t LEFT JOIN {join}" " WHERE v.id IS NULL", (bid,), ).rowcount out["mats_returned"] = conn.execute( f"INSERT INTO visual_material_changes ({cols}, before, after)" # noqa: S608 f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'returned', {_MAT_BEFORE}, t.doc" f" FROM t_mats t JOIN {join} WHERE v.gone_batch IS NOT NULL", (bid,), ).rowcount out["mats_changed"] = conn.execute( f"INSERT INTO visual_material_changes ({cols}, before, after)" # noqa: S608 f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'changed', {_MAT_BEFORE}, t.doc" f" FROM t_mats t JOIN {join} WHERE v.gone_batch IS NULL AND v.mat_hash <> t.mat_hash", (bid,), ).rowcount conn.execute( "INSERT INTO visual_materials (wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat," " dwg, descr, comm, cons, mat_hash, first_batch, last_batch)" " SELECT wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat, dwg, descr, comm," " cons, mat_hash, %s, %s FROM t_mats" " ON CONFLICT (wo_base, lot, split, sub, seq, piece) DO UPDATE SET part = EXCLUDED.part, qty = EXCLUDED.qty," " issued = EXCLUDED.issued, whs = EXCLUDED.whs, loc = EXCLUDED.loc, st = EXCLUDED.st," " istat = EXCLUDED.istat, dwg = EXCLUDED.dwg, descr = EXCLUDED.descr, comm = EXCLUDED.comm," " cons = EXCLUDED.cons, mat_hash = EXCLUDED.mat_hash, last_batch = EXCLUDED.last_batch, gone_batch = NULL", (bid, bid), ) out["mats_gone"] = 0 if not suspect: bases = sorted({r.wo for r in data.rows} | {m.wo for m in data.materials or []}) out["mats_gone"] = conn.execute( "WITH gone AS (UPDATE visual_materials v SET gone_batch = %s WHERE v.gone_batch IS NULL" # noqa: S608 " AND v.last_batch <> %s AND v.wo_base = ANY(%s) RETURNING v.*)" f" INSERT INTO visual_material_changes ({cols}, before)" f" SELECT %s, v.wo_base, v.lot, v.split, v.sub, v.seq, v.piece, 'gone', {_MAT_BEFORE} FROM gone v", (bid, bid, bases, bid), ).rowcount return out def _write_orders(conn: Any, bid: int, data: FeedData) -> int: by_base = data.orders_by_base or {} by_project = data.orders_by_project or {} every: dict[str, Order] = {} for lists in (by_base, by_project): for orders in lists.values(): for o in orders: every.setdefault(o.cust_order_id, o) for o in every.values(): conn.execute( "INSERT INTO visual_orders (cust_order_id, status, customer_id, customer_name, customer_po_ref," " order_date, desired_ship_date, promise_date, last_batch) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)" " ON CONFLICT (cust_order_id) DO UPDATE SET status = EXCLUDED.status, customer_id = EXCLUDED.customer_id," " customer_name = EXCLUDED.customer_name, customer_po_ref = EXCLUDED.customer_po_ref," " order_date = coalesce(EXCLUDED.order_date, visual_orders.order_date)," " desired_ship_date = EXCLUDED.desired_ship_date, promise_date = EXCLUDED.promise_date," " last_batch = EXCLUDED.last_batch", ( o.cust_order_id, o.status, o.customer_id, o.customer_name, o.customer_po_ref, o.order_date, o.desired_ship_date, o.promise_date, bid, ), ) bases = sorted({r.wo for r in data.rows} | set(by_base)) projects = sorted({r.wo.partition("-")[0] or r.wo for r in data.rows} | set(by_project)) conn.execute( "DELETE FROM visual_order_links WHERE (kind = 'base' AND key = ANY(%s))" " OR (kind = 'project' AND key = ANY(%s))", (bases, projects), ) n = 0 for kind, lists in (("base", by_base), ("project", by_project)): for key, orders in lists.items(): seen: set[str] = set() for rank, o in enumerate(orders): if o.cust_order_id in seen: continue seen.add(o.cust_order_id) conn.execute( "INSERT INTO visual_order_links (kind, key, cust_order_id, rank, lines, top_jobs, last_batch)" " VALUES (%s, %s, %s, %s, %s, %s, %s)", (kind, key, o.cust_order_id, rank, o.lines, o.top_jobs, bid), ) n += 1 return len(every) def _write_labor(conn: Any, data: FeedData) -> int: n = 0 for lg in data.labor or []: conn.execute( "INSERT INTO visual_labor (wo_base, lot, split, sub, seq, hours, tickets, last_ticket, observed_at)" " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now())" " ON CONFLICT (wo_base, lot, split, sub, seq) DO UPDATE SET hours = EXCLUDED.hours," " tickets = EXCLUDED.tickets, last_ticket = EXCLUDED.last_ticket, observed_at = now()", (lg.wo, lg.lot, lg.split, lg.sub, lg.seq, lg.hours, lg.tickets, lg.last_ticket), ) n += 1 return n # ---------------------------------------------------------------- read back def row_from_cache(r: Mapping[str, Any]) -> Row: return Row( wo=r["wo_base"], lot=r["lot"], split=r["split"], sub=r["sub"], seq=int(r["seq"]), res=r["res"], dwg=r["dwg"], rev=r["rev"], ps=r["ps"], desc=r["descr"], qty=num(r["qty"]), tgt=num(r["tgt"]), rel=r["rel"], want=r["want"], st=r["st"], part=r["part"], barcode=r["barcode"] or "", ) def rows_for_apply(conn: Any, keys_in_play: Collection[WoKey]) -> tuple[list[Row], list[Material], set[WoKey]]: """Live cached rows and material lines of every work order (base and lot) among these keys, so folded operations and planner instructions of the same work order come along. material_legs = legs with live material lines whose status is F or R.""" pairs = sorted({(k.base, k.lot) for k in keys_in_play}) if not pairs: return [], [], set() bases = [p[0] for p in pairs] lots = [p[1] for p in pairs] sql_join = "JOIN unnest(%s::text[], %s::text[]) AS k(b, l) ON v.wo_base = k.b AND v.lot = k.l" with conn.cursor(row_factory=dict_row) as cur: rows = cur.execute( f"SELECT v.* FROM visual_rows v {sql_join} WHERE v.gone_batch IS NULL" # noqa: S608 " ORDER BY v.wo_base, v.lot, v.split, v.sub, v.seq", (bases, lots), ).fetchall() mats = cur.execute( f"SELECT v.* FROM visual_materials v {sql_join} WHERE v.gone_batch IS NULL" # noqa: S608 " ORDER BY v.wo_base, v.lot, v.split, v.sub, v.seq, v.piece", (bases, lots), ).fetchall() materials = [material_from_row(m) for m in mats] legs = {m.key for m in materials if m.st in ("F", "R")} return [row_from_cache(r) for r in rows], materials, legs def orders( conn: Any, *, bases: Iterable[str] | None = None, projects: Iterable[str] | None = None ) -> tuple[dict[str, list[Order]], dict[str, list[Order]]]: """The cached order links as (by_base, by_project), in rank order. Without filters: every link.""" params: list[Any] = [] where = "" if bases is not None or projects is not None: where = " WHERE (l.kind = 'base' AND l.key = ANY(%s)) OR (l.kind = 'project' AND l.key = ANY(%s))" params = [sorted(set(bases or ())), sorted(set(projects or ()))] with conn.cursor(row_factory=dict_row) as cur: rows = cur.execute( "SELECT l.kind, l.key, l.rank, l.lines, l.top_jobs, o.* FROM visual_order_links l" # noqa: S608 f" JOIN visual_orders o ON o.cust_order_id = l.cust_order_id{where} ORDER BY l.kind, l.key, l.rank", params, ).fetchall() by_base: dict[str, list[Order]] = {} by_project: dict[str, list[Order]] = {} for r in rows: o = Order( cust_order_id=r["cust_order_id"], status=r["status"], customer_id=r["customer_id"], customer_name=r["customer_name"], customer_po_ref=r["customer_po_ref"], order_date=r["order_date"], desired_ship_date=r["desired_ship_date"], promise_date=r["promise_date"], lines=list(r["lines"] or []), top_jobs=list(r["top_jobs"] or []), ) (by_base if r["kind"] == "base" else by_project).setdefault(r["key"], []).append(o) return by_base, by_project def labor_hours(conn: Any) -> dict[tuple[WoKey, int], Decimal]: """Cached ticket hours per (leg, seq).""" out: dict[tuple[WoKey, int], Decimal] = {} for r in conn.execute("SELECT wo_base, lot, split, sub, seq, hours FROM visual_labor"): out[(WoKey(r[0], r[1], r[2], r[3]), int(r[4]))] = num(r[5]) return out def last_labor_read(conn: Any) -> Any: """When labor was last read (a batch that carried labor), or None.""" r = conn.execute("SELECT max(started_at) FROM visual_batches WHERE counts ? 'labor_read'").fetchone() return r[0] if r else None def finish_batch(conn: Any, batch_id: int, counts: Mapping[str, Any]) -> None: """Merge the apply's counts into the batch row and set finished_at.""" conn.execute( "UPDATE visual_batches SET finished_at = now(), counts = counts || %s WHERE id = %s", (Jsonb(dict(counts)), batch_id), )