565 lines
22 KiB
Python
565 lines
22 KiB
Python
 
|
"""The VISUAL cache in PostgreSQL: one batch per read, the rows and material lines as VISUAL last showed them,
|
||
|
|
the orders and their links, labor per operation, and a change row for everything VISUAL added, changed, dropped
|
||
|
|
or brought back.
|
||
|
|
|
||
|
|
`write_batch` runs in one transaction. Rows and materials go in through COPY into a temp table, then an upsert
|
||
|
|
keyed on the typed key (wo_base, lot, split, sub, seq[, piece]), all text but seq and piece. Rows in play that this
|
||
|
|
read did not return get `gone_batch`, unless the batch is suspect: no window rows where the previous batch had
|
||
|
|
some, or more than 5 tracked keys missing and more than 10 % of them.
|
||
|
|
|
||
|
|
The apply reads back from here (`rows_for_apply`, `orders`, `labor_hours`), so it works on one consistent snapshot
|
||
|
|
and keeps the last good orders when this run's order read failed.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import hashlib
|
||
|
|
from collections.abc import Collection, Iterable, Mapping
|
||
|
|
from dataclasses import dataclass, field
|
||
|
|
from datetime import date
|
||
|
|
from decimal import Decimal
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
from psycopg.rows import dict_row
|
||
|
|
from psycopg.types.json import Jsonb
|
||
|
|
|
||
|
|
from .fingerprint import canonical, norm_decimal
|
||
|
|
from .materials import material_from_row
|
||
|
|
from .types import FeedData, Material, Order, Row, WoKey, num
|
||
|
|
|
||
|
|
ROW_COLS = (
|
||
|
|
"wo_base",
|
||
|
|
"lot",
|
||
|
|
"split",
|
||
|
|
"sub",
|
||
|
|
"seq",
|
||
|
|
"res",
|
||
|
|
"dwg",
|
||
|
|
"rev",
|
||
|
|
"ps",
|
||
|
|
"descr",
|
||
|
|
"qty",
|
||
|
|
"tgt",
|
||
|
|
"rel",
|
||
|
|
"want",
|
||
|
|
"st",
|
||
|
|
"part",
|
||
|
|
"barcode",
|
||
|
|
)
|
||
|
|
MAT_COLS = (
|
||
|
|
"wo_base",
|
||
|
|
"lot",
|
||
|
|
"split",
|
||
|
|
"sub",
|
||
|
|
"seq",
|
||
|
|
"piece",
|
||
|
|
"part",
|
||
|
|
"qty",
|
||
|
|
"issued",
|
||
|
|
"whs",
|
||
|
|
"loc",
|
||
|
|
"st",
|
||
|
|
"istat",
|
||
|
|
"dwg",
|
||
|
|
"descr",
|
||
|
|
"comm",
|
||
|
|
"cons",
|
||
|
|
)
|
||
|
|
SUSPECT_MISSING_MIN = 5
|
||
|
|
SUSPECT_MISSING_SHARE = Decimal("0.10")
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class Batch:
|
||
|
|
id: int
|
||
|
|
suspect: bool
|
||
|
|
counts: dict[str, Any] = field(default_factory=dict)
|
||
|
|
reason: str = ""
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------- row dicts and hashes
|
||
|
|
|
||
|
|
|
||
|
|
def _d(v: Any) -> str | None:
|
||
|
|
return v.isoformat() if isinstance(v, date) else None
|
||
|
|
|
||
|
|
|
||
|
|
def row_doc(r: Row) -> dict[str, Any]:
|
||
|
|
"""The row as a change-row dict: text values, numbers as text, dates ISO."""
|
||
|
|
return {
|
||
|
|
"wo": r.wo,
|
||
|
|
"lot": r.lot,
|
||
|
|
"split": r.split,
|
||
|
|
"sub": r.sub,
|
||
|
|
"seq": r.seq,
|
||
|
|
"res": r.res,
|
||
|
|
"dwg": r.dwg,
|
||
|
|
"rev": r.rev,
|
||
|
|
"ps": r.ps,
|
||
|
|
"desc": r.desc,
|
||
|
|
"qty": norm_decimal(r.qty),
|
||
|
|
"tgt": norm_decimal(r.tgt),
|
||
|
|
"rel": _d(r.rel),
|
||
|
|
"want": _d(r.want),
|
||
|
|
"st": r.st,
|
||
|
|
"part": r.part,
|
||
|
|
"barcode": r.barcode,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def mat_doc(m: Material) -> dict[str, Any]:
|
||
|
|
return {
|
||
|
|
"wo": m.wo,
|
||
|
|
"lot": m.lot,
|
||
|
|
"split": m.split,
|
||
|
|
"sub": m.sub,
|
||
|
|
"seq": m.seq,
|
||
|
|
"piece": m.piece,
|
||
|
|
"part": m.part,
|
||
|
|
"qty": norm_decimal(m.qty),
|
||
|
|
"issued": norm_decimal(m.issued),
|
||
|
|
"whs": m.whs,
|
||
|
|
"loc": m.loc,
|
||
|
|
"st": m.st,
|
||
|
|
"istat": m.istat,
|
||
|
|
"dwg": m.dwg,
|
||
|
|
"desc": m.desc,
|
||
|
|
"comm": m.comm,
|
||
|
|
"cons": m.cons,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def doc_hash(doc: Mapping[str, Any]) -> bytes:
|
||
|
|
return hashlib.sha256(canonical(dict(doc))).digest()
|
||
|
|
|
||
|
|
|
||
|
|
# SQL that rebuilds the same dict from a cached row, for the change rows' `before`.
|
||
|
|
_ROW_BEFORE = (
|
||
|
|
"jsonb_build_object('wo', v.wo_base, 'lot', v.lot, 'split', v.split, 'sub', v.sub, 'seq', v.seq, 'res', v.res,"
|
||
|
|
" 'dwg', v.dwg, 'rev', v.rev, 'ps', v.ps, 'desc', v.descr, 'qty', v.qty::text, 'tgt', v.tgt::text,"
|
||
|
|
" 'rel', v.rel, 'want', v.want, 'st', v.st, 'part', v.part, 'barcode', v.barcode)"
|
||
|
|
)
|
||
|
|
_MAT_BEFORE = (
|
||
|
|
"jsonb_build_object('wo', v.wo_base, 'lot', v.lot, 'split', v.split, 'sub', v.sub, 'seq', v.seq,"
|
||
|
|
" 'piece', v.piece, 'part', v.part, 'qty', v.qty::text, 'issued', v.issued::text, 'whs', v.whs, 'loc', v.loc,"
|
||
|
|
" 'st', v.st, 'istat', v.istat, 'dwg', v.dwg, 'desc', v.descr, 'comm', v.comm, 'cons', v.cons)"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _key_arrays(keys: Iterable[WoKey]) -> tuple[list[str], list[str], list[str], list[str]]:
|
||
|
|
ks = list(dict.fromkeys(keys))
|
||
|
|
return [k.base for k in ks], [k.lot for k in ks], [k.split for k in ks], [k.sub for k in ks]
|
||
|
|
|
||
|
|
|
||
|
|
_KEYS_UNNEST = "unnest(%s::text[], %s::text[], %s::text[], %s::text[]) AS k(b, l, sp, su)"
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------- write
|
||
|
|
|
||
|
|
|
||
|
|
def is_suspect(window_rows: int, prev_window_rows: int | None, missing: int, tracked: int) -> str:
|
||
|
|
"""The reason a batch is suspect, or ''."""
|
||
|
|
if window_rows == 0 and (prev_window_rows or 0) > 0:
|
||
|
|
return f"the window read returned nothing (the previous read had {prev_window_rows})"
|
||
|
|
if missing > SUSPECT_MISSING_MIN and Decimal(missing) > SUSPECT_MISSING_SHARE * tracked:
|
||
|
|
return f"{missing} of {tracked} tracked work orders missing"
|
||
|
|
return ""
|
||
|
|
|
||
|
|
|
||
|
|
def write_batch(conn: Any, run_id: int | None, data: FeedData, source_desc: str, window: tuple[date, date]) -> Batch:
|
||
|
|
"""Record one read: the batch row, rows, materials, orders, labor and the change rows. One transaction."""
|
||
|
|
since, until = window
|
||
|
|
with conn.transaction():
|
||
|
|
prev = conn.execute("SELECT window_rows FROM visual_batches ORDER BY id DESC LIMIT 1").fetchone()
|
||
|
|
missing = len(set(data.tracked_missing))
|
||
|
|
tracked_n = len(set(data.tracked)) or data.tracked_count
|
||
|
|
reason = is_suspect(data.window_count, prev[0] if prev else None, missing, tracked_n)
|
||
|
|
suspect = bool(reason)
|
||
|
|
server, _, database = source_desc.partition(" / ")
|
||
|
|
bid = int(
|
||
|
|
conn.execute(
|
||
|
|
"INSERT INTO visual_batches (run_id, server, database, site, since, until, window_rows, tracked_keys,"
|
||
|
|
" missing_keys, suspect) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id",
|
||
|
|
(
|
||
|
|
run_id,
|
||
|
|
str(data.meta.get("server") or server),
|
||
|
|
str(data.meta.get("database") or database) or None,
|
||
|
|
data.meta.get("site"),
|
||
|
|
since,
|
||
|
|
until,
|
||
|
|
data.window_count,
|
||
|
|
tracked_n,
|
||
|
|
missing,
|
||
|
|
suspect,
|
||
|
|
),
|
||
|
|
).fetchone()[0]
|
||
|
|
)
|
||
|
|
counts: dict[str, Any] = {"rows": len(data.rows), "suspect": int(suspect)}
|
||
|
|
if reason:
|
||
|
|
counts["suspect_reason"] = reason
|
||
|
|
counts.update(_write_rows(conn, bid, data, window, suspect))
|
||
|
|
if data.materials is not None:
|
||
|
|
counts.update(_write_materials(conn, bid, data, suspect))
|
||
|
|
counts["materials"] = len(data.materials)
|
||
|
|
if data.orders_by_base is not None and data.orders_by_project is not None:
|
||
|
|
counts["orders"] = _write_orders(conn, bid, data)
|
||
|
|
if data.labor is not None:
|
||
|
|
counts["labor_legs"] = _write_labor(conn, data)
|
||
|
|
counts["labor_read"] = 1
|
||
|
|
conn.execute("UPDATE visual_batches SET counts = %s WHERE id = %s", (Jsonb(counts), bid))
|
||
|
|
return Batch(id=bid, suspect=suspect, counts=counts, reason=reason)
|
||
|
|
|
||
|
|
|
||
|
|
def _write_rows(conn: Any, bid: int, data: FeedData, window: tuple[date, date], suspect: bool) -> dict[str, int]:
|
||
|
|
uniq: dict[tuple[WoKey, int], Row] = {}
|
||
|
|
for r in data.rows:
|
||
|
|
uniq[(r.key, r.seq)] = r
|
||
|
|
conn.execute(
|
||
|
|
"CREATE TEMP TABLE t_rows (wo_base text, lot text, split text, sub text, seq int, res text, dwg text,"
|
||
|
|
" rev text, ps text, descr text, qty numeric, tgt numeric, rel date, want date, st text, part text,"
|
||
|
|
" barcode text, row_hash bytea, doc jsonb) ON COMMIT DROP"
|
||
|
|
)
|
||
|
|
with (
|
||
|
|
conn.cursor() as cur,
|
||
|
|
cur.copy(
|
||
|
|
"COPY t_rows (wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st, part,"
|
||
|
|
" barcode, row_hash, doc) FROM STDIN"
|
||
|
|
) as cp,
|
||
|
|
):
|
||
|
|
for r in uniq.values():
|
||
|
|
doc = row_doc(r)
|
||
|
|
cp.write_row(
|
||
|
|
(
|
||
|
|
r.wo,
|
||
|
|
r.lot,
|
||
|
|
r.split,
|
||
|
|
r.sub,
|
||
|
|
r.seq,
|
||
|
|
r.res,
|
||
|
|
r.dwg,
|
||
|
|
r.rev,
|
||
|
|
r.ps,
|
||
|
|
r.desc,
|
||
|
|
r.qty,
|
||
|
|
r.tgt,
|
||
|
|
r.rel,
|
||
|
|
r.want,
|
||
|
|
r.st,
|
||
|
|
r.part,
|
||
|
|
r.barcode or None,
|
||
|
|
doc_hash(doc),
|
||
|
|
Jsonb(doc),
|
||
|
|
)
|
||
|
|
)
|
||
|
|
out: dict[str, int] = {}
|
||
|
|
join = (
|
||
|
|
"visual_rows v ON v.wo_base = t.wo_base AND v.lot = t.lot AND v.split = t.split AND v.sub = t.sub"
|
||
|
|
" AND v.seq = t.seq"
|
||
|
|
)
|
||
|
|
out["rows_added"] = conn.execute(
|
||
|
|
"INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, after)" # noqa: S608
|
||
|
|
f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'added', t.doc FROM t_rows t LEFT JOIN {join}"
|
||
|
|
" WHERE v.id IS NULL",
|
||
|
|
(bid,),
|
||
|
|
).rowcount
|
||
|
|
out["rows_returned"] = conn.execute(
|
||
|
|
"INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before, after)" # noqa: S608
|
||
|
|
f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'returned', {_ROW_BEFORE}, t.doc FROM t_rows t"
|
||
|
|
f" JOIN {join} WHERE v.gone_batch IS NOT NULL",
|
||
|
|
(bid,),
|
||
|
|
).rowcount
|
||
|
|
out["rows_changed"] = conn.execute(
|
||
|
|
"INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before, after)" # noqa: S608
|
||
|
|
f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'changed', {_ROW_BEFORE}, t.doc FROM t_rows t"
|
||
|
|
f" JOIN {join} WHERE v.gone_batch IS NULL AND v.row_hash <> t.row_hash",
|
||
|
|
(bid,),
|
||
|
|
).rowcount
|
||
|
|
conn.execute(
|
||
|
|
"INSERT INTO visual_rows (wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st,"
|
||
|
|
" part, barcode, row_hash, first_batch, last_batch)"
|
||
|
|
" SELECT wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st, part, barcode,"
|
||
|
|
" row_hash, %s, %s FROM t_rows"
|
||
|
|
" ON CONFLICT (wo_base, lot, split, sub, seq) DO UPDATE SET res = EXCLUDED.res, dwg = EXCLUDED.dwg,"
|
||
|
|
" rev = EXCLUDED.rev, ps = EXCLUDED.ps, descr = EXCLUDED.descr, qty = EXCLUDED.qty, tgt = EXCLUDED.tgt,"
|
||
|
|
" rel = EXCLUDED.rel, want = EXCLUDED.want, st = EXCLUDED.st, part = EXCLUDED.part,"
|
||
|
|
" barcode = EXCLUDED.barcode, row_hash = EXCLUDED.row_hash, last_batch = EXCLUDED.last_batch,"
|
||
|
|
" gone_batch = NULL",
|
||
|
|
(bid, bid),
|
||
|
|
)
|
||
|
|
out["rows_gone"] = 0
|
||
|
|
if not suspect:
|
||
|
|
b, lot, sp, su = _key_arrays(data.tracked)
|
||
|
|
out["rows_gone"] = conn.execute(
|
||
|
|
"WITH gone AS (UPDATE visual_rows v SET gone_batch = %s" # noqa: S608
|
||
|
|
" WHERE v.gone_batch IS NULL AND v.last_batch <> %s"
|
||
|
|
f" AND (EXISTS (SELECT 1 FROM {_KEYS_UNNEST} WHERE k.b = v.wo_base AND k.l = v.lot AND k.sp = v.split"
|
||
|
|
" AND k.su = v.sub) OR (v.st = 'F' AND v.rel BETWEEN %s AND %s)) RETURNING v.*)"
|
||
|
|
" INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before)"
|
||
|
|
f" SELECT %s, v.wo_base, v.lot, v.split, v.sub, v.seq, 'gone', {_ROW_BEFORE} FROM gone v",
|
||
|
|
(bid, bid, b, lot, sp, su, window[0], window[1], bid),
|
||
|
|
).rowcount
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def _write_materials(conn: Any, bid: int, data: FeedData, suspect: bool) -> dict[str, int]:
|
||
|
|
uniq: dict[tuple[WoKey, int, int], Material] = {}
|
||
|
|
for m in data.materials or []:
|
||
|
|
uniq[(m.key, m.seq, m.piece)] = m
|
||
|
|
conn.execute(
|
||
|
|
"CREATE TEMP TABLE t_mats (wo_base text, lot text, split text, sub text, seq int, piece int, part text,"
|
||
|
|
" qty numeric, issued numeric, whs text, loc text, st text, istat int, dwg text, descr text, comm text,"
|
||
|
|
" cons text, mat_hash bytea, doc jsonb) ON COMMIT DROP"
|
||
|
|
)
|
||
|
|
with (
|
||
|
|
conn.cursor() as cur,
|
||
|
|
cur.copy(
|
||
|
|
"COPY t_mats (wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat, dwg, descr,"
|
||
|
|
" comm, cons, mat_hash, doc) FROM STDIN"
|
||
|
|
) as cp,
|
||
|
|
):
|
||
|
|
for m in uniq.values():
|
||
|
|
doc = mat_doc(m)
|
||
|
|
cp.write_row(
|
||
|
|
(
|
||
|
|
m.wo,
|
||
|
|
m.lot,
|
||
|
|
m.split,
|
||
|
|
m.sub,
|
||
|
|
m.seq,
|
||
|
|
m.piece,
|
||
|
|
m.part,
|
||
|
|
m.qty,
|
||
|
|
m.issued,
|
||
|
|
m.whs,
|
||
|
|
m.loc,
|
||
|
|
m.st,
|
||
|
|
m.istat,
|
||
|
|
m.dwg,
|
||
|
|
m.desc,
|
||
|
|
m.comm,
|
||
|
|
m.cons,
|
||
|
|
doc_hash(doc),
|
||
|
|
Jsonb(doc),
|
||
|
|
)
|
||
|
|
)
|
||
|
|
join = (
|
||
|
|
"visual_materials v ON v.wo_base = t.wo_base AND v.lot = t.lot AND v.split = t.split AND v.sub = t.sub"
|
||
|
|
" AND v.seq = t.seq AND v.piece = t.piece"
|
||
|
|
)
|
||
|
|
cols = "batch_id, wo_base, lot, split, sub, seq, piece, kind"
|
||
|
|
out: dict[str, int] = {}
|
||
|
|
out["mats_added"] = conn.execute(
|
||
|
|
f"INSERT INTO visual_material_changes ({cols}, after)" # noqa: S608
|
||
|
|
f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'added', t.doc FROM t_mats t LEFT JOIN {join}"
|
||
|
|
" WHERE v.id IS NULL",
|
||
|
|
(bid,),
|
||
|
|
).rowcount
|
||
|
|
out["mats_returned"] = conn.execute(
|
||
|
|
f"INSERT INTO visual_material_changes ({cols}, before, after)" # noqa: S608
|
||
|
|
f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'returned', {_MAT_BEFORE}, t.doc"
|
||
|
|
f" FROM t_mats t JOIN {join} WHERE v.gone_batch IS NOT NULL",
|
||
|
|
(bid,),
|
||
|
|
).rowcount
|
||
|
|
out["mats_changed"] = conn.execute(
|
||
|
|
f"INSERT INTO visual_material_changes ({cols}, before, after)" # noqa: S608
|
||
|
|
f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'changed', {_MAT_BEFORE}, t.doc"
|
||
|
|
f" FROM t_mats t JOIN {join} WHERE v.gone_batch IS NULL AND v.mat_hash <> t.mat_hash",
|
||
|
|
(bid,),
|
||
|
|
).rowcount
|
||
|
|
conn.execute(
|
||
|
|
"INSERT INTO visual_materials (wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat,"
|
||
|
|
" dwg, descr, comm, cons, mat_hash, first_batch, last_batch)"
|
||
|
|
" SELECT wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat, dwg, descr, comm,"
|
||
|
|
" cons, mat_hash, %s, %s FROM t_mats"
|
||
|
|
" ON CONFLICT (wo_base, lot, split, sub, seq, piece) DO UPDATE SET part = EXCLUDED.part, qty = EXCLUDED.qty,"
|
||
|
|
" issued = EXCLUDED.issued, whs = EXCLUDED.whs, loc = EXCLUDED.loc, st = EXCLUDED.st,"
|
||
|
|
" istat = EXCLUDED.istat, dwg = EXCLUDED.dwg, descr = EXCLUDED.descr, comm = EXCLUDED.comm,"
|
||
|
|
" cons = EXCLUDED.cons, mat_hash = EXCLUDED.mat_hash, last_batch = EXCLUDED.last_batch, gone_batch = NULL",
|
||
|
|
(bid, bid),
|
||
|
|
)
|
||
|
|
out["mats_gone"] = 0
|
||
|
|
if not suspect:
|
||
|
|
bases = sorted({r.wo for r in data.rows} | {m.wo for m in data.materials or []})
|
||
|
|
out["mats_gone"] = conn.execute(
|
||
|
|
"WITH gone AS (UPDATE visual_materials v SET gone_batch = %s WHERE v.gone_batch IS NULL" # noqa: S608
|
||
|
|
" AND v.last_batch <> %s AND v.wo_base = ANY(%s) RETURNING v.*)"
|
||
|
|
f" INSERT INTO visual_material_changes ({cols}, before)"
|
||
|
|
f" SELECT %s, v.wo_base, v.lot, v.split, v.sub, v.seq, v.piece, 'gone', {_MAT_BEFORE} FROM gone v",
|
||
|
|
(bid, bid, bases, bid),
|
||
|
|
).rowcount
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def _write_orders(conn: Any, bid: int, data: FeedData) -> int:
|
||
|
|
by_base = data.orders_by_base or {}
|
||
|
|
by_project = data.orders_by_project or {}
|
||
|
|
every: dict[str, Order] = {}
|
||
|
|
for lists in (by_base, by_project):
|
||
|
|
for orders in lists.values():
|
||
|
|
for o in orders:
|
||
|
|
every.setdefault(o.cust_order_id, o)
|
||
|
|
for o in every.values():
|
||
|
|
conn.execute(
|
||
|
|
"INSERT INTO visual_orders (cust_order_id, status, customer_id, customer_name, customer_po_ref,"
|
||
|
|
" order_date, desired_ship_date, promise_date, last_batch) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)"
|
||
|
|
" ON CONFLICT (cust_order_id) DO UPDATE SET status = EXCLUDED.status, customer_id = EXCLUDED.customer_id,"
|
||
|
|
" customer_name = EXCLUDED.customer_name, customer_po_ref = EXCLUDED.customer_po_ref,"
|
||
|
|
" order_date = coalesce(EXCLUDED.order_date, visual_orders.order_date),"
|
||
|
|
" desired_ship_date = EXCLUDED.desired_ship_date, promise_date = EXCLUDED.promise_date,"
|
||
|
|
" last_batch = EXCLUDED.last_batch",
|
||
|
|
(
|
||
|
|
o.cust_order_id,
|
||
|
|
o.status,
|
||
|
|
o.customer_id,
|
||
|
|
o.customer_name,
|
||
|
|
o.customer_po_ref,
|
||
|
|
o.order_date,
|
||
|
|
o.desired_ship_date,
|
||
|
|
o.promise_date,
|
||
|
|
bid,
|
||
|
|
),
|
||
|
|
)
|
||
|
|
bases = sorted({r.wo for r in data.rows} | set(by_base))
|
||
|
|
projects = sorted({r.wo.partition("-")[0] or r.wo for r in data.rows} | set(by_project))
|
||
|
|
conn.execute(
|
||
|
|
"DELETE FROM visual_order_links WHERE (kind = 'base' AND key = ANY(%s))"
|
||
|
|
" OR (kind = 'project' AND key = ANY(%s))",
|
||
|
|
(bases, projects),
|
||
|
|
)
|
||
|
|
n = 0
|
||
|
|
for kind, lists in (("base", by_base), ("project", by_project)):
|
||
|
|
for key, orders in lists.items():
|
||
|
|
seen: set[str] = set()
|
||
|
|
for rank, o in enumerate(orders):
|
||
|
|
if o.cust_order_id in seen:
|
||
|
|
continue
|
||
|
|
seen.add(o.cust_order_id)
|
||
|
|
conn.execute(
|
||
|
|
"INSERT INTO visual_order_links (kind, key, cust_order_id, rank, lines, top_jobs, last_batch)"
|
||
|
|
" VALUES (%s, %s, %s, %s, %s, %s, %s)",
|
||
|
|
(kind, key, o.cust_order_id, rank, o.lines, o.top_jobs, bid),
|
||
|
|
)
|
||
|
|
n += 1
|
||
|
|
return len(every)
|
||
|
|
|
||
|
|
|
||
|
|
def _write_labor(conn: Any, data: FeedData) -> int:
|
||
|
|
n = 0
|
||
|
|
for lg in data.labor or []:
|
||
|
|
conn.execute(
|
||
|
|
"INSERT INTO visual_labor (wo_base, lot, split, sub, seq, hours, tickets, last_ticket, observed_at)"
|
||
|
|
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now())"
|
||
|
|
" ON CONFLICT (wo_base, lot, split, sub, seq) DO UPDATE SET hours = EXCLUDED.hours,"
|
||
|
|
" tickets = EXCLUDED.tickets, last_ticket = EXCLUDED.last_ticket, observed_at = now()",
|
||
|
|
(lg.wo, lg.lot, lg.split, lg.sub, lg.seq, lg.hours, lg.tickets, lg.last_ticket),
|
||
|
|
)
|
||
|
|
n += 1
|
||
|
|
return n
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------- read back
|
||
|
|
|
||
|
|
|
||
|
|
def row_from_cache(r: Mapping[str, Any]) -> Row:
|
||
|
|
return Row(
|
||
|
|
wo=r["wo_base"],
|
||
|
|
lot=r["lot"],
|
||
|
|
split=r["split"],
|
||
|
|
sub=r["sub"],
|
||
|
|
seq=int(r["seq"]),
|
||
|
|
res=r["res"],
|
||
|
|
dwg=r["dwg"],
|
||
|
|
rev=r["rev"],
|
||
|
|
ps=r["ps"],
|
||
|
|
desc=r["descr"],
|
||
|
|
qty=num(r["qty"]),
|
||
|
|
tgt=num(r["tgt"]),
|
||
|
|
rel=r["rel"],
|
||
|
|
want=r["want"],
|
||
|
|
st=r["st"],
|
||
|
|
part=r["part"],
|
||
|
|
barcode=r["barcode"] or "",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def rows_for_apply(conn: Any, keys_in_play: Collection[WoKey]) -> tuple[list[Row], list[Material], set[WoKey]]:
|
||
|
|
"""Live cached rows and material lines of every work order (base and lot) among these keys, so folded
|
||
|
|
operations and planner instructions of the same work order come along. material_legs = legs with live
|
||
|
|
material lines whose status is F or R."""
|
||
|
|
pairs = sorted({(k.base, k.lot) for k in keys_in_play})
|
||
|
|
if not pairs:
|
||
|
|
return [], [], set()
|
||
|
|
bases = [p[0] for p in pairs]
|
||
|
|
lots = [p[1] for p in pairs]
|
||
|
|
sql_join = "JOIN unnest(%s::text[], %s::text[]) AS k(b, l) ON v.wo_base = k.b AND v.lot = k.l"
|
||
|
|
with conn.cursor(row_factory=dict_row) as cur:
|
||
|
|
rows = cur.execute(
|
||
|
|
f"SELECT v.* FROM visual_rows v {sql_join} WHERE v.gone_batch IS NULL" # noqa: S608
|
||
|
|
" ORDER BY v.wo_base, v.lot, v.split, v.sub, v.seq",
|
||
|
|
(bases, lots),
|
||
|
|
).fetchall()
|
||
|
|
mats = cur.execute(
|
||
|
|
f"SELECT v.* FROM visual_materials v {sql_join} WHERE v.gone_batch IS NULL" # noqa: S608
|
||
|
|
" ORDER BY v.wo_base, v.lot, v.split, v.sub, v.seq, v.piece",
|
||
|
|
(bases, lots),
|
||
|
|
).fetchall()
|
||
|
|
materials = [material_from_row(m) for m in mats]
|
||
|
|
legs = {m.key for m in materials if m.st in ("F", "R")}
|
||
|
|
return [row_from_cache(r) for r in rows], materials, legs
|
||
|
|
|
||
|
|
|
||
|
|
def orders(
|
||
|
|
conn: Any, *, bases: Iterable[str] | None = None, projects: Iterable[str] | None = None
|
||
|
|
) -> tuple[dict[str, list[Order]], dict[str, list[Order]]]:
|
||
|
|
"""The cached order links as (by_base, by_project), in rank order. Without filters: every link."""
|
||
|
|
params: list[Any] = []
|
||
|
|
where = ""
|
||
|
|
if bases is not None or projects is not None:
|
||
|
|
where = " WHERE (l.kind = 'base' AND l.key = ANY(%s)) OR (l.kind = 'project' AND l.key = ANY(%s))"
|
||
|
|
params = [sorted(set(bases or ())), sorted(set(projects or ()))]
|
||
|
|
with conn.cursor(row_factory=dict_row) as cur:
|
||
|
|
rows = cur.execute(
|
||
|
|
"SELECT l.kind, l.key, l.rank, l.lines, l.top_jobs, o.* FROM visual_order_links l" # noqa: S608
|
||
|
|
f" JOIN visual_orders o ON o.cust_order_id = l.cust_order_id{where} ORDER BY l.kind, l.key, l.rank",
|
||
|
|
params,
|
||
|
|
).fetchall()
|
||
|
|
by_base: dict[str, list[Order]] = {}
|
||
|
|
by_project: dict[str, list[Order]] = {}
|
||
|
|
for r in rows:
|
||
|
|
o = Order(
|
||
|
|
cust_order_id=r["cust_order_id"],
|
||
|
|
status=r["status"],
|
||
|
|
customer_id=r["customer_id"],
|
||
|
|
customer_name=r["customer_name"],
|
||
|
|
customer_po_ref=r["customer_po_ref"],
|
||
|
|
order_date=r["order_date"],
|
||
|
|
desired_ship_date=r["desired_ship_date"],
|
||
|
|
promise_date=r["promise_date"],
|
||
|
|
lines=list(r["lines"] or []),
|
||
|
|
top_jobs=list(r["top_jobs"] or []),
|
||
|
|
)
|
||
|
|
(by_base if r["kind"] == "base" else by_project).setdefault(r["key"], []).append(o)
|
||
|
|
return by_base, by_project
|
||
|
|
|
||
|
|
|
||
|
|
def labor_hours(conn: Any) -> dict[tuple[WoKey, int], Decimal]:
|
||
|
|
"""Cached ticket hours per (leg, seq)."""
|
||
|
|
out: dict[tuple[WoKey, int], Decimal] = {}
|
||
|
|
for r in conn.execute("SELECT wo_base, lot, split, sub, seq, hours FROM visual_labor"):
|
||
|
|
out[(WoKey(r[0], r[1], r[2], r[3]), int(r[4]))] = num(r[5])
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def last_labor_read(conn: Any) -> Any:
|
||
|
|
"""When labor was last read (a batch that carried labor), or None."""
|
||
|
|
r = conn.execute("SELECT max(started_at) FROM visual_batches WHERE counts ? 'labor_read'").fetchone()
|
||
|
|
return r[0] if r else None
|
||
|
|
|
||
|
|
|
||
|
|
def finish_batch(conn: Any, batch_id: int, counts: Mapping[str, Any]) -> None:
|
||
|
|
"""Merge the apply's counts into the batch row and set finished_at."""
|
||
|
|
conn.execute(
|
||
|
|
"UPDATE visual_batches SET finished_at = now(), counts = counts || %s WHERE id = %s",
|
||
|
|
(Jsonb(dict(counts)), batch_id),
|
||
|
|
)
|