"""The VISUAL feed end to end on PostgreSQL: the generator's fixture files, the FixtureSource, the cache, the candidates and the apply, through the visual-feed job as the worker runs it.""" from __future__ import annotations import json from datetime import date, timedelta from decimal import Decimal from pathlib import Path from typing import Any import pytest from feedhelpers import ( CHANGE_KEYS, cos_of, feed_settings_over, gen, legs_of, move_status, packet_by_key, packet_id, run_drawings, run_feed, set_feed, switch_state, system_audit, ) from pshelpers import app_conn, needs_pg from psycopg.rows import dict_row from packetstation.feed import fingerprint from packetstation.settings import Settings pytestmark = needs_pg TODAY = date.today() @pytest.fixture def fx(pg_settings: Settings, tmp_path: Path) -> Settings: s = feed_settings_over(pg_settings, tmp_path) set_feed(s, auto_release=False) # most tests look at drafts; auto-release has its own test return s def row(conn: Any, sql: str, params: Any = None) -> dict[str, Any]: with conn.cursor(row_factory=dict_row) as cur: r = cur.execute(sql, params).fetchone() assert r is not None, sql return dict(r) def no_human_writes(conn: Any) -> None: """Every audit row the feed wrote has role_snapshot 'system' and no user.""" bad = conn.execute( "SELECT action, role_snapshot FROM audit_events WHERE user_id IS NULL" " AND (action LIKE 'packet.%' OR action LIKE 'feed.%' OR action LIKE 'project.%' OR action LIKE 'unit.%'" " OR action LIKE 'drawing.%') AND role_snapshot IS DISTINCT FROM 'system'" ).fetchall() assert bad == [] logs = conn.execute("SELECT DISTINCT by_name, role FROM packet_log WHERE by IS NULL ORDER BY 1").fetchall() assert all(r[1] == "system" for r in logs), logs def window_firmed_wos(state: int = 1) -> int: """Firmed work order legs released inside the default window (what visual_batches.window_rows counts).""" since, until = TODAY - timedelta(days=14), TODAY + timedelta(days=30) return len( { (r["wo"], r["lot"], r["split"], r["sub"]) for r in gen.feed_state(state, TODAY)["rows"] if r["st"] == "Firmed" and r["rel"] and since <= date.fromisoformat(r["rel"]) <= until } ) def window_firmed_rows(state: int = 1) -> int: since, until = TODAY - timedelta(days=14), TODAY + timedelta(days=30) return sum( 1 for r in gen.feed_state(state, TODAY)["rows"] if r["st"] == "Firmed" and r["rel"] and since <= date.fromisoformat(r["rel"]) <= until ) # ---------------------------------------------------------------- state 1 def test_state_1_creates_projects_units_and_drafts(fx: Settings) -> None: c = run_feed(fx, 1) assert c["_status"] == "ok", c assert c["new_drafts"] == len(gen.STATE1_KEYS) assert c["projects_created"] == 2 and c["units_created"] == 3 assert c["unmapped_resources"] >= 1 and c["unmapped"] == {"NEWRES": 2} assert c["suspect"] == 0 and c["rows"] >= window_firmed_rows() with app_conn(fx) as conn: keys = {r[0] for r in conn.execute("SELECT visual_key FROM packets WHERE removed_at IS NULL").fetchall()} assert keys == set(gen.STATE1_KEYS) p = row(conn, "SELECT * FROM projects WHERE number = 'L900'") assert p["source"] == "visual" and p["needs_details"] is True and p["name"] is None assert p["created_by"] is None assert (p["customer"], p["customer_src"], p["customer_visual"]) == ("Acme Refining", "visual", "Acme Refining") assert p["po"] == "PO-1001" and p["order_ref"]["cust_order_id"] == gen.ORDER_ACME u1 = row( conn, "SELECT u.* FROM units u JOIN projects p ON p.id = u.project_id WHERE p.number = 'L900' AND u.lot = '001'", ) assert u1["customer"] == "Bayou Chemical" and u1["po"] == "PO-2002" and u1["customer_src"] == "visual" assert u1["order_ref"]["cust_order_id"] == gen.ORDER_BAYOU and u1["order_ref"]["via"] == "unit_top_job" assert u1["ship_date"] == TODAY + timedelta(days=70) and u1["ship_date_visual"] == TODAY + timedelta(days=70) u2 = row( conn, "SELECT u.* FROM units u JOIN projects p ON p.id = u.project_id WHERE p.number = 'L900' AND u.lot = '002'", ) assert u2["customer"] == "Acme Refining" and u2["order_ref"]["cust_order_id"] == gen.ORDER_ACME z = row(conn, "SELECT u.* FROM units u JOIN projects p ON p.id = u.project_id WHERE p.number = 'L901'") assert z["customer"] == "Delta Gas" and z["order_ref"]["status"] == "C" assert row(conn, "SELECT order_ref FROM projects WHERE number = 'L901'")["order_ref"]["via"] in ( "project_newest", "unit_top_job", "base", ) aliases = {r[0] for r in conn.execute("SELECT alias FROM project_aliases WHERE kind = 'import'").fetchall()} assert {"L900", "L900001", "L900002", "L901", "L901001"} <= aliases assert conn.execute("SELECT count(*) FROM units WHERE lot = '7A'").fetchone()[0] == 0 wp = packet_by_key(conn, gen.KEY_WELD_PART) assert wp is not None assert (wp["status"], wp["source_kind"], wp["dept_code"], wp["op"]) == ("draft", "visual", "03", "WELD-PART") assert (wp["drawing"], wp["rev"], wp["drawing_state"]) == ("L900-0101", "1", "needed") assert wp["flags"] == ["drawing_needed"] and wp["created_by"] is None assert wp["title"] == "Inlet spool 6 in" and wp["customer"] == "Acme Refining" assert wp["visual_rel"] == TODAY - timedelta(days=3) and wp["visual_want"] == TODAY + timedelta(days=20) assert wp["visual_fp_applied"] and set(wp["visual"]["fp_parts"]) >= {"legs", "qty", "rev", "drawing", "bom"} assert wp["visual_fp_applied"] == fingerprint._full(wp["visual"]["fp_parts"]) assert {f["seq"] for f in wp["visual"]["folded"]} == {30, 50} assert wp["visual"]["partspecs"] == ["Cut to length per sheet 1", "Weld per WPS-11\nBack purge"] assert wp["source"]["kind"] == "visual" legs = legs_of(conn, wp["id"]) assert len(legs) == 1 and legs[0][:5] == ("L900-01", "1", "0", "0", 20) assert (legs[0][5], legs[0][6], legs[0][7]) == (Decimal(2), Decimal(3), "F") wo = row(conn, "SELECT * FROM packet_work_orders WHERE packet_id = %s", (wp["id"],)) assert (wo["dwg"], wo["rev"], wo["op"]) == ("L900-0101", "1", "WELD-PART") bom = conn.execute( "SELECT coalesce(cl_item, item::text), part_id, qty, loc, sub, na, src, vkey FROM bom_lines" " WHERE packet_id = %s ORDER BY sort", (wp["id"],), ).fetchall() assert {b[1] for b in bom} == {"PIPE-6-SCH40", "L900-01-SUB", "WOL-1-6"} assert all(b[6] == "visual" for b in bom) sub = next(b for b in bom if b[1] == "L900-01-SUB") assert sub[5] is True main = next(b for b in bom if b[1] == "PIPE-6-SCH40") assert main[3] == "A-01" and main[7] == "L900-01|1|0|20|1" vs = row(conn, "SELECT * FROM packet_visual_state WHERE packet_id = %s", (wp["id"],)) assert vs["ticket_hours"] == Decimal("1.25") and vs["observed_at"] is not None assert {(lg["seq"], lg["st"]) for lg in vs["legs"]} == {(20, "F")} two = packet_by_key(conn, gen.KEY_TWO_OPS) assert two is not None assert [lg[4] for lg in legs_of(conn, two["id"])] == [10, 20] # two operations, one packet nd = packet_by_key(conn, gen.KEY_NO_DWG) assert nd is not None and nd["drawing_state"] == "none" and "no_drawing" in nd["flags"] assert nd["drawing"] in (None, "") rev_a = packet_by_key(conn, gen.KEY_REV_A) assert rev_a is not None and rev_a["rev"] == "A" items = [ r[0] for r in conn.execute( "SELECT coalesce(cl_item, item::text) FROM bom_lines WHERE packet_id = %s ORDER BY sort", (rev_a["id"],) ).fetchall() ] assert sorted(items) == ["1", "1-1", "2"] log = conn.execute( "SELECT by, by_name, role FROM packet_log WHERE packet_id = %s AND action = 'Packet created from VISUAL'", (wp["id"],), ).fetchall() assert log == [(None, "VISUAL feed", "system")] created = system_audit(conn, "packet.created") assert len(created) == len(gen.STATE1_KEYS) and all(r[0] == "system" and r[1] is None for r in created) assert all(r[0] == "system" for r in system_audit(conn, "project.created")) assert len(system_audit(conn, "project.created")) == 2 assert len(system_audit(conn, "unit.created")) == 3 assert len(system_audit(conn, "feed.run")) == 1 no_human_writes(conn) batch = row(conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT 1") assert batch["suspect"] is False and batch["window_rows"] == window_firmed_wos() assert conn.execute("SELECT count(*) FROM visual_rows WHERE gone_batch IS NULL").fetchone()[0] >= ( window_firmed_rows() ) assert conn.execute("SELECT count(*) FROM visual_orders").fetchone()[0] == 3 assert conn.execute("SELECT count(*) FROM visual_rows WHERE lot = '7A'").fetchone()[0] >= 1 assert conn.execute("SELECT count(*) FROM visual_labor").fetchone()[0] == 2 def test_second_run_changes_nothing(fx: Settings) -> None: run_feed(fx, 1) with app_conn(fx) as conn: before = conn.execute("SELECT id, version FROM packets ORDER BY id").fetchall() logs = conn.execute("SELECT count(*) FROM packet_log").fetchone()[0] c = run_feed(fx, 1) assert c["_status"] == "ok" assert {k: c.get(k, 0) for k in CHANGE_KEYS} == dict.fromkeys(CHANGE_KEYS, 0) with app_conn(fx) as conn: assert conn.execute("SELECT id, version FROM packets ORDER BY id").fetchall() == before assert conn.execute("SELECT count(*) FROM packet_log").fetchone()[0] == logs assert conn.execute("SELECT count(*) FROM visual_row_changes WHERE kind <> 'added'").fetchone()[0] == 0 def test_hand_customer_is_never_overwritten(fx: Settings) -> None: run_feed(fx, 1) with app_conn(fx) as conn: conn.execute( "UPDATE units SET customer = 'Hand Co', customer_src = 'hand' WHERE lot = '001'" " AND project_id = (SELECT id FROM projects WHERE number = 'L900')" ) run_feed(fx, 1) with app_conn(fx) as conn: u = row( conn, "SELECT u.* FROM units u JOIN projects p ON p.id = u.project_id WHERE p.number = 'L900' AND u.lot = '001'", ) assert (u["customer"], u["customer_src"], u["customer_visual"]) == ("Hand Co", "hand", "Bayou Chemical") def test_order_pin_wins(fx: Settings) -> None: run_feed(fx, 1) with app_conn(fx) as conn: conn.execute( "UPDATE units SET order_pin = %s, order_pin_reason = 'Spool is for Acme' WHERE lot = '001'" " AND project_id = (SELECT id FROM projects WHERE number = 'L900')", (gen.ORDER_ACME,), ) run_feed(fx, 1) with app_conn(fx) as conn: u = row( conn, "SELECT u.* FROM units u JOIN projects p ON p.id = u.project_id WHERE p.number = 'L900' AND u.lot = '001'", ) assert u["order_ref"]["cust_order_id"] == gen.ORDER_ACME and u["order_ref"]["via"] == "pin" assert u["customer"] == "Acme Refining" def test_golive_ignores_and_pull_in(fx: Settings) -> None: set_feed(fx, golive_date=TODAY + timedelta(days=gen.GOLIVE_OFFSET)) c = run_feed(fx, 1) assert c["ignored_golive"] == 1 and c["new_drafts"] == len(gen.STATE1_KEYS) - 1 with app_conn(fx) as conn: assert packet_by_key(conn, gen.KEY_GOLIVE, live=False) is None ig = row(conn, "SELECT * FROM visual_ignored_wos") assert (ig["wo_base"], ig["lot"], ig["split"], ig["sub"]) == ("L900-13", "2", "0", "0") assert ig["pulled_in_at"] is None and ig["rel"] == TODAY - timedelta(days=10) conn.execute( "UPDATE visual_ignored_wos SET pulled_in_at = now(), pulled_in_reason = 'needed' WHERE id = %s", (ig["id"],) ) c = run_feed(fx, 1) assert c["new_drafts"] == 1 with app_conn(fx) as conn: assert packet_by_key(conn, gen.KEY_GOLIVE) is not None # ---------------------------------------------------------------- state 2 def _shop(fx: Settings) -> tuple[int, int]: """Run state 1, then put the WELD-PART packet in the shop (ready) and the not-found one at the warehouse.""" run_feed(fx, 1) with app_conn(fx) as conn: wp, g = packet_id(conn, gen.KEY_WELD_PART), packet_id(conn, gen.KEY_NOT_FOUND) move_status(conn, wp, "ready") move_status(conn, g, "warehouse") return wp, g def test_state_2_change_orders_updates_gone_and_moved(fx: Settings) -> None: wp, g = _shop(fx) with app_conn(fx) as conn: fp1 = row(conn, "SELECT visual_fp_applied FROM packets WHERE id = %s", (wp,))["visual_fp_applied"] two_id = packet_id(conn, gen.KEY_TWO_OPS) moves_id = packet_id(conn, gen.KEY_MOVES) c = run_feed(fx, 2) assert c["_status"] == "ok", c assert c["cos_opened"] == 2 and c["drafts_updated"] >= 2 and c["retired"] == 1, c assert c["wo_gone_flagged"] == 0 # the first miss only counts with app_conn(fx) as conn: # rev 1 -> 2 on an in-shop packet: a change order waiting for the rev 2 drawing; the packet stays as it was p = row(conn, "SELECT * FROM packets WHERE id = %s", (wp,)) assert (p["status"], p["rev"], p["visual_fp_applied"]) == ("ready", "1", fp1) cos = cos_of(conn, wp) assert len(cos) == 1 co = cos[0] assert (co["origin"], co["review"], co["minor"], co["ecn"], co["issued_by"]) == ( "visual", "pending", False, "VISUAL", None, ) assert co["target_fp"] and co["target_fp"] != fp1 assert co["vis"]["to_rev"] == "2" and co["vis"]["from_rev"] == "1" and co["vis"]["fp_parts"] assert co["waiting"]["dwg"] == "L900-0101" and co["waiting"]["rev"] == "2" assert co["staged"] is None # tgt only: a minor change order with nothing to wait for gco = cos_of(conn, g) assert len(gco) == 1 and gco[0]["minor"] is True and gco[0]["waiting"] is None # qty change on a draft: updated in place, both operations two = row(conn, "SELECT * FROM packets WHERE id = %s", (two_id,)) assert two["status"] == "draft" and cos_of(conn, two_id) == [] assert [lg[5] for lg in legs_of(conn, two_id)] == [Decimal(3), Decimal(3)] assert ( conn.execute( "SELECT count(*) FROM packet_log WHERE packet_id = %s AND action = 'Updated from VISUAL'", (two_id,) ).fetchone()[0] == 1 ) # the LAYOUT draft shares the drawing: rev moved in place lay = packet_by_key(conn, gen.KEY_LAYOUT_A) assert lay is not None and lay["rev"] == "2" and lay["drawing_state"] == "needed" # retire-moved old = row(conn, "SELECT * FROM packets WHERE id = %s", (moves_id,)) assert old["removed_at"] is not None and old["removed_by"] is None and old["removed_reason"] == "moved" assert old["visual"]["moved"]["to"] assert packet_by_key(conn, gen.KEY_MOVED_TO) is not None # gone: one miss counted, no flag yet e = packet_id(conn, gen.KEY_DRAFTING) assert [lg[8] for lg in legs_of(conn, e)] == [1] assert "wo_gone" not in row(conn, "SELECT flags FROM packets WHERE id = %s", (e,))["flags"] # closed and cancelled observed j = row(conn, "SELECT * FROM packets WHERE visual_key = %s", (gen.KEY_CLOSES,)) assert j["visual_closed_at"] is not None and legs_of(conn, j["id"])[0][7] == "C" k = row(conn, "SELECT * FROM packets WHERE visual_key = %s", (gen.KEY_CANCELS,)) assert "cancelled" in k["flags"] and k["visual_cancelled_at"] is not None and k["removed_at"] is None assert all(r[0] == "system" for r in system_audit(conn, "packet.vis_open_co")) assert all(r[0] == "system" for r in system_audit(conn, "packet.vis_update_draft")) assert len(system_audit(conn, "packet.vis_retire_draft")) == 1 no_human_writes(conn) # the second miss flags the work order gone; nothing else moves c = run_feed(fx, 2) assert c["wo_gone_flagged"] == 1 and c["cos_opened"] == 0 and c["cos_reaimed"] == 0, c assert c["drafts_updated"] == 0 and c["retired"] == 0 and c["new_drafts"] == 0 with app_conn(fx) as conn: e = row(conn, "SELECT * FROM packets WHERE visual_key = %s AND removed_at IS NULL", (gen.KEY_DRAFTING,)) assert "wo_gone" in e["flags"] and e["wo_gone_at"] is not None assert [lg[8:] for lg in legs_of(conn, e["id"])] == [(2, True)] assert len(cos_of(conn, wp)) == 1 # the same CO, not a second one assert ( conn.execute( "SELECT count(*) FROM packet_log WHERE packet_id = %s" " AND action LIKE 'Work order%%no longer in VISUAL'", (e["id"],), ).fetchone()[0] == 1 ) # VISUAL reverts: the pending feed change orders are withdrawn and the gone work order is back c = run_feed(fx, 1) assert c["cos_withdrawn"] == 2, c with app_conn(fx) as conn: co = cos_of(conn, wp)[0] assert co["review"] == "rejected" and co["rejected_by"] is None and co["rejected_reason"] e = row(conn, "SELECT * FROM packets WHERE visual_key = %s AND removed_at IS NULL", (gen.KEY_DRAFTING,)) assert "wo_gone" not in e["flags"] and e["wo_gone_at"] is None assert [lg[8:] for lg in legs_of(conn, e["id"])] == [(0, False)] k = row(conn, "SELECT * FROM packets WHERE visual_key = %s", (gen.KEY_CANCELS,)) assert "cancelled" not in k["flags"] # the retired key is never made again assert ( conn.execute( "SELECT count(*) FROM packets WHERE visual_key = %s AND removed_at IS NULL", (gen.KEY_MOVES,) ).fetchone()[0] == 0 ) assert c["skipped_removed"] >= 1 def test_rejected_target_is_not_reopened(fx: Settings) -> None: wp, _ = _shop(fx) run_feed(fx, 2) with app_conn(fx) as conn: conn.execute( "UPDATE change_orders SET review = 'rejected', rejected_at = now(), rejected_reason = 'not this one'," " row_version = row_version + 1 WHERE packet_id = %s", (wp,), ) c = run_feed(fx, 2) assert c["cos_opened"] == 0, c with app_conn(fx) as conn: assert len(cos_of(conn, wp)) == 1 def test_complete_packet_gets_a_change_order(fx: Settings) -> None: run_feed(fx, 1) with app_conn(fx) as conn: wp = packet_id(conn, gen.KEY_WELD_PART) move_status(conn, wp, "complete") fl = packet_id(conn, gen.KEY_TWO_OPS) move_status(conn, fl, "filed") c = run_feed(fx, 2) with app_conn(fx) as conn: assert len(cos_of(conn, wp)) == 1 assert cos_of(conn, fl) == [] # filed: nothing assert [lg[5] for lg in legs_of(conn, fl)] == [Decimal(2), Decimal(2)] assert c["cos_opened"] >= 1 def test_removed_packet_key_is_never_recreated(fx: Settings) -> None: run_feed(fx, 1) with app_conn(fx) as conn: pid = packet_id(conn, gen.KEY_NO_DWG) conn.execute("UPDATE packets SET removed_at = now(), removed_reason = 'not needed' WHERE id = %s", (pid,)) c = run_feed(fx, 1) assert c["new_drafts"] == 0 and c["skipped_removed"] >= 1 with app_conn(fx) as conn: assert packet_by_key(conn, gen.KEY_NO_DWG) is None def test_closed_unit_is_skipped(fx: Settings) -> None: run_feed(fx, 1) with app_conn(fx) as conn: conn.execute( "UPDATE units SET status = 'closed' WHERE project_id = (SELECT id FROM projects WHERE number = 'L901')" ) c = run_feed(fx, 2) assert c["skipped_closed"] >= 1 # ---------------------------------------------------------------- pause, suspect, auto-release def test_paused_skips_a_scheduled_run_but_a_request_runs(fx: Settings) -> None: set_feed(fx, paused=True) c = run_feed(fx, 1) assert c["_status"] == "skipped" and c["reason"] == "paused" with app_conn(fx) as conn: assert conn.execute("SELECT count(*) FROM packets").fetchone()[0] == 0 c = run_feed(fx, 1, requested=True) assert c["_status"] == "ok" and c["_trigger"] == "request" and c["new_drafts"] == len(gen.STATE1_KEYS) def test_suspect_batch_marks_nothing_gone(fx: Settings) -> None: run_feed(fx, 1) empty = gen.feed_state(1, TODAY) empty.update({"rows": [], "materials": [], "labor": []}) (fx.fixture_dir / "feed.3.json").write_text(json.dumps(empty), encoding="utf-8") c = run_feed(fx, 3) assert c["_status"] == "degraded" and c["suspect"] == 1, c assert c["_errors"] with app_conn(fx) as conn: assert conn.execute("SELECT count(*) FROM visual_rows WHERE gone_batch IS NOT NULL").fetchone()[0] == 0 assert conn.execute("SELECT max(missing_runs) FROM packet_work_orders").fetchone()[0] == 0 assert row(conn, "SELECT suspect FROM visual_batches ORDER BY id DESC LIMIT 1")["suspect"] is True assert conn.execute("SELECT count(*) FROM packets WHERE removed_at IS NOT NULL").fetchone()[0] == 0 # back to normal: nothing was lost c = run_feed(fx, 1) assert c["suspect"] == 0 and c["new_drafts"] == 0 def test_auto_release(fx: Settings) -> None: run_feed(fx, 1) run_drawings(fx) set_feed(fx, auto_release=True, auto_release_max=1) c = run_feed(fx, 1) assert c["auto_released"] == 1, c set_feed(fx, auto_release_max=25) c = run_feed(fx, 1) assert c["auto_released"] >= 1 with app_conn(fx) as conn: wp = packet_by_key(conn, gen.KEY_WELD_PART) assert wp is not None and wp["status"] == "warehouse" and wp["heat_map"] is True wh = row(conn, "SELECT * FROM packet_wh WHERE packet_id = %s", (wp["id"],)) assert wh["stage"] == "incoming" and wh["by"] is None tr = row(conn, "SELECT * FROM packet_transitions WHERE packet_id = %s", (wp["id"],)) assert tr["action"] == "vis_auto_release" and tr["by"] is None # VISUAL calls rev 2, which M-Files holds as Retired: attached without a flag, so it is released too two = packet_by_key(conn, gen.KEY_TWO_OPS) assert two is not None and two["status"] == "warehouse" and two["visual"]["vault"]["controlled"] == "Retired" # flagged drafts stay with Clerical for key in (gen.KEY_NOT_FOUND, gen.KEY_DRAFTING, gen.KEY_NO_DWG): p = packet_by_key(conn, key) assert p is not None and p["status"] == "draft", key assert all(r[0] == "system" for r in system_audit(conn, "packet.vis_auto_release")) no_human_writes(conn) def test_paused_stops_auto_release(fx: Settings) -> None: run_feed(fx, 1) run_drawings(fx) set_feed(fx, auto_release=True, paused=True) c = run_feed(fx, 1, requested=True) assert c.get("auto_released", 0) == 0 with app_conn(fx) as conn: assert conn.execute("SELECT count(*) FROM packets WHERE status = 'warehouse'").fetchone()[0] == 0 def test_switch_state_file(fx: Settings) -> None: switch_state(fx.fixture_dir, 2) assert (fx.fixture_dir / "state").read_text() == "2"