PacketStation/tests/server/test_feed_api.py

"""The Clerical feed API (/api/feed), rev approval, hand uploads and feed change order approval, on the generator's
fixture files run through the real feed and drawing jobs."""

from __future__ import annotations

import io
import json
from collections.abc import Iterator
from pathlib import Path
from typing import Any

import pytest
from fakedir import FAKE_PASSWORD, FakeDirectory
from fastapi.testclient import TestClient
from feedhelpers import (
    cos_of,
    feed_settings_over,
    gen,
    golive_today_offset,
    move_status,
    packet_id,
    run_drawings,
    run_feed,
    set_feed,
)
from pkthelpers import SIGN, seed_users, tiny_pdf, upload
from pshelpers import ORIGIN_HEADERS, app_conn, needs_pg, sign_in

from packetstation import db
from packetstation.settings import Settings

pytestmark = needs_pg


@pytest.fixture
def fx(pg_settings: Settings, tmp_path: Path) -> Settings:
    s = feed_settings_over(pg_settings, tmp_path)
    set_feed(s, auto_release=False)
    return s


@pytest.fixture
def client(fx: Settings, fake_dir: FakeDirectory, dev_visual: None) -> Iterator[TestClient]:
    from packetstation.main import create_app

    with TestClient(create_app(fx, directory=fake_dir), headers=ORIGIN_HEADERS) as c:
        seed_users(c, ("admin.ann", "clerk.clara", "clerk.leo", "wh.wade"))
        yield c


def as_user(c: TestClient, sam: str) -> None:
    assert sign_in(c, sam).status_code == 200, sam


def drawing_pages(conn: Any, pid: int) -> list[tuple[Any, ...]]:
    return [
        tuple(r)
        for r in conn.execute(
            "SELECT n, rev, co_id IS NOT NULL, void_at IS NOT NULL, void_co_id IS NOT NULL FROM packet_pages"
            " WHERE packet_id = %s AND type = 'drawing' ORDER BY n",
            (pid,),
        ).fetchall()
    ]


def packet_row(conn: Any, pid: int) -> dict[str, Any]:
    from psycopg.rows import dict_row

    with conn.cursor(row_factory=dict_row) as cur:
        r = cur.execute("SELECT * FROM packets WHERE id = %s", (pid,)).fetchone()
    assert r is not None
    return dict(r)


# ---------------------------------------------------------------- the From VISUAL panel


def test_overview_shape(client: TestClient, fx: Settings) -> None:
    set_feed(fx, golive_date=golive_today_offset())
    run_feed(fx, 1)
    run_drawings(fx)
    as_user(client, "clerk.clara")
    r = client.get("/api/feed/overview")
    assert r.status_code == 200, r.text
    o = r.json()
    assert set(o) >= {
        "status",
        "last_run",
        "runs",
        "drawing_runs",
        "ignored",
        "unmapped",
        "projects_without_names",
        "golive_date",
        "departments",
    }
    assert o["status"]["paused"] is False
    assert o["status"]["worker"] is None or o["status"]["worker"]["stale"] is True
    assert o["last_run"]["status"] == "ok" and o["last_run"]["counts"]["new_drafts"] == len(gen.STATE1_KEYS) - 1
    assert len(o["runs"]) == 1 and len(o["drawing_runs"]) == 1
    assert o["unmapped"] == [{"resource": "NEWRES", "rows": 2}]
    assert [(i["wo_base"], i["lot"]) for i in o["ignored"]] == [("L900-13", "2")]
    assert {p["number"] for p in o["projects_without_names"]} == {"L900", "L901"}
    l900 = next(p for p in o["projects_without_names"] if p["number"] == "L900")
    assert l900["units"] == 2 and l900["packets"] == len(gen.STATE1_KEYS) - 2
    assert o["golive_date"] == golive_today_offset().isoformat() and o["departments"] == []


def test_feed_api_is_clerical_only(client: TestClient) -> None:
    as_user(client, "wh.wade")
    assert client.get("/api/feed/overview").status_code == 403
    assert client.post("/api/feed/run", json={}).status_code == 403
    assert client.get("/api/feed/batches").status_code == 403


def test_run_now_inserts_a_request_and_notifies(client: TestClient, fx: Settings) -> None:
    listener = db.connect_as("worker", fx, autocommit=True)
    listener.execute("LISTEN ps_job_request")
    try:
        as_user(client, "clerk.clara")
        r = client.post("/api/feed/run", json={})
        assert r.status_code == 200 and r.json()["requested"] is True and r.json()["pending"] == 1, r.text
        got = list(listener.notifies(timeout=3, stop_after=1))
        assert got and json.loads(got[0].payload) == {"job": "visual-feed"}
        r = client.post("/api/feed/run", json={"job": "drawing-lookup"})
        assert r.status_code == 200 and r.json()["job"] == "drawing-lookup"
        assert client.post("/api/feed/run", json={"job": "housekeeping"}).status_code == 422
    finally:
        listener.close()
    with app_conn(fx) as conn:
        rows = conn.execute("SELECT job FROM job_requests WHERE claimed_at IS NULL ORDER BY id").fetchall()
        assert [r[0] for r in rows] == ["visual-feed", "drawing-lookup"]
        assert conn.execute("SELECT count(*) FROM audit_events WHERE action = 'feed.run_requested'").fetchone()[0] == 2
    # the worker's next run claims the request and records the trigger
    c = run_feed(fx, 1)
    assert c["_trigger"] == "request"
    assert client.get("/api/feed/overview").json()["last_run"]["trigger"] == "request"


def test_pull_in(client: TestClient, fx: Settings) -> None:
    set_feed(fx, golive_date=golive_today_offset())
    run_feed(fx, 1)
    as_user(client, "clerk.clara")
    ig = client.get("/api/feed/overview").json()["ignored"][0]
    r = client.post(f"/api/feed/ignored/{ig['id']}/pull-in", json={"reason": ""})
    assert r.status_code == 400 and r.json()["code"] == "reason_required"
    r = client.post(f"/api/feed/ignored/{ig['id']}/pull-in", json={"reason": "Customer still wants it"})
    assert r.status_code == 200 and r.json()["pulled_in_at"], r.text
    r = client.post(f"/api/feed/ignored/{ig['id']}/pull-in", json={"reason": "again please"})
    assert r.status_code == 409
    assert client.post("/api/feed/ignored/999999/pull-in", json={"reason": "nothing"}).status_code == 404
    with app_conn(fx) as conn:
        assert conn.execute("SELECT count(*) FROM job_requests WHERE claimed_at IS NULL").fetchone()[0] == 1
        assert conn.execute("SELECT count(*) FROM audit_events WHERE action = 'feed.pulled_in'").fetchone()[0] == 1
    c = run_feed(fx, 1)
    assert c["new_drafts"] == 1 and c["_trigger"] == "request"
    assert client.get("/api/feed/overview").json()["ignored"] == []


def test_batches_and_changes(client: TestClient, fx: Settings) -> None:
    run_feed(fx, 1)
    run_feed(fx, 2)
    as_user(client, "clerk.clara")
    batches = client.get("/api/feed/batches").json()
    assert len(batches) == 2 and batches[0]["id"] > batches[1]["id"]
    ch = client.get("/api/feed/changes").json()
    assert ch["batch"]["id"] == batches[0]["id"]
    kinds = {(r["wo_base"], r["kind"]) for r in ch["rows"]}
    assert ("L900-01", "changed") in kinds and ("L900-05", "gone") in kinds
    first = client.get(f"/api/feed/changes?batch_id={batches[1]['id']}").json()
    assert all(r["kind"] == "added" for r in first["rows"]) and first["rows"]
    assert client.get("/api/feed/changes?batch_id=999999").status_code == 404


# ---------------------------------------------------------------- rev approval


def test_rev_approve(client: TestClient, fx: Settings) -> None:
    run_feed(fx, 1)
    run_drawings(fx)
    with app_conn(fx) as conn:
        pid = packet_id(conn, gen.KEY_TWO_OPS)  # VISUAL rev 2, M-Files rev 3: staged
        nf = packet_id(conn, gen.KEY_NOT_FOUND)
    body = {"reason": "Engineering confirmed rev 3 is right", "sign": SIGN}
    as_user(client, "clerk.clara")
    r = client.post(f"/api/packets/{pid}/rev-approve", json=body)
    assert r.status_code == 403, r.text
    as_user(client, "clerk.leo")
    r = client.post(f"/api/packets/{pid}/rev-approve", json={"reason": "no signature"})
    assert r.status_code == 400 and r.json()["code"] == "signature_required", r.text
    r = client.post(f"/api/packets/{nf}/rev-approve", json=body)
    assert r.status_code == 409 and r.json()["code"] == "nothing_staged", r.text
    r = client.post(f"/api/packets/{pid}/rev-approve", json=body)
    assert r.status_code == 200, r.text
    with app_conn(fx) as conn:
        p = packet_row(conn, pid)
        assert p["drawing_state"] == "attached" and "rev_mismatch" not in p["flags"]
        assert p["visual"]["rev_ok"]["rev"] == "3" and p["visual"]["rev_ok"]["vis_rev"] == "2"
        assert p["visual"]["rev_confirmed"] is True
        pages = drawing_pages(conn, pid)
        assert pages and all(pg[1] == "3" for pg in pages)
        assert (
            conn.execute(
                "SELECT count(*) FROM packet_log WHERE packet_id = %s AND action = 'Different rev approved by hand'",
                (pid,),
            ).fetchone()[0]
            == 1
        )
        assert (
            conn.execute(
                "SELECT count(*) FROM audit_events WHERE action = 'packet.rev_approve' AND packet_id = %s", (pid,)
            ).fetchone()[0]
            == 1
        )
    # the drawing job leaves the approved rev alone
    with app_conn(fx) as conn:
        conn.execute("UPDATE drawing_lookups SET checked_at = now() - interval '2 days', next_check_at = now()")
    run_drawings(fx)
    with app_conn(fx) as conn:
        p = packet_row(conn, pid)
        assert p["drawing_state"] == "attached" and "rev_mismatch" not in p["flags"]


def test_rev_approve_keeps_the_rev_visual_calls(client: TestClient, fx: Settings) -> None:
    run_feed(fx, 1)
    run_drawings(fx)
    with app_conn(fx) as conn:
        pid = packet_id(conn, gen.KEY_TWO_OPS)  # VISUAL rev 2 (Retired in M-Files), M-Files rev 3
        nf = packet_id(conn, gen.KEY_NOT_FOUND)
    body = {"reason": "Engineering says build lot 1 to rev 2", "sign": SIGN, "use": "called"}
    as_user(client, "clerk.leo")
    r = client.post(f"/api/packets/{nf}/rev-approve", json=body)
    assert r.status_code == 409 and r.json()["code"] == "nothing_staged", r.text
    r = client.post(f"/api/packets/{pid}/rev-approve", json={**body, "use": "oldest"})
    assert r.status_code == 422, r.text
    r = client.post(f"/api/packets/{pid}/rev-approve", json=body)
    assert r.status_code == 200, r.text
    with app_conn(fx) as conn:
        p = packet_row(conn, pid)
        assert p["drawing_state"] == "attached" and "rev_mismatch" not in p["flags"]
        ok = p["visual"]["rev_ok"]
        assert (ok["rev"], ok["vis_rev"], ok["use"], ok["latest"], ok["controlled"]) == (
            "2",
            "2",
            "called",
            "3",
            "Retired",
        )
        assert p["visual"]["rev_confirmed"] is True
        pages = drawing_pages(conn, pid)
        assert pages and all(pg[1] == "2" for pg in pages)
        log = conn.execute(
            "SELECT note FROM packet_log WHERE packet_id = %s AND action = 'Rev VISUAL calls kept by hand'", (pid,)
        ).fetchall()
        assert len(log) == 1 and "Retired in M-Files" in log[0][0] and "M-Files has rev 3" in log[0][0]
    # the drawing job leaves the kept rev alone while M-Files' Effective rev is still 3
    with app_conn(fx) as conn:
        conn.execute("UPDATE drawing_lookups SET checked_at = now() - interval '2 days', next_check_at = now()")
    run_drawings(fx)
    with app_conn(fx) as conn:
        p = packet_row(conn, pid)
        assert p["drawing_state"] == "attached" and "rev_mismatch" not in p["flags"]
        assert p["visual"]["rev_confirmed"] is True and p["visual"].get("called") is None
        assert all(pg[1] == "2" for pg in drawing_pages(conn, pid))


# ---------------------------------------------------------------- hand uploads


def test_hand_drawing(client: TestClient, fx: Settings) -> None:
    run_feed(fx, 1)
    run_drawings(fx)
    with app_conn(fx) as conn:
        nf = packet_id(conn, gen.KEY_NOT_FOUND)
        wp = packet_id(conn, gen.KEY_WELD_PART)
    as_user(client, "clerk.clara")
    a = upload(client, tiny_pdf(2), "L900-0701 hand.pdf")
    assert a.status_code == 201, a.text
    asset = a.json()["id"]
    r = client.post(f"/api/packets/{wp}/hand/drawing", json={"asset_id": asset, "reason": "Have it", "sign": SIGN})
    assert r.status_code == 409 and r.json()["code"] == "not_flagged", r.text
    r = client.post(f"/api/packets/{nf}/hand/drawing", json={"asset_id": asset, "reason": "From drafting"})
    assert r.status_code == 400 and r.json()["code"] == "signature_required", r.text
    r = client.post(
        f"/api/packets/{nf}/hand/drawing", json={"asset_id": asset, "reason": "From drafting by email", "sign": SIGN}
    )
    assert r.status_code == 200, r.text
    with app_conn(fx) as conn:
        p = packet_row(conn, nf)
        assert p["drawing_state"] == "hand" and "hand_drawing" in p["flags"] and "not_found" not in p["flags"]
        assert p["visual"]["manual_dwg"]["reason"] == "From drafting by email"
        assert len(drawing_pages(conn, nf)) == 2
        assert (
            conn.execute(
                "SELECT count(*) FROM packet_log WHERE packet_id = %s AND action = 'Drawing uploaded by hand'", (nf,)
            ).fetchone()[0]
            == 1
        )
    # a hand flag keeps it off auto-release, and the drawing job does not take it away
    run_drawings(fx)
    with app_conn(fx) as conn:
        assert packet_row(conn, nf)["drawing_state"] == "hand"


def _xlsx(rows: list[list[Any]]) -> bytes:
    from openpyxl import Workbook

    wb = Workbook()
    ws = wb.active
    assert ws is not None
    for r in rows:
        ws.append(r)
    buf = io.BytesIO()
    wb.save(buf)
    return buf.getvalue()


def _hand_bom(client: TestClient, pid: int, data: bytes, name: str, **form: str) -> Any:
    fields = {"reason": "Engineering sent the BOM", "sign_factor": "password", "sign_secret": FAKE_PASSWORD}
    fields.update(form)
    return client.post(
        f"/api/packets/{pid}/hand/bom", files={"file": (name, data, "application/octet-stream")}, data=fields
    )


@pytest.mark.parametrize("kind", ["xlsx", "csv"])
def test_hand_bom(client: TestClient, fx: Settings, kind: str) -> None:
    run_feed(fx, 1)
    run_drawings(fx)
    with app_conn(fx) as conn:
        mm = packet_id(conn, gen.KEY_TWO_OPS)  # rev mismatch: flagged
        wp = packet_id(conn, gen.KEY_WELD_PART)  # attached, no flags
        visual_lines = conn.execute("SELECT count(*) FROM bom_lines WHERE packet_id = %s", (mm,)).fetchone()[0]
    if kind == "xlsx":
        data, name = (
            _xlsx(
                [
                    ["Item", "Qty", "Part ID", "Description"],
                    [1, 4, "PLT-0.5-A36", "Plate"],
                    [2, 2, "CPL-2-3000", "Coupling"],
                    [3, 1, None, "Gusset"],
                ]
            ),
            "bom.xlsx",
        )
    else:
        data, name = (
            b"Item,Qty,Part ID,Description\r\n1,4,PLT-0.5-A36,Plate\r\n2,2,CPL-2-3000,Coupling\r\n3,1,,Gusset\r\n",
            "bom.csv",
        )
    as_user(client, "clerk.clara")
    r = _hand_bom(client, wp, data, name)
    assert r.status_code == 409 and r.json()["code"] == "not_flagged", r.text
    r = _hand_bom(client, mm, data, name, sign_secret="wrong password")
    assert r.status_code in (400, 401, 403), r.text
    r = _hand_bom(client, mm, b"Name,Colour\nPipe,Red\n", "bad.csv")
    assert r.status_code == 422 and r.json()["code"] == "bad_file", r.text
    r = _hand_bom(client, mm, data, name)
    assert r.status_code == 200, r.text
    with app_conn(fx) as conn:
        lines = conn.execute(
            "SELECT item, qty, part_id, description, src FROM bom_lines WHERE packet_id = %s ORDER BY item", (mm,)
        ).fetchall()
        assert [(ln[0], ln[2], ln[4]) for ln in lines] == [
            (1, "PLT-0.5-A36", "hand"),
            (2, "CPL-2-3000", "hand"),
            (3, None, "hand"),
        ]
        p = packet_row(conn, mm)
        assert "hand_bom" in p["flags"] and p["visual"]["manual_bom"]["lines"] == 3
        assert (
            conn.execute(
                "SELECT count(*) FROM packet_log WHERE packet_id = %s AND action = 'BOM uploaded by hand'", (mm,)
            ).fetchone()[0]
            == 1
        )
    # the feed keeps a hand BOM on a draft
    run_feed(fx, 2)
    with app_conn(fx) as conn:
        assert (
            conn.execute("SELECT count(*) FROM bom_lines WHERE packet_id = %s AND src = 'hand'", (mm,)).fetchone()[0]
            == 3
        )
    # back to VISUAL's BOM
    r = client.request("DELETE", f"/api/packets/{mm}/hand/bom", json={"sign": SIGN})
    assert r.status_code == 200, r.text
    with app_conn(fx) as conn:
        src = conn.execute("SELECT DISTINCT src FROM bom_lines WHERE packet_id = %s", (mm,)).fetchall()
        assert src == [("visual",)]
        assert conn.execute("SELECT count(*) FROM bom_lines WHERE packet_id = %s", (mm,)).fetchone()[0] == visual_lines
        p = packet_row(conn, mm)
        assert "hand_bom" not in p["flags"] and "manual_bom" not in p["visual"]


# ---------------------------------------------------------------- feed change orders


def test_co_approve_applies_vis_and_voids_pages(client: TestClient, fx: Settings) -> None:
    run_feed(fx, 1)
    run_drawings(fx)
    with app_conn(fx) as conn:
        wp = packet_id(conn, gen.KEY_WELD_PART)
        assert packet_row(conn, wp)["drawing_state"] == "attached"
        move_status(conn, wp, "ready")
        old_pages = drawing_pages(conn, wp)
        assert old_pages and all(pg[1] == "1" for pg in old_pages)
    run_feed(fx, 2)
    with app_conn(fx) as conn:
        co = cos_of(conn, wp)[0]
        assert co["waiting"] is not None and co["staged"] is None
    as_user(client, "clerk.clara")
    r = client.post(f"/api/packets/{wp}/change-orders/{co['co_no']}/approve", json={"sign": SIGN})
    assert r.status_code == 409 and r.json()["code"] == "co_not_staged", r.text
    assert "not been fetched" in r.json()["detail"]
    r = client.get(f"/api/packets/{wp}")
    assert r.status_code == 200
    assert r.json()["status"] == "ready"
    run_drawings(fx, 2)
    with app_conn(fx) as conn:
        co = cos_of(conn, wp)[0]
        assert co["staged"] is not None and co["staged"]["rev"] == "2" and co["waiting"] is None
    r = client.post(f"/api/packets/{wp}/change-orders/{co['co_no']}/approve", json={})
    assert r.status_code == 400 and r.json()["code"] == "signature_required", r.text
    r = client.post(f"/api/packets/{wp}/change-orders/{co['co_no']}/approve", json={"sign": SIGN})
    assert r.status_code == 200, r.text
    with app_conn(fx) as conn:
        p = packet_row(conn, wp)
        co = cos_of(conn, wp)[0]
        assert co["review"] == "approved" and co["approved_by"] is not None
        assert p["rev"] == "2" and p["status"] == "ready"
        assert p["visual_fp_applied"] == co["target_fp"]
        assert p["visual"]["fp_parts"] == co["vis"]["fp_parts"]
        pages = drawing_pages(conn, wp)
        void = [pg for pg in pages if pg[3]]
        live = [pg for pg in pages if not pg[3]]
        assert len(void) == len(old_pages) and all(pg[4] for pg in void)  # VOID by this CO
        assert live and all(pg[1] == "2" and pg[2] for pg in live)
        items = conn.execute("SELECT old_rev, new_rev FROM co_items WHERE co_id = %s", (co["id"],)).fetchall()
        assert ("1", "2") in [tuple(i) for i in items]
        required = {
            r[0]
            for r in conn.execute("SELECT dept_code FROM co_required_depts WHERE co_id = %s", (co["id"],)).fetchall()
        }
        assert "03" in required
    # the next feed run sees the packet matches VISUAL: no new change order
    c = run_feed(fx, 2)
    assert c["cos_opened"] == 0, c
    with app_conn(fx) as conn:
        assert len(cos_of(conn, wp)) == 1


def test_minor_co_approve_needs_no_departments(client: TestClient, fx: Settings) -> None:
    run_feed(fx, 1)
    with app_conn(fx) as conn:
        g = packet_id(conn, gen.KEY_NOT_FOUND)
        move_status(conn, g, "warehouse")
    run_feed(fx, 2)
    with app_conn(fx) as conn:
        co = cos_of(conn, g)[0]
        assert co["minor"] is True
    as_user(client, "clerk.clara")
    r = client.post(f"/api/packets/{g}/change-orders/{co['co_no']}/approve", json={"sign": SIGN})
    assert r.status_code == 200, r.text
    with app_conn(fx) as conn:
        assert conn.execute("SELECT count(*) FROM co_required_depts WHERE co_id = %s", (co["id"],)).fetchone()[0] == 0
        assert conn.execute("SELECT target_hrs FROM packet_work_orders WHERE packet_id = %s", (g,)).fetchone()[0] == 5


def test_changes_to_approve_filter(client: TestClient, fx: Settings) -> None:
    run_feed(fx, 1)
    with app_conn(fx) as conn:
        wp = packet_id(conn, gen.KEY_WELD_PART)
        move_status(conn, wp, "ready")
    run_feed(fx, 2)
    as_user(client, "clerk.clara")
    r = client.get("/api/packets?co=pending")
    assert r.status_code == 200, r.text
    body = r.json()
    items = body["packets"]
    assert [i["id"] for i in items] == [wp]
    assert items[0]["co_pending"] is True