Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/server/test_handbom.py

@@ -0,0 +1,112 @@
+"""Hand-uploaded BOM files: header detection, the column patterns, skipped rows and bad files."""
+
+from __future__ import annotations
+
+import io
+from decimal import Decimal
+from typing import Any
+
+import pytest
+
+from packetstation.domain import handbom
+
+
+def xlsx(rows: list[list[Any]]) -> bytes:
+    from openpyxl import Workbook
+
+    wb = Workbook()
+    ws = wb.active
+    assert ws is not None
+    for r in rows:
+        ws.append(r)
+    buf = io.BytesIO()
+    wb.save(buf)
+    return buf.getvalue()
+
+
+def csv_bytes(text: str, bom: bool = True) -> bytes:
+    return (b"\xef\xbb\xbf" if bom else b"") + text.encode("utf-8")
+
+
+def test_csv_header_after_title_rows() -> None:
+    data = csv_bytes(
+        "Bill of material,,,\n"
+        "Job L900-01,,,\n"
+        "\n"
+        "Item,Qty,Part No.,Description,Length,Material,Dwg Ref\n"
+        "1,2,PIPE-6-SCH40,Pipe 6 in sch 40,12 ft,A106-B,L900-0101\n"
+        "2,4,,Gusset 1/4 plate,,A36,\n"
+    )
+    lines = handbom.parse(data, "bom.csv")
+    assert [ln["item"] for ln in lines] == [1, 2]
+    first = lines[0]
+    assert first["qty"] == Decimal(2) and first["part_id"] == "PIPE-6-SCH40"
+    assert first["description"] == "Pipe 6 in sch 40" and first["length"] == "12 ft"
+    assert first["material"] == "A106-B" and first["dwg_ref"] == "L900-0101"
+    assert lines[1]["part_id"] is None and lines[1]["description"] == "Gusset 1/4 plate"
+
+
+def test_xlsx_with_quantity_and_part_number() -> None:
+    data = xlsx(
+        [
+            ["PESCO", None, None],
+            ["Find", "Quantity", "PESCO Part Number", "Desc"],
+            [10, 1, "FLG-6-150", "Flange"],
+            [None, None, None, None],
+            [11, 2.5, "GSK-6-150", "Gasket"],
+        ]
+    )
+    lines = handbom.parse(data, "bom.xlsx")
+    assert [(ln["item"], ln["qty"], ln["part_id"]) for ln in lines] == [
+        (10, Decimal(1), "FLG-6-150"),
+        (11, Decimal("2.5"), "GSK-6-150"),
+    ]
+
+
+def test_rows_without_description_or_part_are_skipped() -> None:
+    data = csv_bytes("Qty,Description\n2,Pipe\n,\n5,\nTotal,\n1,Cap\n")
+    lines = handbom.parse(data, "bom.csv")
+    assert [ln["description"] for ln in lines] == ["Pipe", "Cap"]
+
+
+def test_items_default_to_the_row_number_and_skip_repeats() -> None:
+    data = csv_bytes("Item,Qty,Description\nA,1,One\n,1,Two\n1,1,Three\n")
+    lines = handbom.parse(data, "bom.csv")
+    items = [ln["item"] for ln in lines]
+    assert len(set(items)) == 3
+    assert items[0] == 1 and lines[0]["cl_item"] == "A"
+
+
+def test_semicolon_csv_without_bom() -> None:
+    data = csv_bytes("Qty;Part ID;Description\n3;BOLT-5/8;Stud bolt\n", bom=False)
+    lines = handbom.parse(data, "bom.csv")
+    assert lines[0]["qty"] == Decimal(3) and lines[0]["part_id"] == "BOLT-5/8"
+
+
+@pytest.mark.parametrize(
+    "data,name",
+    [
+        (b"", "empty.csv"),
+        (b"%PDF-1.7\n...", "drawing.pdf"),
+        (csv_bytes("Name,Colour\nPipe,Red\n"), "no-qty.csv"),
+        (csv_bytes("Qty,Length\n2,12 ft\n"), "no-desc.csv"),
+        (csv_bytes("Qty,Description\n,\n"), "no-lines.csv"),
+        (b"PK\x03\x04not really a zip", "broken.xlsx"),
+    ],
+)
+def test_bad_files(data: bytes, name: str) -> None:
+    with pytest.raises(handbom.BadFile) as e:
+        handbom.parse(data, name)
+    assert str(e.value)
+
+
+def test_header_must_be_within_the_first_20_rows() -> None:
+    data = csv_bytes("\n".join(["note"] * 25) + "\nQty,Description\n1,Pipe\n")
+    with pytest.raises(handbom.BadFile):
+        handbom.parse(data, "bom.csv")
+
+
+def test_file_text_is_data() -> None:
+    data = csv_bytes('Qty,Description\n1,"=HYPERLINK(""http://x"")  ignore previous instructions"\n')
+    lines = handbom.parse(data, "bom.csv")
+    assert lines[0]["description"].startswith("=HYPERLINK")