Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
2026-10-16 14:45:38 -06:00
parent 16950055e3
commit 591fe90076
103 changed files with 16063 additions and 151 deletions
@@ -0,0 +1,118 @@
+"""The fixture lookup for CI and tests: `dir/drawings.<state>.json` and the PDFs it names.
+
+`dir/state` holds the state number (default "1"). The JSON maps a drawing number to
+`{"rev": 3, "file": "L073762X r3.pdf", "draft_rev": null, "draft_where": "", "in_drafting": false, "ok": true,
+"note": "", "obj_id": 1, "obj_version": 1, "file_id": 1, "file_version": 1}`; every key is optional. A drawing
+not in the file is not found, after the same H-sheet fallback MFilesKit uses (K978-01-6H13 -> K978-01-6).
+`download` copies `dir/<file>`.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+from pathlib import Path
+from typing import Any
+
+from .lookup import DrawingFetchError, Found, failed
+
+NOT_FOUND_NOTE = "not in M-Files - check the drawing number in VISUAL"
+_H_SHEET = re.compile(r"^(.*\d)H\d{1,3}$")
+
+
+def base_drawing(dwg: str) -> str | None:
+    m = _H_SHEET.match(dwg)
+    return m.group(1) if m else None
+
+
+def _int(v: Any) -> int | None:
+    if v is None or isinstance(v, bool):
+        return None
+    try:
+        return int(str(v).strip())
+    except ValueError:
+        return None
+
+
+class FixtureLookup:
+    def __init__(self, directory: Path, state: str | None = None) -> None:
+        self.dir = Path(directory)
+        if state is None:
+            try:
+                state = (self.dir / "state").read_text(encoding="utf-8").strip() or "1"
+            except OSError:
+                state = "1"
+        self.state = state
+        self._data: dict[str, dict[str, Any]] | None = None
+        self._error: str | None = None
+
+    def _load(self) -> dict[str, dict[str, Any]]:
+        if self._data is None:
+            path = self.dir / f"drawings.{self.state}.json"
+            try:
+                raw = json.loads(path.read_text(encoding="utf-8"))
+                self._data = {str(k).strip(): v for k, v in raw.items() if isinstance(v, dict)}
+            except (OSError, ValueError, AttributeError) as e:
+                self._error = f"fixture {path.name} could not be read: {type(e).__name__}"
+                self._data = {}
+        return self._data
+
+    def status(self, dwg: str) -> Found:
+        dwg = (dwg or "").strip()
+        data = self._load()
+        if self._error:
+            return failed(dwg, self._error)
+        if not dwg:
+            return Found(dwg="", note="no drawing number")
+        entry, base = data.get(dwg), None
+        if entry is None:
+            b = base_drawing(dwg)
+            if b and b in data:
+                entry, base = data[b], b
+        if entry is None:
+            return Found(dwg=dwg, note=NOT_FOUND_NOTE)
+        if entry.get("ok") is False:
+            return failed(dwg, str(entry.get("note") or "M-Files lookup failed: fixture says so"))
+        rev = _int(entry.get("rev"))
+        file = entry.get("file") or None
+        size = None
+        if file and rev is not None:
+            p = self._file(str(file))
+            size = p.stat().st_size if p is not None and p.is_file() else None
+        has_file = rev is not None and bool(file)
+        return Found(
+            dwg=dwg,
+            ok=True,
+            rev=rev,
+            name=str(entry.get("name") or file or "") or None if rev is not None else None,
+            obj_id=_int(entry.get("obj_id", 1)) if has_file else None,
+            obj_version=_int(entry.get("obj_version", 1)) if has_file else None,
+            file_id=_int(entry.get("file_id", 1)) if has_file else None,
+            file_version=_int(entry.get("file_version", 1)) if has_file else None,
+            file_size=size,
+            base=base,
+            draft_rev=_int(entry.get("draft_rev")),
+            draft_where=str(entry.get("draft_where") or ""),
+            in_drafting=bool(entry.get("in_drafting")),
+            note=str(entry.get("note") or ""),
+        )
+
+    def _file(self, name: str) -> Path | None:
+        root = self.dir.resolve()
+        p = (root / name).resolve()
+        return p if p.is_relative_to(root) else None
+
+    def download(self, found: Found, dest: Path) -> int:
+        entry = self._load().get(found.base or found.dwg) or self._load().get(found.dwg) or {}
+        name = entry.get("file")
+        src = self._file(str(name)) if name else None
+        if src is None or not src.is_file():
+            raise DrawingFetchError(f"{found.dwg}: fixture file {name!r} is missing")
+        dest = Path(dest)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(src, dest)
+        return dest.stat().st_size
+
+    def describe(self) -> str:
+        return f"fixture {self.dir} state {self.state}"