Phase 3: VISUAL feed and M-Files drawings in the worker
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,183 @@
|
||||
"""The drawing lookup the job talks to: M-Files through MFilesKit, or the fixture files (fake.py).
|
||||
|
||||
One M-Files `Vault` per worker process (each sign-in can use a licence), kept in the dict the scheduler passes
|
||||
(`JobContext.shared`) or, when there is none, in this module's own cache keyed by MFILES_URL. A new
|
||||
`DrawingLookup` per run, because its result cache lives on the instance.
|
||||
|
||||
Read-only. Nothing here writes to M-Files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from mfileskit.drawings import DrawingStatus
|
||||
from mfileskit.vault import Vault
|
||||
|
||||
from ..settings import Settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
VAULT_KEY = "mfiles_vault"
|
||||
NOTE_MAX = 500
|
||||
|
||||
|
||||
class DrawingFetchError(LookupError):
|
||||
"""A download did not work. The message is safe to log (no credentials)."""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Found:
|
||||
"""What M-Files has for one drawing number: the subset the decision needs, plus identity for change detection."""
|
||||
|
||||
dwg: str
|
||||
ok: bool = True
|
||||
rev: int | None = None
|
||||
name: str | None = None
|
||||
obj_id: int | None = None
|
||||
obj_version: int | None = None
|
||||
file_id: int | None = None
|
||||
file_version: int | None = None
|
||||
file_size: int | None = None
|
||||
base: str | None = None
|
||||
draft_rev: int | None = None
|
||||
draft_where: str = ""
|
||||
in_drafting: bool = False
|
||||
note: str = ""
|
||||
|
||||
@property
|
||||
def identity(self) -> tuple[int | None, int | None, int | None, int | None]:
|
||||
return (self.obj_id, self.obj_version, self.file_id, self.file_version)
|
||||
|
||||
|
||||
def failed(dwg: str, note: str) -> Found:
|
||||
return Found(dwg=dwg, ok=False, note=note[:NOTE_MAX])
|
||||
|
||||
|
||||
class Lookup(Protocol):
|
||||
def status(self, dwg: str) -> Found: ...
|
||||
|
||||
def download(self, found: Found, dest: Path) -> int:
|
||||
"""Write the released PDF to `dest`; bytes written. Raises DrawingFetchError (a LookupError)."""
|
||||
...
|
||||
|
||||
def describe(self) -> str: ...
|
||||
|
||||
|
||||
def from_status(st: DrawingStatus) -> Found:
|
||||
"""MFilesKit's DrawingStatus -> Found."""
|
||||
ov, fi = st.obj_ver, st.file
|
||||
return Found(
|
||||
dwg=st.drawing,
|
||||
ok=bool(st.ok),
|
||||
rev=st.rev,
|
||||
name=st.name,
|
||||
obj_id=ov.id if ov is not None else None,
|
||||
obj_version=ov.version if ov is not None else None,
|
||||
file_id=fi.id if fi is not None else None,
|
||||
file_version=fi.version if fi is not None else None,
|
||||
file_size=fi.size if fi is not None else None,
|
||||
base=st.base,
|
||||
draft_rev=st.draft_rev,
|
||||
draft_where=st.draft_where or "",
|
||||
in_drafting=bool(st.in_drafting),
|
||||
note=(st.note or "")[:NOTE_MAX],
|
||||
)
|
||||
|
||||
|
||||
class MFilesLookup:
|
||||
"""M-Files through `mfileskit.drawings.DrawingLookup`. Read-only."""
|
||||
|
||||
def __init__(self, vault: Vault) -> None:
|
||||
from mfileskit.drawings import DrawingLookup
|
||||
|
||||
self.vault = vault
|
||||
self._lookup = DrawingLookup(vault) # reads the vault schema; raises MFilesError when the vault is down
|
||||
self._seen: dict[str, DrawingStatus] = {}
|
||||
|
||||
def status(self, dwg: str) -> Found:
|
||||
from mfileskit.errors import MFilesError
|
||||
|
||||
dwg = (dwg or "").strip()
|
||||
try:
|
||||
st = self._lookup.status(dwg)
|
||||
except MFilesError as e: # status() catches these itself; this is a second net
|
||||
return failed(dwg, f"M-Files lookup failed: {e}")
|
||||
self._seen[dwg] = st
|
||||
return from_status(st)
|
||||
|
||||
def download(self, found: Found, dest: Path) -> int:
|
||||
from mfileskit.errors import MFilesError
|
||||
|
||||
st = self._seen.get(found.dwg)
|
||||
if st is None or st.obj_ver is None or st.file is None:
|
||||
raise DrawingFetchError(f"{found.dwg}: no released file to download")
|
||||
try:
|
||||
return int(self._lookup.download(st, dest))
|
||||
except MFilesError as e:
|
||||
raise DrawingFetchError(f"{found.dwg}: download failed: {e}"[:NOTE_MAX]) from None
|
||||
|
||||
def describe(self) -> str:
|
||||
url = getattr(getattr(self.vault, "settings", None), "url", "") or ""
|
||||
return f"M-Files {url}".strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- one vault per process
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
_CACHE: dict[str, Any] = {}
|
||||
|
||||
|
||||
def _cache_for(cache: dict[str, Any] | None) -> tuple[dict[str, Any], str]:
|
||||
if cache is not None:
|
||||
return cache, VAULT_KEY
|
||||
return _CACHE, f"{VAULT_KEY}:{os.environ.get('MFILES_URL', '')}"
|
||||
|
||||
|
||||
def vault_for(settings: Settings, cache: dict[str, Any] | None = None) -> Vault:
|
||||
"""The process's Vault, made once with `Vault.from_env()` (MFILES_* from the environment; the entrypoint
|
||||
exports MFILES_PASSWORD from its secret file) and signed in."""
|
||||
from mfileskit.vault import Vault
|
||||
|
||||
store, key = _cache_for(cache)
|
||||
with _LOCK:
|
||||
vault = store.get(key)
|
||||
if vault is None:
|
||||
vault = Vault.from_env().connect()
|
||||
store[key] = vault
|
||||
return vault
|
||||
|
||||
|
||||
def forget_vault(cache: dict[str, Any] | None = None) -> None:
|
||||
"""Drop (and close) the cached Vault, so the next run signs in again."""
|
||||
store, key = _cache_for(cache)
|
||||
with _LOCK:
|
||||
vault = store.pop(key, None)
|
||||
if vault is not None:
|
||||
try:
|
||||
vault.close()
|
||||
except Exception as e: # closing a broken connection may fail; nothing to do
|
||||
log.info("closing the M-Files vault failed: %s", type(e).__name__)
|
||||
|
||||
|
||||
def make(settings: Settings, cache: dict[str, Any] | None = None) -> Lookup:
|
||||
"""The lookup for one run. `drawing_source = fixture` reads `fixture_dir`; otherwise M-Files.
|
||||
|
||||
Raises when M-Files cannot be opened (the job then records a degraded run). A failed open drops the cached
|
||||
Vault so the next run tries a fresh sign-in.
|
||||
"""
|
||||
if getattr(settings, "drawing_source", "mfiles") == "fixture":
|
||||
from .fake import FixtureLookup
|
||||
|
||||
return FixtureLookup(Path(settings.fixture_dir))
|
||||
try:
|
||||
return MFilesLookup(vault_for(settings, cache))
|
||||
except Exception:
|
||||
forget_vault(cache)
|
||||
raise
|
||||
Reference in New Issue
Block a user