Phase 3: VISUAL feed and M-Files drawings in the worker
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,17 @@
|
||||
"""Drawings from M-Files, read-only: the lookup, the decision table, the queue and the fetch into assets.
|
||||
|
||||
Packet Station never writes to M-Files. MFilesKit enforces that in its own code; this package only calls its
|
||||
read methods (`DrawingLookup.status`, `DrawingLookup.download`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# The packet flags this package owns. The drawing job replaces only these and keeps every other flag.
|
||||
DRAWING_FLAGS: tuple[str, ...] = (
|
||||
"drawing_needed",
|
||||
"rev_mismatch",
|
||||
"not_released",
|
||||
"not_found",
|
||||
"lookup_failed",
|
||||
"bad_rev",
|
||||
)
|
||||
@@ -0,0 +1,190 @@
|
||||
"""The drawing decision table. Pure: no database, no M-Files.
|
||||
|
||||
`decide(target, found, same_file=...)` says what the drawing job does to one packet for one lookup. Note texts are
|
||||
the app's own (from the old revMismatch and draftLine in visual.js, "-" in place of the em-dash), never M-Files'
|
||||
note passed through, except inside the lookup_failed text.
|
||||
|
||||
Rows, in the order they are tried:
|
||||
1. lookup failed -> flag lookup_failed, keep what is attached (state lookup_failed only when it was needed).
|
||||
2. past draft, a pending feed change order waits for this drawing at the released rev -> stage_for_co.
|
||||
Past draft, the pending change order already stages this drawing at the released rev -> unchanged, or
|
||||
stage_for_co again when M-Files has a different file for that rev.
|
||||
3. no released rev, a draft rev or a drafting assignment -> flag not_released with the drafting status.
|
||||
4. no released rev otherwise -> flag not_found.
|
||||
5. VISUAL rev not a number (and no approved rev equal to M-Files') -> flag bad_rev, not attached.
|
||||
6. revs differ (and no approved rev equal to M-Files') -> flag rev_mismatch. The job stages the M-Files copy
|
||||
for Clerical Supervisor approval on drafts only.
|
||||
7. revs match, draft with a hand drawing -> unchanged (the feed already took the hand drawing off on a rev move).
|
||||
8. revs match, draft with this very file attached at this rev -> unchanged.
|
||||
9. revs match, draft -> attach ("sheet of drawing <base>" for an H sheet).
|
||||
10. revs match, past draft, nothing attached -> unchanged.
|
||||
11. revs match, past draft, same file as attached -> unchanged.
|
||||
12. revs match, past draft, a pending change order already stages this file, or Clerical rejected a change order
|
||||
for this file -> unchanged.
|
||||
13. revs match, past draft, attached rev lower -> open_co "M-Files released rev N of <dwg>; this packet has rev M".
|
||||
14. revs match, past draft, attached rev equal, new file -> open_co "M-Files re-released rev N of <dwg> (new file)".
|
||||
15. revs match, past draft, attached rev higher or not a number -> unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .lookup import Found
|
||||
|
||||
LOOKUP_FAILED = "lookup_failed"
|
||||
NOT_RELEASED = "not_released"
|
||||
NOT_FOUND = "not_found"
|
||||
BAD_REV = "bad_rev"
|
||||
REV_MISMATCH = "rev_mismatch"
|
||||
ATTACH = "attach"
|
||||
UNCHANGED = "unchanged"
|
||||
OPEN_CO = "open_co"
|
||||
STAGE_FOR_CO = "stage_for_co"
|
||||
|
||||
KINDS = (ATTACH, UNCHANGED, REV_MISMATCH, NOT_RELEASED, NOT_FOUND, LOOKUP_FAILED, BAD_REV, OPEN_CO, STAGE_FOR_CO)
|
||||
FLAG_KINDS = (LOOKUP_FAILED, NOT_RELEASED, NOT_FOUND, BAD_REV, REV_MISMATCH)
|
||||
|
||||
# drawing_state enum values (0003). bad_rev has no state of its own: it is a rev mismatch Clerical can approve.
|
||||
STATE_OF = {NOT_RELEASED: "not_released", NOT_FOUND: "not_found", BAD_REV: "rev_mismatch", REV_MISMATCH: "rev_mismatch"}
|
||||
HELD_STATES = ("attached", "hand")
|
||||
|
||||
NOT_FOUND_NOTE = "Drawing not in M-Files - check the drawing number in VISUAL"
|
||||
_FAILED_PREFIX = "M-Files lookup failed: "
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Target:
|
||||
dwg: str
|
||||
visual_rev: str
|
||||
attached_rev: str | None # rev of the live drawing pages; None when no drawing page is attached
|
||||
attached_sha: bytes | None
|
||||
status: str # packet status
|
||||
drawing_state: str
|
||||
has_hand_drawing: bool
|
||||
rev_ok: str | None # the rev Clerical approved for this VISUAL rev, if any
|
||||
# Additions to the contract, all defaulted: the pending feed change order (origin visual or drawing), if any.
|
||||
co_waiting: tuple[str, str] | None = None # (dwg, rev) the pending CO waits for
|
||||
co_staged: tuple[str, str] | None = None # (dwg, rev) the pending CO already stages
|
||||
co_pending: bool = False
|
||||
co_staged_sha: bytes | None = None # sha256 of the file the pending CO already stages
|
||||
rejected_sha: bytes | None = None # sha256 staged on the last drawing-origin CO Clerical rejected
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Decision:
|
||||
kind: str
|
||||
flag: str | None
|
||||
drawing_state: str
|
||||
note: str
|
||||
|
||||
|
||||
def _int(v: object) -> int | None:
|
||||
s = "" if v is None else str(v).strip()
|
||||
return int(s) if s.isdigit() else None
|
||||
|
||||
|
||||
def draft_line(f: Found) -> str:
|
||||
"""The old draftLine: "rev N <where>", or "in drafting, no released PDF yet", or ""."""
|
||||
if f.draft_rev is not None and f.draft_where:
|
||||
return f"rev {f.draft_rev} {f.draft_where}"
|
||||
if f.in_drafting:
|
||||
return "in drafting, no released PDF yet"
|
||||
if f.draft_rev is not None:
|
||||
return f"rev {f.draft_rev}"
|
||||
return ""
|
||||
|
||||
|
||||
def failed_note(f: Found) -> str:
|
||||
detail = (f.note or "").strip()
|
||||
if detail.startswith(_FAILED_PREFIX): # MFilesKit's own note already carries the prefix
|
||||
detail = detail[len(_FAILED_PREFIX) :]
|
||||
return f"{_FAILED_PREFIX}{detail or 'no detail'}"
|
||||
|
||||
|
||||
def rev_confirmed(t: Target, f: Found) -> bool:
|
||||
"""The old revConfirmed: M-Files' released rev is the rev VISUAL calls, or Clerical approved that rev."""
|
||||
if not f.ok or f.rev is None:
|
||||
return False
|
||||
v = _int(t.visual_rev)
|
||||
if v is not None and v == f.rev:
|
||||
return True
|
||||
return t.rev_ok is not None and str(t.rev_ok).strip() == str(f.rev)
|
||||
|
||||
|
||||
def decide(t: Target, f: Found, *, same_file: bool, fetched_sha: bytes | None = None) -> Decision:
|
||||
"""What to do to one packet.
|
||||
|
||||
`same_file`: the M-Files file has the sha256 of the attached drawing asset. `fetched_sha`: the sha256 of the
|
||||
M-Files file, when it was fetched (row 12 compares it with the pending and the rejected change orders).
|
||||
"""
|
||||
held = t.drawing_state in HELD_STATES
|
||||
draft = t.status == "draft"
|
||||
|
||||
def flag(kind: str, note: str) -> Decision:
|
||||
return Decision(kind, kind, t.drawing_state if held else STATE_OF[kind], note)
|
||||
|
||||
def keep(kind: str, note: str = "") -> Decision:
|
||||
return Decision(kind, None, t.drawing_state, note)
|
||||
|
||||
# 1
|
||||
if not f.ok:
|
||||
return Decision(
|
||||
LOOKUP_FAILED,
|
||||
LOOKUP_FAILED,
|
||||
LOOKUP_FAILED if t.drawing_state == "needed" else t.drawing_state,
|
||||
failed_note(f),
|
||||
)
|
||||
# 2
|
||||
if not draft and f.rev is not None and t.co_waiting is not None:
|
||||
w_dwg, w_rev = t.co_waiting
|
||||
if w_dwg.strip() == t.dwg.strip() and str(w_rev).strip() == str(f.rev):
|
||||
return keep(STAGE_FOR_CO, f"Drawing for the change order is in M-Files (rev {f.rev})")
|
||||
if not draft and f.rev is not None and t.co_staged is not None:
|
||||
s_dwg, s_rev = t.co_staged
|
||||
if s_dwg.strip() == t.dwg.strip() and str(s_rev).strip() == str(f.rev):
|
||||
if fetched_sha is not None and t.co_staged_sha is not None and fetched_sha != t.co_staged_sha:
|
||||
return keep(STAGE_FOR_CO, f"New file for rev {f.rev} in M-Files; staged again for the change order")
|
||||
return keep(UNCHANGED, f"Rev {f.rev} is staged on the pending change order")
|
||||
# 3, 4
|
||||
if f.rev is None:
|
||||
d = draft_line(f)
|
||||
if d:
|
||||
on = f" (sheet of {f.base})" if f.base else ""
|
||||
return flag(NOT_RELEASED, f"Not released in M-Files yet{on} - drafting status: {d}")
|
||||
return flag(NOT_FOUND, NOT_FOUND_NOTE)
|
||||
approved = t.rev_ok is not None and str(t.rev_ok).strip() == str(f.rev)
|
||||
vrev = (t.visual_rev or "").strip()
|
||||
v = _int(vrev)
|
||||
if not approved:
|
||||
# 5
|
||||
if v is None:
|
||||
return flag(BAD_REV, f"VISUAL rev {vrev or '-'} is not a number; M-Files has rev {f.rev} - not attached")
|
||||
# 6
|
||||
if v != f.rev:
|
||||
d = f"; drafting status: {draft_line(f)}" if f.draft_rev is not None and f.draft_rev > f.rev else ""
|
||||
return flag(REV_MISMATCH, f"Rev mismatch: VISUAL calls rev {vrev}, M-Files has rev {f.rev}{d}")
|
||||
sheet = f"sheet of drawing {f.base}" if f.base else ""
|
||||
if draft:
|
||||
# 7
|
||||
if t.has_hand_drawing:
|
||||
return keep(UNCHANGED, "A drawing uploaded by hand stays at the same rev")
|
||||
# 8
|
||||
if t.drawing_state == "attached" and same_file and t.attached_rev == str(f.rev):
|
||||
return keep(UNCHANGED, sheet)
|
||||
# 9
|
||||
return Decision(ATTACH, None, "attached", sheet)
|
||||
# 10, 11
|
||||
if t.attached_rev is None or same_file:
|
||||
return keep(UNCHANGED, sheet)
|
||||
# 12
|
||||
if fetched_sha is not None and fetched_sha in (t.co_staged_sha, t.rejected_sha):
|
||||
return keep(UNCHANGED, sheet)
|
||||
att = _int(t.attached_rev)
|
||||
if att is None or att > f.rev:
|
||||
return keep(UNCHANGED, sheet) # 15
|
||||
if att < f.rev:
|
||||
reason = f"M-Files released rev {f.rev} of {t.dwg}; this packet has rev {att}"
|
||||
else:
|
||||
reason = f"M-Files re-released rev {f.rev} of {t.dwg} (new file)"
|
||||
return keep(OPEN_CO, reason)
|
||||
@@ -0,0 +1,118 @@
|
||||
"""The fixture lookup for CI and tests: `dir/drawings.<state>.json` and the PDFs it names.
|
||||
|
||||
`dir/state` holds the state number (default "1"). The JSON maps a drawing number to
|
||||
`{"rev": 3, "file": "L073762X r3.pdf", "draft_rev": null, "draft_where": "", "in_drafting": false, "ok": true,
|
||||
"note": "", "obj_id": 1, "obj_version": 1, "file_id": 1, "file_version": 1}`; every key is optional. A drawing
|
||||
not in the file is not found, after the same H-sheet fallback MFilesKit uses (K978-01-6H13 -> K978-01-6).
|
||||
`download` copies `dir/<file>`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .lookup import DrawingFetchError, Found, failed
|
||||
|
||||
NOT_FOUND_NOTE = "not in M-Files - check the drawing number in VISUAL"
|
||||
_H_SHEET = re.compile(r"^(.*\d)H\d{1,3}$")
|
||||
|
||||
|
||||
def base_drawing(dwg: str) -> str | None:
|
||||
m = _H_SHEET.match(dwg)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _int(v: Any) -> int | None:
|
||||
if v is None or isinstance(v, bool):
|
||||
return None
|
||||
try:
|
||||
return int(str(v).strip())
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
class FixtureLookup:
|
||||
def __init__(self, directory: Path, state: str | None = None) -> None:
|
||||
self.dir = Path(directory)
|
||||
if state is None:
|
||||
try:
|
||||
state = (self.dir / "state").read_text(encoding="utf-8").strip() or "1"
|
||||
except OSError:
|
||||
state = "1"
|
||||
self.state = state
|
||||
self._data: dict[str, dict[str, Any]] | None = None
|
||||
self._error: str | None = None
|
||||
|
||||
def _load(self) -> dict[str, dict[str, Any]]:
|
||||
if self._data is None:
|
||||
path = self.dir / f"drawings.{self.state}.json"
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
self._data = {str(k).strip(): v for k, v in raw.items() if isinstance(v, dict)}
|
||||
except (OSError, ValueError, AttributeError) as e:
|
||||
self._error = f"fixture {path.name} could not be read: {type(e).__name__}"
|
||||
self._data = {}
|
||||
return self._data
|
||||
|
||||
def status(self, dwg: str) -> Found:
|
||||
dwg = (dwg or "").strip()
|
||||
data = self._load()
|
||||
if self._error:
|
||||
return failed(dwg, self._error)
|
||||
if not dwg:
|
||||
return Found(dwg="", note="no drawing number")
|
||||
entry, base = data.get(dwg), None
|
||||
if entry is None:
|
||||
b = base_drawing(dwg)
|
||||
if b and b in data:
|
||||
entry, base = data[b], b
|
||||
if entry is None:
|
||||
return Found(dwg=dwg, note=NOT_FOUND_NOTE)
|
||||
if entry.get("ok") is False:
|
||||
return failed(dwg, str(entry.get("note") or "M-Files lookup failed: fixture says so"))
|
||||
rev = _int(entry.get("rev"))
|
||||
file = entry.get("file") or None
|
||||
size = None
|
||||
if file and rev is not None:
|
||||
p = self._file(str(file))
|
||||
size = p.stat().st_size if p is not None and p.is_file() else None
|
||||
has_file = rev is not None and bool(file)
|
||||
return Found(
|
||||
dwg=dwg,
|
||||
ok=True,
|
||||
rev=rev,
|
||||
name=str(entry.get("name") or file or "") or None if rev is not None else None,
|
||||
obj_id=_int(entry.get("obj_id", 1)) if has_file else None,
|
||||
obj_version=_int(entry.get("obj_version", 1)) if has_file else None,
|
||||
file_id=_int(entry.get("file_id", 1)) if has_file else None,
|
||||
file_version=_int(entry.get("file_version", 1)) if has_file else None,
|
||||
file_size=size,
|
||||
base=base,
|
||||
draft_rev=_int(entry.get("draft_rev")),
|
||||
draft_where=str(entry.get("draft_where") or ""),
|
||||
in_drafting=bool(entry.get("in_drafting")),
|
||||
note=str(entry.get("note") or ""),
|
||||
)
|
||||
|
||||
def _file(self, name: str) -> Path | None:
|
||||
root = self.dir.resolve()
|
||||
p = (root / name).resolve()
|
||||
return p if p.is_relative_to(root) else None
|
||||
|
||||
def download(self, found: Found, dest: Path) -> int:
|
||||
entry = self._load().get(found.base or found.dwg) or self._load().get(found.dwg) or {}
|
||||
name = entry.get("file")
|
||||
src = self._file(str(name)) if name else None
|
||||
if src is None or not src.is_file():
|
||||
raise DrawingFetchError(f"{found.dwg}: fixture file {name!r} is missing")
|
||||
dest = Path(dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(src, dest)
|
||||
return dest.stat().st_size
|
||||
|
||||
def describe(self) -> str:
|
||||
return f"fixture {self.dir} state {self.state}"
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Copy a released drawing PDF from M-Files into assets.
|
||||
|
||||
Downloads only when the M-Files identity (object id and version, file id and version) differs from what the
|
||||
`drawing_lookups` row last stored, or that row has no asset (or the asset's file is gone). The download goes to
|
||||
ASSETS_DIR/tmp/<uuid>; the PDF magic and the size limit are checked, then `assets.store_file` hashes it in chunks,
|
||||
fsyncs and renames it into place, and reuses an existing asset row with the same sha256 and size.
|
||||
|
||||
Read-only to M-Files: only `Lookup.download` is called.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from ..api.errors import ApiError
|
||||
from ..domain import assets
|
||||
from .lookup import Found, Lookup
|
||||
|
||||
PDF_MAGIC = b"%PDF-"
|
||||
|
||||
|
||||
class FetchError(Exception):
|
||||
"""The drawing could not be copied. The message is safe to log."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FetchResult:
|
||||
asset_id: uuid.UUID
|
||||
sha256: bytes
|
||||
pages: int
|
||||
downloaded: bool # False when the stored asset was still current
|
||||
name: str | None = None
|
||||
reused: bool = False # downloaded, but the same bytes were already stored
|
||||
|
||||
|
||||
def _current(conn: Any, settings: Any, found: Found, dwg: str) -> FetchResult | None:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
row = cur.execute(
|
||||
"SELECT l.obj_id, l.obj_version, l.file_id, l.file_version, l.asset_id, a.sha256, a.pages,"
|
||||
" a.storage_key, a.original_name FROM drawing_lookups l LEFT JOIN assets a ON a.id = l.asset_id"
|
||||
" WHERE l.dwg = %s",
|
||||
(dwg,),
|
||||
).fetchone()
|
||||
if row is None or row["asset_id"] is None or row["sha256"] is None:
|
||||
return None
|
||||
if (row["obj_id"], row["obj_version"], row["file_id"], row["file_version"]) != found.identity:
|
||||
return None
|
||||
if None in found.identity:
|
||||
return None
|
||||
try:
|
||||
if not assets.path_of(settings, str(row["storage_key"])).is_file():
|
||||
return None
|
||||
except ApiError:
|
||||
return None
|
||||
return FetchResult(
|
||||
asset_id=row["asset_id"],
|
||||
sha256=bytes(row["sha256"]),
|
||||
pages=int(row["pages"] or 0),
|
||||
downloaded=False,
|
||||
name=row["original_name"],
|
||||
)
|
||||
|
||||
|
||||
def fetch(conn: Any, settings: Any, lookup: Lookup, found: Found, *, dwg: str) -> FetchResult:
|
||||
"""The asset for M-Files' released file of `dwg`. Raises FetchError."""
|
||||
if not found.ok or found.rev is None:
|
||||
raise FetchError(f"{dwg}: no released file in M-Files")
|
||||
cur = _current(conn, settings, found, dwg)
|
||||
if cur is not None:
|
||||
return cur
|
||||
limit = int(settings.upload_max_bytes)
|
||||
if found.file_size is not None and found.file_size > limit:
|
||||
raise FetchError(f"{dwg}: the M-Files file is {found.file_size} bytes, over the {limit} byte limit")
|
||||
tmp_dir = Path(settings.assets_dir) / "tmp"
|
||||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
tmp = tmp_dir / uuid.uuid4().hex
|
||||
try:
|
||||
try:
|
||||
lookup.download(found, tmp)
|
||||
except LookupError as e:
|
||||
raise FetchError(str(e)) from None
|
||||
except OSError as e:
|
||||
raise FetchError(f"{dwg}: could not write the download: {type(e).__name__}") from None
|
||||
try:
|
||||
size = tmp.stat().st_size
|
||||
with tmp.open("rb") as f:
|
||||
head = f.read(len(PDF_MAGIC))
|
||||
except OSError:
|
||||
raise FetchError(f"{dwg}: the download left no file") from None
|
||||
if size > limit:
|
||||
raise FetchError(f"{dwg}: the M-Files file is {size} bytes, over the {limit} byte limit")
|
||||
if head != PDF_MAGIC:
|
||||
raise FetchError(f"{dwg}: the M-Files file is not a PDF")
|
||||
name = found.name or f"{dwg} r{found.rev}.pdf"
|
||||
source_ref = {
|
||||
"dwg": dwg,
|
||||
"obj_id": found.obj_id,
|
||||
"obj_version": found.obj_version,
|
||||
"file_id": found.file_id,
|
||||
"file_version": found.file_version,
|
||||
"rev": found.rev,
|
||||
}
|
||||
try:
|
||||
row = assets.store_file(
|
||||
conn, settings, tmp, mime=assets.PDF, original_name=name, source="mfiles", source_ref=source_ref
|
||||
)
|
||||
except ApiError as e:
|
||||
raise FetchError(f"{dwg}: {e.detail}") from None
|
||||
return FetchResult(
|
||||
asset_id=row["id"],
|
||||
sha256=bytes(row["sha256"]),
|
||||
pages=int(row["pages"] or 0),
|
||||
downloaded=True,
|
||||
name=row.get("original_name") or name,
|
||||
reused=bool(row.get("reused")),
|
||||
)
|
||||
finally:
|
||||
with contextlib.suppress(OSError):
|
||||
tmp.unlink()
|
||||
@@ -0,0 +1,183 @@
|
||||
"""The drawing lookup the job talks to: M-Files through MFilesKit, or the fixture files (fake.py).
|
||||
|
||||
One M-Files `Vault` per worker process (each sign-in can use a licence), kept in the dict the scheduler passes
|
||||
(`JobContext.shared`) or, when there is none, in this module's own cache keyed by MFILES_URL. A new
|
||||
`DrawingLookup` per run, because its result cache lives on the instance.
|
||||
|
||||
Read-only. Nothing here writes to M-Files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from mfileskit.drawings import DrawingStatus
|
||||
from mfileskit.vault import Vault
|
||||
|
||||
from ..settings import Settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
VAULT_KEY = "mfiles_vault"
|
||||
NOTE_MAX = 500
|
||||
|
||||
|
||||
class DrawingFetchError(LookupError):
|
||||
"""A download did not work. The message is safe to log (no credentials)."""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Found:
|
||||
"""What M-Files has for one drawing number: the subset the decision needs, plus identity for change detection."""
|
||||
|
||||
dwg: str
|
||||
ok: bool = True
|
||||
rev: int | None = None
|
||||
name: str | None = None
|
||||
obj_id: int | None = None
|
||||
obj_version: int | None = None
|
||||
file_id: int | None = None
|
||||
file_version: int | None = None
|
||||
file_size: int | None = None
|
||||
base: str | None = None
|
||||
draft_rev: int | None = None
|
||||
draft_where: str = ""
|
||||
in_drafting: bool = False
|
||||
note: str = ""
|
||||
|
||||
@property
|
||||
def identity(self) -> tuple[int | None, int | None, int | None, int | None]:
|
||||
return (self.obj_id, self.obj_version, self.file_id, self.file_version)
|
||||
|
||||
|
||||
def failed(dwg: str, note: str) -> Found:
|
||||
return Found(dwg=dwg, ok=False, note=note[:NOTE_MAX])
|
||||
|
||||
|
||||
class Lookup(Protocol):
|
||||
def status(self, dwg: str) -> Found: ...
|
||||
|
||||
def download(self, found: Found, dest: Path) -> int:
|
||||
"""Write the released PDF to `dest`; bytes written. Raises DrawingFetchError (a LookupError)."""
|
||||
...
|
||||
|
||||
def describe(self) -> str: ...
|
||||
|
||||
|
||||
def from_status(st: DrawingStatus) -> Found:
|
||||
"""MFilesKit's DrawingStatus -> Found."""
|
||||
ov, fi = st.obj_ver, st.file
|
||||
return Found(
|
||||
dwg=st.drawing,
|
||||
ok=bool(st.ok),
|
||||
rev=st.rev,
|
||||
name=st.name,
|
||||
obj_id=ov.id if ov is not None else None,
|
||||
obj_version=ov.version if ov is not None else None,
|
||||
file_id=fi.id if fi is not None else None,
|
||||
file_version=fi.version if fi is not None else None,
|
||||
file_size=fi.size if fi is not None else None,
|
||||
base=st.base,
|
||||
draft_rev=st.draft_rev,
|
||||
draft_where=st.draft_where or "",
|
||||
in_drafting=bool(st.in_drafting),
|
||||
note=(st.note or "")[:NOTE_MAX],
|
||||
)
|
||||
|
||||
|
||||
class MFilesLookup:
|
||||
"""M-Files through `mfileskit.drawings.DrawingLookup`. Read-only."""
|
||||
|
||||
def __init__(self, vault: Vault) -> None:
|
||||
from mfileskit.drawings import DrawingLookup
|
||||
|
||||
self.vault = vault
|
||||
self._lookup = DrawingLookup(vault) # reads the vault schema; raises MFilesError when the vault is down
|
||||
self._seen: dict[str, DrawingStatus] = {}
|
||||
|
||||
def status(self, dwg: str) -> Found:
|
||||
from mfileskit.errors import MFilesError
|
||||
|
||||
dwg = (dwg or "").strip()
|
||||
try:
|
||||
st = self._lookup.status(dwg)
|
||||
except MFilesError as e: # status() catches these itself; this is a second net
|
||||
return failed(dwg, f"M-Files lookup failed: {e}")
|
||||
self._seen[dwg] = st
|
||||
return from_status(st)
|
||||
|
||||
def download(self, found: Found, dest: Path) -> int:
|
||||
from mfileskit.errors import MFilesError
|
||||
|
||||
st = self._seen.get(found.dwg)
|
||||
if st is None or st.obj_ver is None or st.file is None:
|
||||
raise DrawingFetchError(f"{found.dwg}: no released file to download")
|
||||
try:
|
||||
return int(self._lookup.download(st, dest))
|
||||
except MFilesError as e:
|
||||
raise DrawingFetchError(f"{found.dwg}: download failed: {e}"[:NOTE_MAX]) from None
|
||||
|
||||
def describe(self) -> str:
|
||||
url = getattr(getattr(self.vault, "settings", None), "url", "") or ""
|
||||
return f"M-Files {url}".strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- one vault per process
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
_CACHE: dict[str, Any] = {}
|
||||
|
||||
|
||||
def _cache_for(cache: dict[str, Any] | None) -> tuple[dict[str, Any], str]:
|
||||
if cache is not None:
|
||||
return cache, VAULT_KEY
|
||||
return _CACHE, f"{VAULT_KEY}:{os.environ.get('MFILES_URL', '')}"
|
||||
|
||||
|
||||
def vault_for(settings: Settings, cache: dict[str, Any] | None = None) -> Vault:
|
||||
"""The process's Vault, made once with `Vault.from_env()` (MFILES_* from the environment; the entrypoint
|
||||
exports MFILES_PASSWORD from its secret file) and signed in."""
|
||||
from mfileskit.vault import Vault
|
||||
|
||||
store, key = _cache_for(cache)
|
||||
with _LOCK:
|
||||
vault = store.get(key)
|
||||
if vault is None:
|
||||
vault = Vault.from_env().connect()
|
||||
store[key] = vault
|
||||
return vault
|
||||
|
||||
|
||||
def forget_vault(cache: dict[str, Any] | None = None) -> None:
|
||||
"""Drop (and close) the cached Vault, so the next run signs in again."""
|
||||
store, key = _cache_for(cache)
|
||||
with _LOCK:
|
||||
vault = store.pop(key, None)
|
||||
if vault is not None:
|
||||
try:
|
||||
vault.close()
|
||||
except Exception as e: # closing a broken connection may fail; nothing to do
|
||||
log.info("closing the M-Files vault failed: %s", type(e).__name__)
|
||||
|
||||
|
||||
def make(settings: Settings, cache: dict[str, Any] | None = None) -> Lookup:
|
||||
"""The lookup for one run. `drawing_source = fixture` reads `fixture_dir`; otherwise M-Files.
|
||||
|
||||
Raises when M-Files cannot be opened (the job then records a degraded run). A failed open drops the cached
|
||||
Vault so the next run tries a fresh sign-in.
|
||||
"""
|
||||
if getattr(settings, "drawing_source", "mfiles") == "fixture":
|
||||
from .fake import FixtureLookup
|
||||
|
||||
return FixtureLookup(Path(settings.fixture_dir))
|
||||
try:
|
||||
return MFilesLookup(vault_for(settings, cache))
|
||||
except Exception:
|
||||
forget_vault(cache)
|
||||
raise
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Which drawings the drawing job looks up this run, in order.
|
||||
|
||||
Over live packets made from VISUAL that are not filed, have a drawing number, and sit in a department in scope
|
||||
(feed setting `departments`; empty = every department). One item per drawing number (trimmed), the best priority
|
||||
winning:
|
||||
|
||||
1. the VISUAL rev changed since the last lookup (`drawing_lookups.visual_rev_seen`), or a pending feed change
|
||||
order waits for a drawing (the item is the CO's target drawing and rev; at most every 15 minutes);
|
||||
2. drafts still needing a drawing (`drawing_state = 'needed'`, or never looked up), unless a failed fetch set a
|
||||
later `next_check_at`;
|
||||
3. failed lookups whose backoff ran out (`ok = false and next_check_at <= now`);
|
||||
4. rechecks due (`checked_at` older than `drawing_recheck_hours`).
|
||||
|
||||
A packet with a hand drawing joins only through (1) or (4) once the feed has taken the hand drawing off; while
|
||||
`drawing_state = 'hand'` it is skipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
BACKOFF_FIRST = timedelta(minutes=15)
|
||||
BACKOFF_MAX = timedelta(hours=24)
|
||||
CO_RECHECK = timedelta(minutes=15) # a change order's target drawing is looked up at most this often
|
||||
|
||||
QUEUE_SQL = """
|
||||
WITH own AS (
|
||||
SELECT p.id AS packet_id, btrim(p.drawing) AS dwg, coalesce(btrim(p.rev), '') AS visual_rev, NULL::bigint AS co_id,
|
||||
CASE
|
||||
WHEN d.dwg IS NOT NULL AND d.checked_at IS NOT NULL
|
||||
AND d.visual_rev_seen IS DISTINCT FROM coalesce(btrim(p.rev), '') THEN 1
|
||||
WHEN p.status = 'draft' AND (d.dwg IS NULL OR p.drawing_state = 'needed')
|
||||
AND (d.next_check_at IS NULL OR d.next_check_at <= %(now)s) THEN 2
|
||||
WHEN d.ok = false AND (d.next_check_at IS NULL OR d.next_check_at <= %(now)s) THEN 3
|
||||
WHEN d.checked_at IS NULL OR d.checked_at < %(recheck_before)s THEN 4
|
||||
END AS priority,
|
||||
d.checked_at
|
||||
FROM packets p
|
||||
LEFT JOIN drawing_lookups d ON d.dwg = btrim(p.drawing)
|
||||
WHERE p.removed_at IS NULL AND p.source_kind = 'visual' AND p.status <> 'filed'
|
||||
AND coalesce(btrim(p.drawing), '') <> '' AND p.drawing_state <> 'hand'
|
||||
AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[]))
|
||||
),
|
||||
co AS (
|
||||
SELECT p.id AS packet_id, btrim(c.waiting->>'dwg') AS dwg, coalesce(btrim(c.waiting->>'rev'), '') AS visual_rev,
|
||||
c.id AS co_id, 1 AS priority, d.checked_at
|
||||
FROM change_orders c
|
||||
JOIN packets p ON p.id = c.packet_id
|
||||
LEFT JOIN drawing_lookups d ON d.dwg = btrim(c.waiting->>'dwg')
|
||||
WHERE c.review = 'pending' AND c.origin IN ('visual', 'drawing') AND c.waiting IS NOT NULL
|
||||
AND jsonb_typeof(c.waiting) = 'object' AND coalesce(btrim(c.waiting->>'dwg'), '') <> ''
|
||||
AND p.removed_at IS NULL AND p.status NOT IN ('draft', 'filed')
|
||||
AND (d.checked_at IS NULL OR d.checked_at < %(co_before)s)
|
||||
AND (d.ok IS NOT false OR d.next_check_at IS NULL OR d.next_check_at <= %(now)s)
|
||||
AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[]))
|
||||
),
|
||||
items AS (
|
||||
SELECT * FROM own WHERE priority IS NOT NULL
|
||||
UNION ALL
|
||||
SELECT * FROM co
|
||||
),
|
||||
best AS (
|
||||
SELECT DISTINCT ON (dwg) packet_id, dwg, visual_rev, priority, co_id, checked_at
|
||||
FROM items
|
||||
ORDER BY dwg, priority, co_id NULLS LAST, packet_id
|
||||
)
|
||||
SELECT packet_id, dwg, visual_rev, priority, co_id
|
||||
FROM best
|
||||
ORDER BY priority, checked_at NULLS FIRST, dwg
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class QueueItem:
|
||||
packet_id: int
|
||||
dwg: str
|
||||
visual_rev: str
|
||||
priority: int
|
||||
co_id: int | None
|
||||
|
||||
|
||||
def backoff(fail_count: int) -> timedelta:
|
||||
"""15 minutes x 2^(fail_count - 1), at most 24 hours. fail_count counts this failure (1 = first)."""
|
||||
n = max(0, int(fail_count) - 1)
|
||||
if n >= 10:
|
||||
return BACKOFF_MAX
|
||||
return min(BACKOFF_FIRST * (2**n), BACKOFF_MAX)
|
||||
|
||||
|
||||
def params(feed_settings: Any, now: datetime) -> dict[str, Any]:
|
||||
hours = int(getattr(feed_settings, "drawing_recheck_hours", 4))
|
||||
depts = [str(d) for d in (getattr(feed_settings, "departments", None) or [])]
|
||||
return {
|
||||
"now": now,
|
||||
"recheck_before": now - timedelta(hours=hours),
|
||||
"co_before": now - CO_RECHECK,
|
||||
"depts": depts,
|
||||
}
|
||||
|
||||
|
||||
def queue(conn: Any, feed_settings: Any, now: datetime, *, limit: int | None = None) -> list[QueueItem]:
|
||||
"""The ordered lookup list. `limit` cuts it (the job passes the budget)."""
|
||||
sql = QUEUE_SQL
|
||||
p = params(feed_settings, now)
|
||||
if limit is not None:
|
||||
sql += " LIMIT %(limit)s"
|
||||
p["limit"] = max(0, int(limit))
|
||||
rows = conn.execute(sql, p).fetchall()
|
||||
return [
|
||||
QueueItem(int(r[0]), str(r[1]), str(r[2] or ""), int(r[3]), None if r[4] is None else int(r[4])) for r in rows
|
||||
]
|
||||
Reference in New Issue
Block a user