Phase 3: VISUAL feed and M-Files drawings in the worker
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,146 @@
|
||||
"""The job registry and the shapes every job uses.
|
||||
|
||||
Jobs import `from packetstation.worker.jobs import JobContext, JobSpec, register` and expose `SPEC`. The feed and
|
||||
drawing jobs are imported lazily by `load_jobs`, so the worker runs before they exist.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, date, datetime, time, timedelta
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from ...feed.config import FeedSettings
|
||||
from ...settings import Settings
|
||||
from ..redaction import MAX_ERRORS, clip, redact
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
Counts = dict[str, Any]
|
||||
Cadence = Callable[[FeedSettings, datetime], int | None]
|
||||
TRIGGERS = ("schedule", "request", "chain", "cli")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Stack:
|
||||
"""Per-process state shared by every run: the startup guard result and long-lived objects (the M-Files vault)."""
|
||||
|
||||
guard: Any = None
|
||||
cache: dict[str, Any] = field(default_factory=dict)
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
|
||||
def cached(self, key: str, factory: Callable[[], Any]) -> Any:
|
||||
"""The object under `key`, made once by `factory`. A factory that raises leaves nothing cached."""
|
||||
with self.lock:
|
||||
if key not in self.cache:
|
||||
self.cache[key] = factory()
|
||||
return self.cache[key]
|
||||
|
||||
def drop(self, key: str) -> None:
|
||||
with self.lock:
|
||||
self.cache.pop(key, None)
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobContext:
|
||||
"""What a run gets. `conn` is the run's own autocommit connection as the worker role."""
|
||||
|
||||
conn: Any
|
||||
settings: Settings
|
||||
feed_settings: FeedSettings
|
||||
run_id: int
|
||||
trigger: str
|
||||
started_at: datetime
|
||||
log: logging.Logger
|
||||
stack: Stack
|
||||
degraded: bool = False
|
||||
errors: list[str] = field(default_factory=list)
|
||||
visual_target: str | None = None
|
||||
|
||||
def error(self, text: str, *, degrade: bool = True) -> None:
|
||||
"""Record one error (redacted, clipped). It makes the run degraded unless degrade is False."""
|
||||
if len(self.errors) < MAX_ERRORS:
|
||||
self.errors.extend(clip([text]))
|
||||
if degrade:
|
||||
self.degraded = True
|
||||
|
||||
def degrade(self, reason: str | None = None) -> None:
|
||||
self.degraded = True
|
||||
if reason:
|
||||
self.error(reason)
|
||||
|
||||
@property
|
||||
def tz(self) -> ZoneInfo:
|
||||
return ZoneInfo(self.settings.site_tz)
|
||||
|
||||
def now_site(self) -> datetime:
|
||||
return datetime.now(self.tz)
|
||||
|
||||
def today_site(self) -> date:
|
||||
return self.now_site().date()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JobSpec:
|
||||
name: str
|
||||
run: Callable[[JobContext], Counts]
|
||||
cadence: Cadence # seconds to the next run, or None for "no schedule"
|
||||
pausable: bool # the feed pause skips scheduled fires (requests still run)
|
||||
chain: tuple[str, ...] = () # moved to now after a run that did not fail
|
||||
|
||||
|
||||
JOBS: dict[str, JobSpec] = {}
|
||||
_loaded = False
|
||||
_load_lock = threading.Lock()
|
||||
LAZY = ("visual_feed", "drawing_lookup")
|
||||
|
||||
|
||||
def register(spec: JobSpec) -> JobSpec:
|
||||
JOBS[spec.name] = spec
|
||||
return spec
|
||||
|
||||
|
||||
def load_jobs() -> dict[str, JobSpec]:
|
||||
"""Register every job once and return the registry. A job module that cannot import is logged and left out."""
|
||||
global _loaded
|
||||
with _load_lock:
|
||||
if not _loaded:
|
||||
from . import housekeeping
|
||||
|
||||
register(housekeeping.SPEC)
|
||||
for mod in LAZY:
|
||||
try:
|
||||
m = importlib.import_module(f"{__name__}.{mod}")
|
||||
register(m.SPEC)
|
||||
except ImportError as e:
|
||||
log.warning("job module %s not loaded: %s", mod, redact(str(e)))
|
||||
_loaded = True
|
||||
return JOBS
|
||||
|
||||
|
||||
def seconds_until(now_site: datetime, at: time) -> int:
|
||||
"""Seconds from now_site to the next `at` (site wall clock), at least 1."""
|
||||
target = datetime.combine(now_site.date(), at, tzinfo=now_site.tzinfo)
|
||||
if target <= now_site:
|
||||
target = datetime.combine(now_site.date() + timedelta(days=1), at, tzinfo=now_site.tzinfo)
|
||||
# Aware datetimes sharing a tzinfo subtract as wall clock; through UTC a DST change counts right
|
||||
return max(1, int((target.astimezone(UTC) - now_site.astimezone(UTC)).total_seconds()))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"JOBS",
|
||||
"TRIGGERS",
|
||||
"Cadence",
|
||||
"Counts",
|
||||
"JobContext",
|
||||
"JobSpec",
|
||||
"Stack",
|
||||
"load_jobs",
|
||||
"register",
|
||||
"seconds_until",
|
||||
]
|
||||
@@ -0,0 +1,536 @@
|
||||
"""The drawing-lookup job: look drawings up in M-Files (read-only), copy released PDFs into assets, and attach,
|
||||
flag, stage or open a change order on the packets that use them.
|
||||
|
||||
Per run: the queue (drawings/queue.py) cut to `drawing_budget`; per drawing one `Lookup.status`, a fetch when a
|
||||
released file exists (drawings/fetch.py downloads only when its M-Files identity changed), the `drawing_lookups`
|
||||
row, then `decide` for every live packet that uses the drawing (its own drawing, or the target of its pending feed
|
||||
change order), applied through `feed.writes.system_write`. A busy packet is left for the next run.
|
||||
|
||||
Never writes to M-Files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from ... import audit, policy
|
||||
from ...domain import pipeline
|
||||
from ...drawings import DRAWING_FLAGS, decide, queue
|
||||
from ...drawings import lookup as lookups
|
||||
from ...drawings.fetch import FetchError, FetchResult, fetch
|
||||
from ...drawings.lookup import Found, Lookup
|
||||
from ...feed import changes, writes
|
||||
from . import Counts, JobContext, JobSpec
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
A = policy.Action
|
||||
D = decide
|
||||
|
||||
FEED_ORIGINS = ("visual", "drawing")
|
||||
|
||||
|
||||
class _Stale(Exception):
|
||||
"""The packet changed between the read and the lock; the next run decides again."""
|
||||
|
||||
|
||||
def new_counts() -> Counts:
|
||||
return {
|
||||
"checked": 0,
|
||||
"fetched": 0,
|
||||
"unchanged": 0,
|
||||
"attached": 0,
|
||||
"flagged": {},
|
||||
"lookup_failed": 0,
|
||||
"cos_opened": 0,
|
||||
"cos_reaimed": 0,
|
||||
"staged": 0,
|
||||
"budget_left": 0,
|
||||
"fetch_failed": 0,
|
||||
"busy": 0,
|
||||
"refused": 0,
|
||||
"no_file": 0,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the run
|
||||
|
||||
|
||||
def run(ctx: JobContext) -> Counts:
|
||||
counts = new_counts()
|
||||
fs = ctx.feed_settings
|
||||
budget = int(getattr(fs, "drawing_budget", 60))
|
||||
try:
|
||||
lk = lookups.make(ctx.settings, ctx.stack.cache)
|
||||
except Exception as e: # any failure to open M-Files skips the run
|
||||
ctx.error(f"M-Files could not be opened: {type(e).__name__}: {e}")
|
||||
counts["reason"] = "mfiles unavailable"
|
||||
counts["budget_left"] = budget
|
||||
return counts
|
||||
now = datetime.now(UTC)
|
||||
items = queue.queue(ctx.conn, fs, now, limit=budget)
|
||||
counts["budget_left"] = max(0, budget - len(items))
|
||||
counts["source"] = lk.describe()
|
||||
for item in items:
|
||||
try:
|
||||
_drawing(ctx, lk, item, counts, now)
|
||||
except Exception as e: # one bad drawing must not stop the others
|
||||
log.exception("drawing %s failed", item.dwg)
|
||||
ctx.error(f"drawing {item.dwg}: {type(e).__name__}: {e}")
|
||||
return counts
|
||||
|
||||
|
||||
def _drawing(ctx: JobContext, lk: Lookup, item: queue.QueueItem, counts: Counts, now: datetime) -> None:
|
||||
conn = ctx.conn
|
||||
try:
|
||||
found = lk.status(item.dwg)
|
||||
except Exception as e: # a lookup error is a failed lookup, never a crash
|
||||
found = lookups.failed(item.dwg, f"M-Files lookup failed: {type(e).__name__}: {e}")
|
||||
counts["checked"] += 1
|
||||
fetched: FetchResult | None = None
|
||||
fetch_failed = False
|
||||
if found.ok and found.rev is not None:
|
||||
try:
|
||||
fetched = fetch(conn, ctx.settings, lk, found, dwg=item.dwg)
|
||||
except FetchError as e:
|
||||
fetch_failed = True
|
||||
counts["fetch_failed"] += 1
|
||||
ctx.error(str(e))
|
||||
else:
|
||||
if fetched.downloaded:
|
||||
counts["fetched"] += 1
|
||||
audit.append(
|
||||
conn,
|
||||
action="drawing.fetched",
|
||||
role_snapshot="system",
|
||||
subject_type="asset",
|
||||
subject_id=str(fetched.asset_id),
|
||||
detail={
|
||||
"dwg": item.dwg,
|
||||
"rev": found.rev,
|
||||
"sha256": fetched.sha256.hex(),
|
||||
"reused": fetched.reused,
|
||||
},
|
||||
)
|
||||
if not found.ok:
|
||||
counts["lookup_failed"] += 1
|
||||
ctx.error(f"drawing {item.dwg}: {D.failed_note(found)}")
|
||||
vrev_seen = item.visual_rev if item.co_id is None else own_rev(conn, item.dwg)
|
||||
save_lookup(conn, item, found, fetched, now, fetch_failed=fetch_failed, visual_rev_seen=vrev_seen)
|
||||
scope = [str(d) for d in (getattr(ctx.feed_settings, "departments", None) or [])]
|
||||
for pid, relations in sharing(conn, item.dwg, scope):
|
||||
_packet(ctx, item.dwg, found, fetched, pid, relations, counts)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- drawing_lookups
|
||||
|
||||
|
||||
def save_lookup(
|
||||
conn: Any,
|
||||
item: queue.QueueItem,
|
||||
found: Found,
|
||||
fetched: FetchResult | None,
|
||||
now: datetime,
|
||||
*,
|
||||
fetch_failed: bool,
|
||||
visual_rev_seen: str | None,
|
||||
) -> None:
|
||||
"""Upsert the drawing_lookups row for this lookup.
|
||||
|
||||
A failed lookup keeps everything known before (rev, file, asset) and backs off. A good one stores what M-Files
|
||||
has; the file identity and the asset are stored together, and cleared when there is no released file or the
|
||||
copy failed (so the next run downloads again).
|
||||
"""
|
||||
with conn.transaction():
|
||||
prev = conn.execute("SELECT fail_count FROM drawing_lookups WHERE dwg = %s FOR UPDATE", (item.dwg,)).fetchone()
|
||||
if not found.ok:
|
||||
fails = (int(prev[0]) if prev else 0) + 1
|
||||
conn.execute(
|
||||
"INSERT INTO drawing_lookups (dwg, checked_at, ok, note, fail_count, failing_since, next_check_at,"
|
||||
" visual_rev_seen) VALUES (%(dwg)s, %(now)s, false, %(note)s, %(fails)s, %(now)s, %(next)s, %(vrev)s)"
|
||||
" ON CONFLICT (dwg) DO UPDATE SET checked_at = EXCLUDED.checked_at, ok = false, note = EXCLUDED.note,"
|
||||
" fail_count = EXCLUDED.fail_count,"
|
||||
" failing_since = coalesce(drawing_lookups.failing_since, EXCLUDED.failing_since),"
|
||||
" next_check_at = EXCLUDED.next_check_at, visual_rev_seen = EXCLUDED.visual_rev_seen",
|
||||
{
|
||||
"dwg": item.dwg,
|
||||
"now": now,
|
||||
"note": D.failed_note(found),
|
||||
"fails": fails,
|
||||
"next": now + queue.backoff(fails),
|
||||
"vrev": visual_rev_seen,
|
||||
},
|
||||
)
|
||||
return
|
||||
has_file = fetched is not None
|
||||
conn.execute(
|
||||
"INSERT INTO drawing_lookups (dwg, checked_at, ok, rev, name, obj_id, obj_version, file_id, file_version,"
|
||||
" file_size, base, draft_rev, draft_where, in_drafting, note, fail_count, failing_since, asset_id, sha256,"
|
||||
" visual_rev_seen, next_check_at) VALUES (%(dwg)s, %(now)s, true, %(rev)s, %(name)s, %(obj_id)s,"
|
||||
" %(obj_version)s, %(file_id)s, %(file_version)s, %(file_size)s, %(base)s, %(draft_rev)s,"
|
||||
" %(draft_where)s, %(in_drafting)s, %(note)s, 0, NULL, %(asset_id)s, %(sha)s, %(vrev)s, %(next)s)"
|
||||
" ON CONFLICT (dwg) DO UPDATE SET checked_at = EXCLUDED.checked_at, ok = true, rev = EXCLUDED.rev,"
|
||||
" name = EXCLUDED.name, obj_id = EXCLUDED.obj_id, obj_version = EXCLUDED.obj_version,"
|
||||
" file_id = EXCLUDED.file_id, file_version = EXCLUDED.file_version, file_size = EXCLUDED.file_size,"
|
||||
" base = EXCLUDED.base, draft_rev = EXCLUDED.draft_rev, draft_where = EXCLUDED.draft_where,"
|
||||
" in_drafting = EXCLUDED.in_drafting, note = EXCLUDED.note, fail_count = 0, failing_since = NULL,"
|
||||
" asset_id = EXCLUDED.asset_id, sha256 = EXCLUDED.sha256,"
|
||||
" visual_rev_seen = coalesce(EXCLUDED.visual_rev_seen, drawing_lookups.visual_rev_seen),"
|
||||
" next_check_at = EXCLUDED.next_check_at",
|
||||
{
|
||||
"dwg": item.dwg,
|
||||
"now": now,
|
||||
"rev": found.rev,
|
||||
"name": found.name,
|
||||
"obj_id": found.obj_id if has_file else None,
|
||||
"obj_version": found.obj_version if has_file else None,
|
||||
"file_id": found.file_id if has_file else None,
|
||||
"file_version": found.file_version if has_file else None,
|
||||
"file_size": found.file_size,
|
||||
"base": found.base,
|
||||
"draft_rev": found.draft_rev,
|
||||
"draft_where": found.draft_where or None,
|
||||
"in_drafting": found.in_drafting,
|
||||
"note": found.note or None,
|
||||
"asset_id": fetched.asset_id if fetched else None,
|
||||
"sha": fetched.sha256 if fetched else None,
|
||||
"vrev": visual_rev_seen,
|
||||
"next": now + queue.backoff(1) if fetch_failed else None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- packets
|
||||
|
||||
|
||||
def own_rev(conn: Any, dwg: str) -> str | None:
|
||||
"""The rev a live VISUAL packet with this drawing carries (for visual_rev_seen when the item came from a CO)."""
|
||||
row = conn.execute(
|
||||
"SELECT coalesce(btrim(rev), '') FROM packets WHERE removed_at IS NULL AND source_kind = 'visual'"
|
||||
" AND status <> 'filed' AND btrim(drawing) = %s ORDER BY id LIMIT 1",
|
||||
(dwg,),
|
||||
).fetchone()
|
||||
return None if row is None else str(row[0])
|
||||
|
||||
|
||||
SHARING_SQL = """
|
||||
SELECT p.id, 'own' AS rel FROM packets p
|
||||
WHERE p.removed_at IS NULL AND p.source_kind = 'visual' AND p.status <> 'filed' AND btrim(p.drawing) = %(dwg)s
|
||||
AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[]))
|
||||
UNION
|
||||
SELECT p.id, 'co' AS rel FROM change_orders c JOIN packets p ON p.id = c.packet_id
|
||||
WHERE c.review = 'pending' AND c.origin IN ('visual', 'drawing') AND jsonb_typeof(c.waiting) = 'object'
|
||||
AND btrim(c.waiting->>'dwg') = %(dwg)s AND p.removed_at IS NULL AND p.status NOT IN ('draft', 'filed')
|
||||
AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[]))
|
||||
ORDER BY 1
|
||||
"""
|
||||
|
||||
|
||||
def sharing(conn: Any, dwg: str, depts: list[str]) -> list[tuple[int, set[str]]]:
|
||||
"""Live packets that use the drawing: as their own drawing ('own') or as a pending feed CO's target ('co')."""
|
||||
out: dict[int, set[str]] = {}
|
||||
for pid, rel in conn.execute(SHARING_SQL, {"dwg": dwg, "depts": depts}).fetchall():
|
||||
out.setdefault(int(pid), set()).add(str(rel))
|
||||
return sorted(out.items())
|
||||
|
||||
|
||||
@dataclass
|
||||
class PacketFacts:
|
||||
row: dict[str, Any]
|
||||
visual: dict[str, Any]
|
||||
attached_rev: str | None
|
||||
attached_shas: set[bytes]
|
||||
co: dict[str, Any] | None
|
||||
rejected_sha: bytes | None
|
||||
|
||||
|
||||
def _sha(value: Any) -> bytes | None:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return bytes.fromhex(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def facts(conn: Any, pid: int) -> PacketFacts | None:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
row = cur.execute(
|
||||
"SELECT id, status::text AS status, drawing, rev, drawing_state::text AS drawing_state, flags, visual,"
|
||||
" version FROM packets WHERE id = %s AND removed_at IS NULL",
|
||||
(pid,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
pages = cur.execute(
|
||||
"SELECT pg.rev, a.sha256 FROM packet_pages pg LEFT JOIN assets a ON a.id = pg.asset_id"
|
||||
" WHERE pg.packet_id = %s AND pg.type = 'drawing' AND pg.void_at IS NULL ORDER BY pg.n",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
co = cur.execute(
|
||||
"SELECT id, co_no, origin::text AS origin, waiting, staged FROM change_orders"
|
||||
" WHERE packet_id = %s AND review = 'pending' AND origin IN ('visual', 'drawing') ORDER BY id DESC LIMIT 1",
|
||||
(pid,),
|
||||
).fetchone()
|
||||
rej = cur.execute(
|
||||
"SELECT staged FROM change_orders WHERE packet_id = %s AND origin = 'drawing' AND review = 'rejected'"
|
||||
" ORDER BY id DESC LIMIT 1",
|
||||
(pid,),
|
||||
).fetchone()
|
||||
visual = row["visual"] if isinstance(row["visual"], dict) else {}
|
||||
attached_rev = None
|
||||
if pages:
|
||||
attached_rev = str(pages[0]["rev"] or "").strip()
|
||||
shas = {bytes(p["sha256"]) for p in pages if p["sha256"] is not None}
|
||||
rejected = None
|
||||
if rej is not None and isinstance(rej["staged"], dict):
|
||||
rejected = _sha(rej["staged"].get("sha256"))
|
||||
return PacketFacts(dict(row), dict(visual), attached_rev, shas, dict(co) if co else None, rejected)
|
||||
|
||||
|
||||
def _dwg_rev(value: Any) -> tuple[str, str] | None:
|
||||
if isinstance(value, dict) and value.get("dwg") and value.get("rev") is not None:
|
||||
return (str(value["dwg"]).strip(), str(value["rev"]).strip())
|
||||
return None
|
||||
|
||||
|
||||
def target_for(dwg: str, pf: PacketFacts, *, own: bool) -> D.Target:
|
||||
row, visual, co = pf.row, pf.visual, pf.co
|
||||
waiting = co.get("waiting") if co else None
|
||||
staged = co.get("staged") if co else None
|
||||
co_waiting = _dwg_rev(waiting)
|
||||
co_staged = _dwg_rev(staged)
|
||||
packet_rev = str(row["rev"] or "").strip()
|
||||
rev_ok = None
|
||||
ro = visual.get("rev_ok")
|
||||
if isinstance(ro, dict) and ro.get("rev") is not None and str(ro.get("vis_rev") or "").strip() == packet_rev:
|
||||
rev_ok = str(ro["rev"]).strip()
|
||||
visual_rev = packet_rev if own or co_waiting is None else co_waiting[1]
|
||||
return D.Target(
|
||||
dwg=dwg,
|
||||
visual_rev=visual_rev,
|
||||
attached_rev=pf.attached_rev,
|
||||
attached_sha=next(iter(sorted(pf.attached_shas)), None),
|
||||
status=str(row["status"]),
|
||||
drawing_state=str(row["drawing_state"]),
|
||||
has_hand_drawing=row["drawing_state"] == "hand" or bool(visual.get("manual_dwg")),
|
||||
rev_ok=rev_ok,
|
||||
co_waiting=co_waiting,
|
||||
co_staged=co_staged,
|
||||
co_pending=co is not None,
|
||||
co_staged_sha=_sha(staged.get("sha256")) if isinstance(staged, dict) else None,
|
||||
rejected_sha=pf.rejected_sha,
|
||||
)
|
||||
|
||||
|
||||
def appr_of(t: D.Target, f: Found, now: datetime) -> dict[str, Any]:
|
||||
return {
|
||||
"ok": f.ok,
|
||||
"rev": f.rev,
|
||||
"vrev": t.visual_rev,
|
||||
"name": f.name,
|
||||
"note": f.note or "",
|
||||
"base": f.base,
|
||||
"draft_rev": f.draft_rev,
|
||||
"draft_where": f.draft_where,
|
||||
"in_drafting": f.in_drafting,
|
||||
"at": now.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def _same_appr(a: Any, b: dict[str, Any]) -> bool:
|
||||
if not isinstance(a, dict):
|
||||
return False
|
||||
return {k: v for k, v in a.items() if k != "at"} == {k: v for k, v in b.items() if k != "at"}
|
||||
|
||||
|
||||
def new_flags(flags: list[str], d: D.Decision) -> list[str]:
|
||||
"""Every flag but the drawing ones, then the decision's flag."""
|
||||
out = [f for f in flags if f not in DRAWING_FLAGS]
|
||||
if d.flag and d.flag not in out:
|
||||
out.append(d.flag)
|
||||
return out
|
||||
|
||||
|
||||
def _packet(
|
||||
ctx: JobContext,
|
||||
dwg: str,
|
||||
found: Found,
|
||||
fetched: FetchResult | None,
|
||||
pid: int,
|
||||
relations: set[str],
|
||||
counts: Counts,
|
||||
) -> None:
|
||||
conn = ctx.conn
|
||||
pf = facts(conn, pid)
|
||||
if pf is None:
|
||||
return
|
||||
own = "own" in relations
|
||||
t = target_for(dwg, pf, own=own)
|
||||
same_file = fetched is not None and fetched.sha256 in pf.attached_shas
|
||||
d = D.decide(t, found, same_file=same_file, fetched_sha=fetched.sha256 if fetched else None)
|
||||
if not own and d.kind != D.STAGE_FOR_CO:
|
||||
return # only the CO's target drawing: nothing to do until it can be staged
|
||||
if d.kind in (D.ATTACH, D.OPEN_CO, D.STAGE_FOR_CO) and fetched is None:
|
||||
counts["no_file"] += 1 # the copy failed this run; try again next run
|
||||
return
|
||||
now = datetime.now(UTC)
|
||||
row, visual = pf.row, pf.visual
|
||||
flags = list(row["flags"] or [])
|
||||
flags_after = new_flags(flags, d) if own else flags
|
||||
patch: dict[str, Any] = {}
|
||||
if own:
|
||||
appr = appr_of(t, found, now)
|
||||
if not _same_appr(visual.get("appr"), appr):
|
||||
patch["appr"] = appr
|
||||
if found.ok:
|
||||
confirmed = D.rev_confirmed(t, found)
|
||||
if bool(visual.get("rev_confirmed")) != confirmed:
|
||||
patch["rev_confirmed"] = confirmed
|
||||
state_after = d.drawing_state if own else str(row["drawing_state"])
|
||||
material = (
|
||||
d.kind in (D.ATTACH, D.OPEN_CO, D.STAGE_FOR_CO)
|
||||
or flags_after != flags
|
||||
or state_after != row["drawing_state"]
|
||||
or bool(patch)
|
||||
)
|
||||
if not material:
|
||||
counts["unchanged"] += 1
|
||||
return
|
||||
action = A.VIS_UPDATE_DRAFT if row["status"] == "draft" else A.VIS_OPEN_CO
|
||||
label, note = "Drawing status from M-Files", d.note or None
|
||||
if d.kind == D.ATTACH and fetched is not None:
|
||||
label = "Drawing attached from M-Files"
|
||||
note = "; ".join(x for x in (fetched.name or found.name or "", d.note) if x) or None
|
||||
detail = {"dwg": dwg, "rev": found.rev, "kind": d.kind, "asset_id": str(fetched.asset_id) if fetched else None}
|
||||
try:
|
||||
with writes.system_write(conn, pid, action, label=label, note=note, detail=detail) as sw:
|
||||
if int(sw.packet["version"]) != int(row["version"]):
|
||||
raise _Stale
|
||||
if d.kind == D.ATTACH and fetched is not None:
|
||||
_attach(sw, pid, dwg, found, fetched, d, flags_after, patch, now)
|
||||
counts["attached"] += 1
|
||||
else:
|
||||
_status(sw, pid, state_after, flags_after, patch)
|
||||
if d.flag:
|
||||
fl = counts["flagged"]
|
||||
fl[d.kind] = fl.get(d.kind, 0) + 1
|
||||
if d.kind == D.OPEN_CO and fetched is not None:
|
||||
_open_co(sw, t, dwg, found, fetched, d, counts)
|
||||
elif d.kind == D.STAGE_FOR_CO and fetched is not None and pf.co is not None:
|
||||
_stage(sw, pf.co, dwg, found, fetched)
|
||||
counts["staged"] += 1
|
||||
elif d.kind == D.UNCHANGED:
|
||||
counts["unchanged"] += 1
|
||||
except (_Stale, writes.Busy, writes.Missing):
|
||||
counts["busy"] += 1
|
||||
except writes.Refused as e:
|
||||
counts["refused"] += 1
|
||||
ctx.error(f"packet {pid}: {d.kind} refused: {e}", degrade=False)
|
||||
|
||||
|
||||
def _staged(dwg: str, found: Found, fetched: FetchResult) -> dict[str, Any]:
|
||||
return {
|
||||
"asset_id": str(fetched.asset_id),
|
||||
"name": fetched.name or found.name,
|
||||
"rev": str(found.rev),
|
||||
"dwg": dwg,
|
||||
"sha256": fetched.sha256.hex(),
|
||||
"pages": fetched.pages,
|
||||
}
|
||||
|
||||
|
||||
def _attach(
|
||||
sw: Any,
|
||||
pid: int,
|
||||
dwg: str,
|
||||
found: Found,
|
||||
fetched: FetchResult,
|
||||
d: D.Decision,
|
||||
flags: list[str],
|
||||
patch: dict[str, Any],
|
||||
now: datetime,
|
||||
) -> None:
|
||||
conn = sw.conn
|
||||
pages = max(1, int(fetched.pages))
|
||||
conn.execute("DELETE FROM packet_pages WHERE packet_id = %s AND type = 'drawing'", (pid,))
|
||||
n0 = pipeline.next_counter(conn, pid, "page", pages)
|
||||
rev = str(found.rev)
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO packet_pages (packet_id, n, type, asset_id, pn, dwg, rev) VALUES (%s, %s, 'drawing', %s, %s,"
|
||||
" %s, %s)",
|
||||
[(pid, n0 + i, fetched.asset_id, i + 1, dwg, rev) for i in range(pages)],
|
||||
)
|
||||
vault = {
|
||||
"asset_id": str(fetched.asset_id),
|
||||
"name": fetched.name or found.name,
|
||||
"rev": found.rev,
|
||||
"note": d.note,
|
||||
"at": now.isoformat(),
|
||||
}
|
||||
merged = {**patch, "rev_confirmed": True, "vault": vault}
|
||||
conn.execute(
|
||||
"UPDATE packets SET drawing_state = 'attached', flags = %s, visual = (visual - 'manual_dwg') || %s"
|
||||
" WHERE id = %s",
|
||||
(flags, Jsonb(merged), pid),
|
||||
)
|
||||
sw.changed("pages", "drawing_state", "flags", "visual")
|
||||
|
||||
|
||||
def _status(sw: Any, pid: int, state: str, flags: list[str], patch: dict[str, Any]) -> None:
|
||||
sw.conn.execute(
|
||||
"UPDATE packets SET drawing_state = %s::drawing_state, flags = %s, visual = visual || %s WHERE id = %s",
|
||||
(state, flags, Jsonb(patch), pid),
|
||||
)
|
||||
sw.changed("drawing_state", "flags", "visual")
|
||||
|
||||
|
||||
def _open_co(sw: Any, t: D.Target, dwg: str, found: Found, fetched: FetchResult, d: D.Decision, counts: Counts) -> None:
|
||||
vis = {
|
||||
"from_dwg": dwg,
|
||||
"to_dwg": dwg,
|
||||
"dwg": dwg,
|
||||
"from_rev": t.attached_rev,
|
||||
"to_rev": str(found.rev),
|
||||
"dwg_change": True,
|
||||
"changes": [d.note],
|
||||
}
|
||||
res = changes.open_or_reaim(
|
||||
sw,
|
||||
origin="drawing",
|
||||
reason=d.note,
|
||||
vis=vis,
|
||||
waiting=None,
|
||||
staged=_staged(dwg, found, fetched),
|
||||
target_fp=None,
|
||||
minor=False,
|
||||
)
|
||||
# open_or_reaim writes the packet_log label itself
|
||||
counts["cos_opened" if res.fresh else "cos_reaimed"] += 1
|
||||
|
||||
|
||||
def _stage(sw: Any, co: dict[str, Any], dwg: str, found: Found, fetched: FetchResult) -> None:
|
||||
staged = _staged(dwg, found, fetched)
|
||||
sw.conn.execute(
|
||||
"UPDATE change_orders SET staged = %s, waiting = NULL, row_version = row_version + 1"
|
||||
" WHERE id = %s AND review = 'pending'",
|
||||
(Jsonb(staged), co["id"]),
|
||||
)
|
||||
sw.changed("change_orders")
|
||||
sw.log(f"Drawing for change order {co['co_no']} ready for Clerical", staged["name"])
|
||||
|
||||
|
||||
def _cadence(fs: Any, now: datetime) -> int | None:
|
||||
return int(fs.drawing_interval_minutes) * 60
|
||||
|
||||
|
||||
SPEC = JobSpec(name="drawing-lookup", run=run, cadence=_cadence, pausable=True)
|
||||
|
||||
__all__ = ["SPEC", "new_flags", "run", "save_lookup", "sharing", "target_for"]
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Housekeeping, nightly at 03:30 site time: prune the VISUAL change rows and gone cache rows, old job runs and
|
||||
old claimed run requests. It never touches packets, the audit log or anything append-only."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, time
|
||||
|
||||
from ...feed.config import FeedSettings
|
||||
from . import Counts, JobContext, JobSpec, seconds_until
|
||||
|
||||
AT = time(3, 30)
|
||||
CACHE_DAYS = 90
|
||||
RUNS_DAYS = 180
|
||||
REQUESTS_DAYS = 30
|
||||
|
||||
STEPS: tuple[tuple[str, str, int], ...] = (
|
||||
("row_changes", "DELETE FROM visual_row_changes WHERE at < now() - make_interval(days => %s)", CACHE_DAYS),
|
||||
(
|
||||
"material_changes",
|
||||
"DELETE FROM visual_material_changes WHERE at < now() - make_interval(days => %s)",
|
||||
CACHE_DAYS,
|
||||
),
|
||||
(
|
||||
"gone_rows",
|
||||
"DELETE FROM visual_rows WHERE gone_batch IN"
|
||||
" (SELECT id FROM visual_batches WHERE started_at < now() - make_interval(days => %s))",
|
||||
CACHE_DAYS,
|
||||
),
|
||||
(
|
||||
"gone_materials",
|
||||
"DELETE FROM visual_materials WHERE gone_batch IN"
|
||||
" (SELECT id FROM visual_batches WHERE started_at < now() - make_interval(days => %s))",
|
||||
CACHE_DAYS,
|
||||
),
|
||||
(
|
||||
"job_runs",
|
||||
"DELETE FROM job_runs WHERE started_at < now() - make_interval(days => %s) AND status <> 'running'",
|
||||
RUNS_DAYS,
|
||||
),
|
||||
(
|
||||
"job_requests",
|
||||
"DELETE FROM job_requests WHERE claimed_at IS NOT NULL AND claimed_at < now() - make_interval(days => %s)",
|
||||
REQUESTS_DAYS,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def run(ctx: JobContext) -> Counts:
|
||||
counts: Counts = {}
|
||||
for name, sql, days in STEPS:
|
||||
with ctx.conn.transaction():
|
||||
counts[name] = ctx.conn.execute(sql, (days,)).rowcount
|
||||
return counts
|
||||
|
||||
|
||||
def cadence(_: FeedSettings, now_site: datetime) -> int:
|
||||
return seconds_until(now_site, AT)
|
||||
|
||||
|
||||
SPEC = JobSpec(name="housekeeping", run=run, cadence=cadence, pausable=False)
|
||||
@@ -0,0 +1,141 @@
|
||||
"""The visual-feed job: read VISUAL (or the fixture files), record the read in the cache, build the candidates
|
||||
from the cache and apply them.
|
||||
|
||||
Per run: the feed settings; the tracked keys (every work order of a live VISUAL packet that is not filed, plus the
|
||||
go-live rows Clerical pulled in); the window from today in site time; one read through the source (labor at most
|
||||
every 30 minutes); `cache.write_batch`; `cache.rows_for_apply`; `candidates.build`; `apply.apply`. The run is
|
||||
degraded when a read section failed or the batch is suspect.
|
||||
|
||||
VisualKit: one `Visual.from_env()` per run, closed after, and the stack guard checked against its settings before
|
||||
the first read. Read-only: VisualKit enforces it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from ... import guard
|
||||
from ...feed import apply as feed_apply
|
||||
from ...feed import cache, candidates, config
|
||||
from ...feed.config import FeedSettings
|
||||
from ...feed.source import FeedSource, FixtureSource, VisualKitSource
|
||||
from ...feed.types import FeedData, WoKey
|
||||
from . import Counts, JobContext, JobSpec
|
||||
|
||||
LABOR_EVERY = timedelta(minutes=30)
|
||||
|
||||
|
||||
def tracked_keys(conn: Any) -> list[WoKey]:
|
||||
"""Work orders of live VISUAL packets that are not filed, plus pulled-in go-live rows."""
|
||||
rows = conn.execute(
|
||||
"SELECT DISTINCT w.wo_base, w.lot, w.split, w.sub FROM packet_work_orders w JOIN packets p"
|
||||
" ON p.id = w.packet_id WHERE p.removed_at IS NULL AND p.status <> 'filed' AND p.source_kind = 'visual'"
|
||||
" UNION SELECT wo_base, lot, split, sub FROM visual_ignored_wos WHERE pulled_in_at IS NOT NULL"
|
||||
).fetchall()
|
||||
return sorted({WoKey(r[0], r[1], r[2], r[3]) for r in rows}, key=lambda k: k.as_tuple())
|
||||
|
||||
|
||||
def resource_map(conn: Any) -> dict[str, str]:
|
||||
return {r[0]: r[1] for r in conn.execute("SELECT resource, target FROM visual_resource_map")}
|
||||
|
||||
|
||||
def _labor_due(conn: Any, fs: FeedSettings) -> bool:
|
||||
if not fs.labor_tickets:
|
||||
return False
|
||||
last = cache.last_labor_read(conn)
|
||||
return last is None or datetime.now(UTC) - last >= LABOR_EVERY
|
||||
|
||||
|
||||
def _read(source: FeedSource, ctx: JobContext, tracked: list[WoKey], window: tuple[Any, Any], labor: bool) -> FeedData:
|
||||
return source.read(
|
||||
site=ctx.settings.site_id,
|
||||
since=window[0],
|
||||
until=window[1],
|
||||
tracked=tracked,
|
||||
with_materials=True,
|
||||
with_orders=True,
|
||||
with_labor=labor,
|
||||
labor_since=None,
|
||||
)
|
||||
|
||||
|
||||
def run(ctx: JobContext) -> Counts:
|
||||
conn = ctx.conn
|
||||
fs = ctx.feed_settings
|
||||
today = ctx.today_site()
|
||||
window = config.window(fs, today)
|
||||
tracked = tracked_keys(conn)
|
||||
labor_due = _labor_due(conn, fs)
|
||||
|
||||
if ctx.settings.feed_source == "fixture":
|
||||
fixture = FixtureSource(ctx.settings.fixture_dir)
|
||||
describe = fixture.describe()
|
||||
data = _read(fixture, ctx, tracked, window, labor_due)
|
||||
else:
|
||||
from visualkit import Visual
|
||||
|
||||
with Visual.from_env() as v:
|
||||
# The stack guard against this run's own VisualKit settings, before the first read.
|
||||
result = guard.check_stack(ctx.settings, v.settings)
|
||||
ctx.visual_target = ctx.visual_target or result.target
|
||||
vk = VisualKitSource(v)
|
||||
describe = vk.describe()
|
||||
data = _read(vk, ctx, tracked, window, labor_due)
|
||||
for e in data.errors:
|
||||
ctx.error(e)
|
||||
|
||||
batch = cache.write_batch(conn, ctx.run_id, data, describe, window)
|
||||
if batch.suspect:
|
||||
ctx.degrade(f"suspect read: {batch.reason}; nothing marked gone")
|
||||
|
||||
in_play = {r.key for r in data.rows} | set(tracked)
|
||||
rows, mats, mat_legs = cache.rows_for_apply(conn, in_play)
|
||||
if data.material_legs:
|
||||
mat_legs |= data.material_legs
|
||||
built = candidates.build(
|
||||
rows,
|
||||
mats,
|
||||
mat_legs,
|
||||
resource_map=resource_map(conn),
|
||||
window=window,
|
||||
golive=fs.golive_date,
|
||||
departments=fs.departments,
|
||||
tracked_keys=set(tracked),
|
||||
today=today,
|
||||
)
|
||||
bases = {r.wo for r in rows}
|
||||
projects = {candidates.project_of(b) for b in bases} | {c.project for c in built.candidates.values()}
|
||||
orders = cache.orders(conn, bases=bases | projects, projects=projects)
|
||||
counts = feed_apply.apply(
|
||||
conn,
|
||||
ctx=ctx,
|
||||
batch=batch,
|
||||
feed_settings=fs,
|
||||
built=built,
|
||||
orders=orders,
|
||||
labor=cache.labor_hours(conn),
|
||||
today=today,
|
||||
seen={r.key: r.st for r in data.rows},
|
||||
missing=() if batch.suspect else data.tracked_missing,
|
||||
)
|
||||
counts["wos_window"] = data.window_count
|
||||
counts["wos_tracked"] = len(tracked)
|
||||
counts["rows"] = len(data.rows)
|
||||
counts["materials"] = len(data.materials) if data.materials is not None else 0
|
||||
counts["orders"] = int(batch.counts.get("orders", 0))
|
||||
counts["labor_legs"] = int(batch.counts.get("labor_legs", 0))
|
||||
counts["batch_id"] = batch.id
|
||||
if batch.suspect:
|
||||
counts["suspect_reason"] = batch.reason
|
||||
cache.finish_batch(conn, batch.id, counts)
|
||||
return counts
|
||||
|
||||
|
||||
def _cadence(fs: FeedSettings, now_site: datetime) -> int | None:
|
||||
return config.interval_seconds(fs, now_site)
|
||||
|
||||
|
||||
SPEC = JobSpec(name="visual-feed", run=run, cadence=_cadence, pausable=True, chain=("drawing-lookup",))
|
||||
|
||||
__all__ = ["SPEC", "resource_map", "run", "tracked_keys"]
|
||||
Reference in New Issue
Block a user