Phase 3: VISUAL feed and M-Files drawings in the worker
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,226 @@
|
||||
"""Feed settings: one `settings` row with key `feed`, a JSON object read by the worker on every fire.
|
||||
|
||||
Unknown keys are ignored on load and missing keys take the default, so the row may be partial. A save
|
||||
validates, bumps the row version, writes an audit row and NOTIFYs `ps_settings` so the worker reschedules.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import date, datetime, time, timedelta
|
||||
from typing import Any
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator
|
||||
|
||||
from .. import audit
|
||||
|
||||
KEY = "feed"
|
||||
SETTINGS_CHANNEL = "ps_settings"
|
||||
_HHMM = re.compile(r"^([01][0-9]|2[0-3]):[0-5][0-9]$")
|
||||
_DEPT = re.compile(r"^[A-Za-z0-9]{1,10}$")
|
||||
|
||||
|
||||
class SettingsError(ValueError):
|
||||
"""A patch the model refuses. `errors` holds one short line per problem."""
|
||||
|
||||
def __init__(self, errors: list[str]):
|
||||
super().__init__("; ".join(errors))
|
||||
self.errors = errors
|
||||
|
||||
|
||||
class FeedSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
interval_shop_minutes: int = Field(default=5, ge=1, le=1440)
|
||||
interval_off_minutes: int = Field(default=30, ge=1, le=1440)
|
||||
shop_days: list[int] = Field(default_factory=lambda: [0, 1, 2, 3, 4]) # Monday = 0, site time
|
||||
shop_start: str = "06:00" # HH:MM site time
|
||||
shop_end: str = "18:00"
|
||||
window_back_days: int = Field(default=14, ge=0, le=120)
|
||||
window_ahead_days: int = Field(default=30, ge=0, le=365)
|
||||
golive_date: date | None = None # work orders released before this date are ignored
|
||||
departments: list[str] = Field(default_factory=list) # shop codes; empty = every department the map routes to
|
||||
drawing_recheck_hours: int = Field(default=4, ge=1, le=168)
|
||||
drawing_budget: int = Field(default=60, ge=1, le=500)
|
||||
drawing_interval_minutes: int = Field(default=30, ge=1, le=1440)
|
||||
labor_tickets: bool = True # read labor tickets (at most every 30 minutes)
|
||||
auto_release: bool = True
|
||||
auto_release_max: int = Field(default=25, ge=0, le=500)
|
||||
paused: bool = False # stops scheduled feed, drawing and auto-release runs; "run now" still works
|
||||
|
||||
@field_validator("shop_days")
|
||||
@classmethod
|
||||
def _days(cls, v: list[int]) -> list[int]:
|
||||
if any(d < 0 or d > 6 for d in v):
|
||||
raise ValueError("shop days are 0 (Monday) to 6 (Sunday)")
|
||||
return sorted(set(v))
|
||||
|
||||
@field_validator("shop_start", "shop_end")
|
||||
@classmethod
|
||||
def _hhmm(cls, v: str) -> str:
|
||||
v = v.strip()
|
||||
if not _HHMM.match(v):
|
||||
raise ValueError("a time is HH:MM, 00:00 to 23:59")
|
||||
return v
|
||||
|
||||
@field_validator("departments")
|
||||
@classmethod
|
||||
def _depts(cls, v: list[str]) -> list[str]:
|
||||
out: list[str] = []
|
||||
for d in v:
|
||||
d = str(d).strip()
|
||||
if not _DEPT.match(d):
|
||||
raise ValueError("a department is its short code, like 03")
|
||||
if d not in out:
|
||||
out.append(d)
|
||||
return out
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _hours(self) -> FeedSettings:
|
||||
if self.shop_start == self.shop_end:
|
||||
raise ValueError("shop_start and shop_end must differ")
|
||||
return self
|
||||
|
||||
|
||||
FIELDS = tuple(FeedSettings.model_fields)
|
||||
|
||||
|
||||
def _clock(hhmm: str) -> time:
|
||||
h, m = hhmm.split(":")
|
||||
return time(int(h), int(m))
|
||||
|
||||
|
||||
def _errors(e: ValidationError) -> list[str]:
|
||||
out = []
|
||||
for err in e.errors():
|
||||
loc = ".".join(str(x) for x in err.get("loc", ()))
|
||||
msg = str(err.get("msg", "invalid")).removeprefix("Value error, ")
|
||||
out.append(f"{loc}: {msg}" if loc else msg)
|
||||
return out
|
||||
|
||||
|
||||
def from_value(value: Any) -> FeedSettings:
|
||||
"""The model from a stored value. A bad stored value keeps its good fields; the rest take the default."""
|
||||
if not isinstance(value, dict):
|
||||
return FeedSettings()
|
||||
try:
|
||||
return FeedSettings.model_validate(value)
|
||||
except ValidationError:
|
||||
good: dict[str, Any] = {}
|
||||
for k in FIELDS:
|
||||
if k not in value:
|
||||
continue
|
||||
try:
|
||||
FeedSettings.model_validate({**good, k: value[k]})
|
||||
except ValidationError:
|
||||
continue
|
||||
good[k] = value[k]
|
||||
return FeedSettings.model_validate(good)
|
||||
|
||||
|
||||
def load(conn: Any) -> FeedSettings:
|
||||
"""The current feed settings; the defaults when the row is missing."""
|
||||
row = conn.execute("SELECT value FROM settings WHERE key = %s", (KEY,)).fetchone()
|
||||
return from_value(row[0] if row else None)
|
||||
|
||||
|
||||
def load_versioned(conn: Any) -> tuple[FeedSettings, int, Any, Any]:
|
||||
"""(settings, version, updated_at, updated_by); version 0 when the row is missing."""
|
||||
row = conn.execute("SELECT value, version, updated_at, updated_by FROM settings WHERE key = %s", (KEY,)).fetchone()
|
||||
if row is None:
|
||||
return FeedSettings(), 0, None, None
|
||||
return from_value(row[0]), int(row[1]), row[2], row[3]
|
||||
|
||||
|
||||
def seed(conn: Any) -> bool:
|
||||
"""Write the defaults when the row is missing. True when it wrote."""
|
||||
cur = conn.execute(
|
||||
"INSERT INTO settings (key, value, version, updated_at) VALUES (%s, %s, 1, now()) ON CONFLICT (key) DO NOTHING",
|
||||
(KEY, Jsonb(FeedSettings().model_dump(mode="json"))),
|
||||
)
|
||||
return bool(cur.rowcount)
|
||||
|
||||
|
||||
def validate_patch(current: FeedSettings, patch: dict[str, Any]) -> FeedSettings:
|
||||
"""The settings after a partial change. Raises SettingsError, also for a key that is not a setting."""
|
||||
unknown = sorted(k for k in patch if k not in FIELDS)
|
||||
if unknown:
|
||||
raise SettingsError([f"{k}: not a feed setting" for k in unknown])
|
||||
try:
|
||||
return FeedSettings.model_validate({**current.model_dump(mode="json"), **patch})
|
||||
except ValidationError as e:
|
||||
raise SettingsError(_errors(e)) from None
|
||||
|
||||
|
||||
def save(conn: Any, *, who: Any, patch: dict[str, Any]) -> FeedSettings:
|
||||
"""Validate and store a partial change. Raises SettingsError. A patch that changes nothing writes nothing.
|
||||
|
||||
`who` has user_id, session_id, role_snapshot, ip and it_reason (domain.projects.Who).
|
||||
"""
|
||||
with conn.transaction():
|
||||
row = conn.execute("SELECT value FROM settings WHERE key = %s FOR UPDATE", (KEY,)).fetchone()
|
||||
current = from_value(row[0] if row else None)
|
||||
new = validate_patch(current, patch)
|
||||
before, after = current.model_dump(mode="json"), new.model_dump(mode="json")
|
||||
changed = {k: {"from": before[k], "to": after[k]} for k in FIELDS if before[k] != after[k]}
|
||||
if not changed and row is not None:
|
||||
return new
|
||||
version = conn.execute(
|
||||
"INSERT INTO settings (key, value, version, updated_at, updated_by) VALUES (%s, %s, 1, now(), %s)"
|
||||
" ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1,"
|
||||
" updated_at = now(), updated_by = EXCLUDED.updated_by RETURNING version",
|
||||
(KEY, Jsonb(after), who.user_id),
|
||||
).fetchone()[0]
|
||||
audit.append(
|
||||
conn,
|
||||
action="feed.settings_changed",
|
||||
user_id=who.user_id,
|
||||
session_id=getattr(who, "session_id", None),
|
||||
role_snapshot=getattr(who, "role_snapshot", None),
|
||||
subject_type="settings",
|
||||
subject_id=KEY,
|
||||
detail={"changed": changed},
|
||||
it_reason=getattr(who, "it_reason", None),
|
||||
ip=getattr(who, "ip", None),
|
||||
)
|
||||
notify_settings(conn, int(version))
|
||||
return new
|
||||
|
||||
|
||||
def notify_settings(conn: Any, version: int | None = None) -> None:
|
||||
"""Tell the worker the settings changed. Inside a transaction it is delivered on commit."""
|
||||
payload = json.dumps({"key": KEY, "version": version}, separators=(",", ":"))
|
||||
conn.execute("SELECT ps_notify(%s, %s)", (SETTINGS_CHANNEL, payload))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- derived values (pure)
|
||||
|
||||
|
||||
def window(settings: FeedSettings, today: date) -> tuple[date, date]:
|
||||
"""(since, until) for the released read, from the site-local date."""
|
||||
return today - timedelta(days=settings.window_back_days), today + timedelta(days=settings.window_ahead_days)
|
||||
|
||||
|
||||
def in_shop_hours(settings: FeedSettings, now_site: datetime) -> bool:
|
||||
"""True on a shop day from shop_start up to shop_end, site time.
|
||||
|
||||
An end before the start spans midnight; the hours after midnight belong to the day the shift started.
|
||||
"""
|
||||
start, end = _clock(settings.shop_start), _clock(settings.shop_end)
|
||||
t = now_site.time().replace(tzinfo=None)
|
||||
day = now_site.weekday()
|
||||
if start < end:
|
||||
return day in settings.shop_days and start <= t < end
|
||||
if t >= start:
|
||||
return day in settings.shop_days
|
||||
if t < end:
|
||||
return (day - 1) % 7 in settings.shop_days
|
||||
return False
|
||||
|
||||
|
||||
def interval_seconds(settings: FeedSettings, now_site: datetime) -> int:
|
||||
"""The feed's cadence now: the shop interval inside shop hours, the off interval outside."""
|
||||
minutes = settings.interval_shop_minutes if in_shop_hours(settings, now_site) else settings.interval_off_minutes
|
||||
return minutes * 60
|
||||
Reference in New Issue
Block a user