Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
2026-10-16 14:45:38 -06:00
parent 16950055e3
commit 591fe90076
103 changed files with 16063 additions and 151 deletions
@@ -0,0 +1,315 @@
+"""The worker's timing, pause, redaction and run bookkeeping (run_job against PostgreSQL with a fake job)."""
+
+from __future__ import annotations
+
+import threading
+from datetime import date, datetime, timedelta
+from typing import Any
+from zoneinfo import ZoneInfo
+
+import pytest
+from pshelpers import app_conn, needs_pg
+from psycopg.types.json import Jsonb
+
+from packetstation.feed import config
+from packetstation.feed.config import FeedSettings
+from packetstation.settings import Settings
+from packetstation.worker import runs
+from packetstation.worker.jobs import JobContext, JobSpec, load_jobs, seconds_until
+from packetstation.worker.redaction import clip, redact
+from packetstation.worker.scheduler import Scheduler, next_due
+
+TZ = ZoneInfo("America/Chicago")
+FRI = date(2026, 10, 16)  # a Friday
+SAT = date(2026, 10, 17)
+
+
+def at(d: date, hh: int, mm: int = 0) -> datetime:
+    return datetime(d.year, d.month, d.day, hh, mm, tzinfo=TZ)
+
+
+# ---------------------------------------------------------------- cadence (pure)
+
+
+@pytest.mark.parametrize(
+    "when,inside",
+    [
+        (at(FRI, 6, 0), True),
+        (at(FRI, 12, 30), True),
+        (at(FRI, 17, 59), True),
+        (at(FRI, 18, 0), False),
+        (at(FRI, 5, 59), False),
+        (at(SAT, 12, 0), False),
+        (at(date(2026, 10, 18), 12, 0), False),  # Sunday
+        (at(date(2026, 10, 19), 7, 0), True),  # Monday
+    ],
+)
+def test_in_shop_hours(when: datetime, inside: bool) -> None:
+    fs = FeedSettings()
+    assert config.in_shop_hours(fs, when) is inside
+    assert config.interval_seconds(fs, when) == (300 if inside else 1800)
+
+
+def test_interval_follows_settings() -> None:
+    fs = FeedSettings(
+        interval_shop_minutes=2, interval_off_minutes=45, shop_days=[5], shop_start="08:00", shop_end="12:00"
+    )
+    assert config.interval_seconds(fs, at(SAT, 9)) == 120
+    assert config.interval_seconds(fs, at(FRI, 9)) == 45 * 60
+    assert config.interval_seconds(fs, at(SAT, 12)) == 45 * 60
+
+
+def test_overnight_shift_belongs_to_the_day_it_started() -> None:
+    fs = FeedSettings(shop_start="22:00", shop_end="06:00", shop_days=[4])
+    assert config.in_shop_hours(fs, at(FRI, 23))
+    assert config.in_shop_hours(fs, at(SAT, 3))  # Friday's shift
+    assert not config.in_shop_hours(fs, at(SAT, 23))
+
+
+def test_window() -> None:
+    fs = FeedSettings(window_back_days=14, window_ahead_days=30)
+    assert config.window(fs, FRI) == (FRI - timedelta(days=14), FRI + timedelta(days=30))
+
+
+def test_settings_validation() -> None:
+    with pytest.raises(config.SettingsError):
+        config.validate_patch(FeedSettings(), {"interval_shop_minutes": 0})
+    with pytest.raises(config.SettingsError):
+        config.validate_patch(FeedSettings(), {"shop_start": "25:00"})
+    with pytest.raises(config.SettingsError):
+        config.validate_patch(FeedSettings(), {"no_such_key": 1})
+    assert config.from_value({"interval_shop_minutes": 7, "unknown": 1}).interval_shop_minutes == 7
+    assert config.from_value({"interval_shop_minutes": -3, "paused": True}).paused is True  # good fields kept
+
+
+def _spec(name: str = "fake", run: Any = None, pausable: bool = True, cadence: Any = None) -> JobSpec:
+    return JobSpec(
+        name=name,
+        run=run or (lambda ctx: {"done": 1}),
+        cadence=cadence or (lambda fs, now: config.interval_seconds(fs, now)),
+        pausable=pausable,
+    )
+
+
+def test_registered_jobs_and_cadences() -> None:
+    jobs = load_jobs()
+    assert {"housekeeping", "visual-feed", "drawing-lookup"} <= set(jobs)
+    fs = FeedSettings(drawing_interval_minutes=20)
+    assert jobs["visual-feed"].cadence(fs, at(FRI, 10)) == 300
+    assert jobs["visual-feed"].cadence(fs, at(SAT, 10)) == 1800
+    assert jobs["drawing-lookup"].cadence(fs, at(FRI, 10)) == 1200
+    assert "drawing-lookup" in jobs["visual-feed"].chain
+    assert jobs["visual-feed"].pausable and jobs["drawing-lookup"].pausable
+    hk = jobs["housekeeping"].cadence(fs, at(FRI, 3, 0))
+    assert hk == 30 * 60  # 03:30 site time
+
+
+def test_next_due() -> None:
+    spec = _spec()
+    fs = FeedSettings()
+    now = at(FRI, 10)
+    assert next_due(spec, fs, now) == now + timedelta(minutes=5)
+    assert next_due(spec, fs, now, now - timedelta(minutes=2)) == now + timedelta(minutes=3)
+    assert next_due(spec, fs, now, now - timedelta(hours=1)) == now  # overdue: now, never in the past
+    assert next_due(_spec(cadence=lambda fs, n: None), fs, now) is None
+
+
+def test_seconds_until() -> None:
+    assert seconds_until(at(FRI, 3, 0), datetime.min.time().replace(hour=3, minute=30)) == 1800
+    assert seconds_until(at(FRI, 4, 0), datetime.min.time().replace(hour=3, minute=30)) == 23 * 3600 + 1800
+
+
+# ---------------------------------------------------------------- redaction (pure)
+
+
+@pytest.mark.parametrize(
+    "text,secret",
+    [
+        ("connect failed: password=Hunter2xyz host=vsql", "Hunter2xyz"),
+        ("odbc PWD=S3cret!!;UID=visualkit_ro", "S3cret!!"),
+        ("Authorization: Bearer abc.def.ghi", "abc.def.ghi"),
+        ("X-Authentication: tok123456", "tok123456"),
+        ("postgresql://ps_app:pa55word@db/ps", "pa55word"),
+    ],
+)
+def test_redact_patterns(text: str, secret: str) -> None:
+    out = redact(text)
+    assert secret not in out and "[redacted]" in out
+
+
+def test_redact_environment_passwords(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
+    monkeypatch.setenv("MFILES_PASSWORD", "vault-pass-0001")
+    f = tmp_path / "visual_pw"
+    f.write_text("visual-pass-0002\n")
+    monkeypatch.setenv("VISUAL_PASSWORD_FILE", str(f))
+    out = redact("login vault-pass-0001 failed; then visual-pass-0002 failed")
+    assert "vault-pass-0001" not in out and "visual-pass-0002" not in out
+    assert runs.redact is redact or "vault-pass-0001" not in runs.redact("x vault-pass-0001")
+
+
+def test_clip() -> None:
+    out = clip([f"error {i} " + "x" * 600 for i in range(30)])
+    assert len(out) == 20 and all(len(e) <= 500 for e in out)
+
+
+# ---------------------------------------------------------------- run_job (PostgreSQL)
+
+
+def _sched(settings: Settings, spec: JobSpec) -> Scheduler:
+    return Scheduler(settings, build="pytest", jobs={spec.name: spec})
+
+
+def _runs(settings: Settings, job: str = "fake") -> list[tuple[Any, ...]]:
+    with app_conn(settings) as conn:
+        return [
+            tuple(r)
+            for r in conn.execute(
+                "SELECT status, trigger, counts, errors, finished_at IS NOT NULL, build_version FROM job_runs"
+                " WHERE job = %s ORDER BY id",
+                (job,),
+            ).fetchall()
+        ]
+
+
+@needs_pg
+def test_run_job_records_a_run(pg_settings: Settings) -> None:
+    seen: list[JobContext] = []
+
+    def run(ctx: JobContext) -> dict[str, Any]:
+        seen.append(ctx)
+        assert ctx.conn.execute("SELECT current_setting('application_name')").fetchone()[0] == "ps-worker"
+        return {"done": 3}
+
+    row = runs.run_job(_sched(pg_settings, _spec(run=run)), "fake", "schedule")
+    assert row["status"] == "ok" and row["counts"] == {"done": 3} and row["trigger"] == "schedule"
+    assert seen and seen[0].run_id == row["id"] and seen[0].trigger == "schedule"
+    assert _runs(pg_settings) == [("ok", "schedule", {"done": 3}, [], True, "pytest")]
+
+
+@needs_pg
+def test_run_job_degraded_and_failed(pg_settings: Settings) -> None:
+    def degraded(ctx: JobContext) -> dict[str, Any]:
+        ctx.error("orders read failed: password=hunter22")
+        return {"n": 1}
+
+    def boom(ctx: JobContext) -> dict[str, Any]:
+        raise RuntimeError("PWD=topsecret9 broke")
+
+    row = runs.run_job(_sched(pg_settings, _spec(run=degraded)), "fake", "schedule")
+    assert row["status"] == "degraded" and row["errors"] and "hunter22" not in row["errors"][0]
+    row = runs.run_job(_sched(pg_settings, _spec(run=boom)), "fake", "schedule")
+    assert row["status"] == "failed" and len(row["errors"]) == 1 and "topsecret9" not in row["errors"][0]
+    assert row["finished_at"] is not None
+
+
+@needs_pg
+def test_ten_requests_one_run(pg_settings: Settings) -> None:
+    with app_conn(pg_settings) as conn:
+        for _ in range(10):
+            conn.execute("INSERT INTO job_requests (job) VALUES ('fake')")
+    calls: list[int] = []
+    row = runs.run_job(_sched(pg_settings, _spec(run=lambda ctx: calls.append(1) or {})), "fake", "schedule")
+    assert calls == [1] and row["trigger"] == "request"
+    with app_conn(pg_settings) as conn:
+        claimed = conn.execute(
+            "SELECT count(*), count(DISTINCT claimed_run_id) FROM job_requests WHERE claimed_at IS NOT NULL"
+        ).fetchone()
+        assert claimed == (10, 1)
+        assert conn.execute("SELECT count(*) FROM job_requests WHERE claimed_at IS NULL").fetchone()[0] == 0
+    assert len(_runs(pg_settings)) == 1
+
+
+@needs_pg
+def test_skips_when_the_lock_is_held(pg_settings: Settings) -> None:
+    from packetstation import db
+
+    calls: list[int] = []
+    with db.connect_as("worker", pg_settings, autocommit=True) as holder:
+        assert holder.execute("SELECT pg_try_advisory_lock(7310, hashtext('fake'))").fetchone()[0]
+        row = runs.run_job(_sched(pg_settings, _spec(run=lambda ctx: calls.append(1) or {})), "fake", "schedule")
+    assert calls == [] and row["status"] == "skipped" and row["counts"] == {"reason": "already running"}
+    row = runs.run_job(_sched(pg_settings, _spec(run=lambda ctx: calls.append(1) or {})), "fake", "schedule")
+    assert calls == [1] and row["status"] == "ok"
+
+
+@needs_pg
+def test_concurrent_runs_one_skips(pg_settings: Settings) -> None:
+    started, release = threading.Event(), threading.Event()
+
+    def slow(ctx: JobContext) -> dict[str, Any]:
+        started.set()
+        release.wait(10)
+        return {}
+
+    sch = _sched(pg_settings, _spec(run=slow))
+    t = threading.Thread(target=runs.run_job, args=(sch, "fake", "schedule"))
+    t.start()
+    try:
+        assert started.wait(10)
+        assert sch.running() == ["fake"]
+        row = runs.run_job(_sched(pg_settings, _spec(run=lambda ctx: {})), "fake", "schedule")
+        assert row["status"] == "skipped"
+    finally:
+        release.set()
+        t.join(10)
+    assert sorted(r[0] for r in _runs(pg_settings)) == ["ok", "skipped"]
+
+
+def _pause(settings: Settings, paused: bool) -> None:
+    with app_conn(settings) as conn:
+        conn.execute(
+            "INSERT INTO settings (key, value) VALUES ('feed', %s)"
+            " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value",
+            (Jsonb({"paused": paused}),),
+        )
+
+
+@needs_pg
+def test_pause_skips_scheduled_fires_but_requests_run(pg_settings: Settings) -> None:
+    _pause(pg_settings, True)
+    calls: list[str] = []
+    spec = _spec(run=lambda ctx: calls.append(ctx.trigger) or {})
+    row = runs.run_job(_sched(pg_settings, spec), "fake", "schedule")
+    assert row["status"] == "skipped" and row["counts"] == {"reason": "paused"} and calls == []
+    with app_conn(pg_settings) as conn:
+        conn.execute("INSERT INTO job_requests (job) VALUES ('fake')")
+    row = runs.run_job(_sched(pg_settings, spec), "fake", "schedule")
+    assert row["status"] == "ok" and calls == ["request"]
+    # a job that is not pausable runs on schedule while paused
+    row = runs.run_job(
+        _sched(pg_settings, _spec(run=lambda ctx: calls.append("np") or {}, pausable=False)), "fake", "schedule"
+    )
+    assert row["status"] == "ok" and calls[-1] == "np"
+    _pause(pg_settings, False)
+    row = runs.run_job(_sched(pg_settings, spec), "fake", "schedule")
+    assert row["status"] == "ok" and calls[-1] == "schedule"
+
+
+@needs_pg
+def test_chain_after_a_run(pg_settings: Settings) -> None:
+    feed = JobSpec(name="a", run=lambda ctx: {}, cadence=lambda fs, n: 60, pausable=True, chain=("b",))
+    other = JobSpec(name="b", run=lambda ctx: {}, cadence=lambda fs, n: 60, pausable=True)
+    sch = Scheduler(pg_settings, build="pytest", jobs={"a": feed, "b": other})
+    runs.run_job(sch, "a", "schedule")
+    assert sch._next_trigger.get("b") == "chain"  # moved to now with trigger chain
+
+
+@needs_pg
+def test_settings_save_audits_and_notifies(pg_settings: Settings) -> None:
+    from packetstation import db
+    from packetstation.domain.projects import Who
+
+    with db.connect_as("worker", pg_settings, autocommit=True) as listener:
+        listener.execute("LISTEN ps_settings")
+        with app_conn(pg_settings) as conn:
+            who = Who(user_id=None, session_id=None, role_snapshot="admin", ip=None)
+            new = config.save(conn, who=who, patch={"interval_shop_minutes": 7})
+            assert new.interval_shop_minutes == 7
+            row = conn.execute("SELECT value, version FROM settings WHERE key = 'feed'").fetchone()
+            assert row[0]["interval_shop_minutes"] == 7
+            audit = conn.execute(
+                "SELECT detail FROM audit_events WHERE action = 'feed.settings_changed' ORDER BY id DESC LIMIT 1"
+            ).fetchone()
+            assert audit[0]["changed"]["interval_shop_minutes"] == {"from": 5, "to": 7}
+        notes = list(listener.notifies(timeout=5, stop_after=1))
+        assert notes and notes[0].channel == "ps_settings"