Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

deploy/ci/ci.env

@@ -94,6 +94,15 @@ SMTP_CA_BUNDLE=/etc/ssl/pesco/ca.pem
 # true logs notifications and sends nothing.
 NOTIFY_DRY_RUN=true
 
+# ---- Worker sources ----
+# CI reads the synthetic fixtures, never VISUAL or M-Files. The e2e job generates them into
+# tests/fixtures/generated, which compose.ci.yaml mounts at /fixtures.
+FEED_SOURCE=fixture
+# fixture: drawings.<state>.json and the PDFs it names, under FIXTURE_DIR.
+DRAWING_SOURCE=fixture
+# Fixture folder inside the containers. The scenario writes /fixtures/state to switch states.
+FIXTURE_DIR=/fixtures
+
 # ---- VisualKit (VISUAL ERP, read only) ----
 # SQL Server instance. Full name: containers have no search domain.
 VISUAL_SERVER=

deploy/compose.ci.yaml

@@ -10,6 +10,10 @@
 #   docker compose -p packetstation-ci --env-file deploy/ci/ci.env \
 #     -f deploy/compose.yaml -f deploy/compose.ci.yaml down -v
 #
+# The worker reads the synthetic feed and drawings fixtures (FEED_SOURCE=fixture, DRAWING_SOURCE=fixture in
+# ci.env) from ../tests/fixtures/generated, mounted at /fixtures. Generate them first:
+#   python -m tests.fixtures.generator --out tests/fixtures/generated
+#
 # Never combine this overlay with compose.proxy.yaml.
 
 x-logging: &logging
@@ -55,6 +59,7 @@ services:
     pull_policy: build
     volumes:
       - ci-dc-export:/etc/ssl/pstest:ro
+      - ../tests/fixtures/generated:/fixtures:ro   # synthetic feed and drawings; the scenario switches state
 
   # Throwaway Samba AD DC (PSTEST.LOCAL) so LDAP sign-in runs over real TLS in CI.
   # Built from deploy/samba/. Runs as root, as Samba needs. No published port.
@@ -101,9 +106,11 @@ services:
     environment:
       E2E_BASE_URL: http://ps-app:8000
       E2E_DC_USERS_FILE: /export/users.json
+      E2E_FIXTURE_DIR: /fixtures
       CI: "true"
     volumes:
       - ../tests/e2e:/work/tests/e2e
+      - ../tests/fixtures/generated:/fixtures      # read-write: the scenario writes /fixtures/state
       - /work/tests/e2e/node_modules       # anonymous volume: node_modules stays out of the checkout
       - ci-dc-export:/export:ro
     networks:

deploy/env/prod.env.example

@@ -36,6 +36,12 @@ TRUSTED_PROXY_HOST=npm_app_1
 LOG_LEVEL=INFO
 # Worker heartbeat file read by its healthcheck. /tmp is a tmpfs.
 WORKER_BEAT_FILE=/tmp/packetstation-worker.beat
+# Where the worker reads work orders: visualkit (the real VISUAL through VisualKit) or fixture (CI only; prod refuses it).
+FEED_SOURCE=visualkit
+# Where the worker looks up drawings: mfiles (the vault through MFilesKit) or fixture (CI only; prod refuses it).
+DRAWING_SOURCE=mfiles
+# Fixture folder inside the containers. Only read when a source is fixture.
+FIXTURE_DIR=/fixtures
 
 # ---- PostgreSQL (roles made by deploy/postgres/initdb/01-roles.sh) ----
 # postgres alias on the internal backend network.

deploy/env/test.env.example

@@ -36,6 +36,12 @@ TRUSTED_PROXY_HOST=npm_app_1
 LOG_LEVEL=INFO
 # Worker heartbeat file read by its healthcheck. /tmp is a tmpfs.
 WORKER_BEAT_FILE=/tmp/packetstation-worker.beat
+# Where the worker reads work orders: visualkit (the real VISUAL through VisualKit) or fixture (CI only; prod refuses it).
+FEED_SOURCE=visualkit
+# Where the worker looks up drawings: mfiles (the vault through MFilesKit) or fixture (CI only; prod refuses it).
+DRAWING_SOURCE=mfiles
+# Fixture folder inside the containers. Only read when a source is fixture.
+FIXTURE_DIR=/fixtures
 
 # ---- PostgreSQL (roles made by deploy/postgres/initdb/01-roles.sh) ----
 # postgres alias on the internal backend network.

deploy/postgres/initdb/01-roles.sh

@@ -39,7 +39,7 @@ CREATE ROLE ps_app     LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION P
 
 CREATE DATABASE packetstation OWNER ps_migrate ENCODING 'UTF8' TEMPLATE template0;
 REVOKE ALL ON DATABASE packetstation FROM PUBLIC;
-GRANT CONNECT ON DATABASE packetstation TO ps_app;
+GRANT CONNECT, TEMPORARY ON DATABASE packetstation TO ps_app;  -- TEMP: the feed COPYs into temp tables
 
 \connect packetstation
 -- PostgreSQL 15+ already denies CREATE on public to PUBLIC. Make it explicit and give