Phase 3: VISUAL feed and M-Files drawings in the worker
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,422 @@
|
||||
-- 0004_feed: the VISUAL and drawing caches, the go-live ignored list, the resource map, the worker's
|
||||
-- job tables and the order-derived columns on projects and units. Runs as ps_migrate; 0001's default
|
||||
-- privileges give ps_app SELECT, INSERT, UPDATE, DELETE on every table here. Every work order ID part
|
||||
-- (base, lot, split, sub) and every part ID is text: leading zeros are kept and lot 7A occurs.
|
||||
|
||||
-- ---------------------------------------------------------------- VISUAL cache
|
||||
|
||||
-- One row per feed read. suspect batches skip the "gone" marking.
|
||||
CREATE TABLE visual_batches (
|
||||
id bigserial PRIMARY KEY,
|
||||
run_id bigint,
|
||||
started_at timestamptz NOT NULL DEFAULT now(),
|
||||
finished_at timestamptz,
|
||||
server text,
|
||||
database text,
|
||||
site text,
|
||||
since date,
|
||||
until date,
|
||||
window_rows int,
|
||||
tracked_keys int,
|
||||
missing_keys int,
|
||||
suspect bool NOT NULL DEFAULT false,
|
||||
counts jsonb NOT NULL DEFAULT '{}'
|
||||
);
|
||||
|
||||
-- One row per TYPE W operation as VISUAL last showed it. st is the status code (F R U C X) or ''.
|
||||
CREATE TABLE visual_rows (
|
||||
id bigserial PRIMARY KEY,
|
||||
wo_base text NOT NULL,
|
||||
lot text NOT NULL DEFAULT '',
|
||||
split text NOT NULL DEFAULT '',
|
||||
sub text NOT NULL DEFAULT '',
|
||||
seq int NOT NULL,
|
||||
res text NOT NULL DEFAULT '',
|
||||
dwg text NOT NULL DEFAULT '',
|
||||
rev text NOT NULL DEFAULT '',
|
||||
ps text NOT NULL DEFAULT '',
|
||||
descr text NOT NULL DEFAULT '',
|
||||
qty numeric,
|
||||
tgt numeric,
|
||||
rel date,
|
||||
want date,
|
||||
st text NOT NULL DEFAULT '',
|
||||
part text NOT NULL DEFAULT '',
|
||||
barcode text,
|
||||
row_hash bytea NOT NULL,
|
||||
first_batch bigint NOT NULL REFERENCES visual_batches (id),
|
||||
last_batch bigint NOT NULL REFERENCES visual_batches (id),
|
||||
gone_batch bigint REFERENCES visual_batches (id),
|
||||
UNIQUE (wo_base, lot, split, sub, seq)
|
||||
);
|
||||
|
||||
CREATE INDEX visual_rows_wo ON visual_rows (wo_base, lot);
|
||||
CREATE INDEX visual_rows_dwg ON visual_rows (dwg);
|
||||
CREATE INDEX visual_rows_rel_live ON visual_rows (rel) WHERE gone_batch IS NULL;
|
||||
|
||||
-- Material requirement lines. part is NULL on a line with no part (a group header).
|
||||
CREATE TABLE visual_materials (
|
||||
id bigserial PRIMARY KEY,
|
||||
wo_base text NOT NULL,
|
||||
lot text NOT NULL DEFAULT '',
|
||||
split text NOT NULL DEFAULT '',
|
||||
sub text NOT NULL DEFAULT '',
|
||||
seq int NOT NULL,
|
||||
piece int NOT NULL,
|
||||
part text,
|
||||
qty numeric,
|
||||
issued numeric,
|
||||
whs text NOT NULL DEFAULT '',
|
||||
loc text NOT NULL DEFAULT '',
|
||||
st text NOT NULL DEFAULT '',
|
||||
istat int,
|
||||
dwg text NOT NULL DEFAULT '',
|
||||
descr text NOT NULL DEFAULT '',
|
||||
comm text NOT NULL DEFAULT '',
|
||||
cons text NOT NULL DEFAULT '',
|
||||
mat_hash bytea NOT NULL,
|
||||
first_batch bigint NOT NULL REFERENCES visual_batches (id),
|
||||
last_batch bigint NOT NULL REFERENCES visual_batches (id),
|
||||
gone_batch bigint REFERENCES visual_batches (id),
|
||||
UNIQUE (wo_base, lot, split, sub, seq, piece)
|
||||
);
|
||||
|
||||
CREATE INDEX visual_materials_wo ON visual_materials (wo_base, lot);
|
||||
|
||||
CREATE TABLE visual_orders (
|
||||
cust_order_id text PRIMARY KEY,
|
||||
status text,
|
||||
customer_id text,
|
||||
customer_name text,
|
||||
customer_po_ref text,
|
||||
order_date date,
|
||||
desired_ship_date date,
|
||||
promise_date date,
|
||||
last_batch bigint
|
||||
);
|
||||
|
||||
-- kind base: key is a work order base. kind project: key is a project number.
|
||||
CREATE TABLE visual_order_links (
|
||||
id bigserial PRIMARY KEY,
|
||||
kind text NOT NULL CHECK (kind IN ('base', 'project')),
|
||||
key text NOT NULL,
|
||||
cust_order_id text NOT NULL REFERENCES visual_orders (cust_order_id),
|
||||
rank int NOT NULL,
|
||||
lines int[] NOT NULL DEFAULT '{}',
|
||||
top_jobs text[] NOT NULL DEFAULT '{}',
|
||||
last_batch bigint,
|
||||
UNIQUE (kind, key, cust_order_id)
|
||||
);
|
||||
|
||||
CREATE INDEX visual_order_links_key ON visual_order_links (kind, key);
|
||||
|
||||
-- Hours and ticket counts per operation. No cost columns, ever.
|
||||
CREATE TABLE visual_labor (
|
||||
id bigserial PRIMARY KEY,
|
||||
wo_base text NOT NULL,
|
||||
lot text NOT NULL DEFAULT '',
|
||||
split text NOT NULL DEFAULT '',
|
||||
sub text NOT NULL DEFAULT '',
|
||||
seq int NOT NULL,
|
||||
hours numeric NOT NULL DEFAULT 0,
|
||||
tickets int NOT NULL DEFAULT 0,
|
||||
last_ticket date,
|
||||
observed_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (wo_base, lot, split, sub, seq)
|
||||
);
|
||||
|
||||
-- What VISUAL changed, per batch, for the Admin view. Pruned at 90 days.
|
||||
CREATE TABLE visual_row_changes (
|
||||
id bigserial PRIMARY KEY,
|
||||
batch_id bigint NOT NULL REFERENCES visual_batches (id),
|
||||
wo_base text NOT NULL,
|
||||
lot text NOT NULL DEFAULT '',
|
||||
split text NOT NULL DEFAULT '',
|
||||
sub text NOT NULL DEFAULT '',
|
||||
seq int,
|
||||
kind text NOT NULL CHECK (kind IN ('added', 'changed', 'gone', 'returned')),
|
||||
before jsonb,
|
||||
after jsonb,
|
||||
at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX visual_row_changes_batch ON visual_row_changes (batch_id);
|
||||
CREATE INDEX visual_row_changes_at_brin ON visual_row_changes USING brin (at);
|
||||
|
||||
CREATE TABLE visual_material_changes (
|
||||
id bigserial PRIMARY KEY,
|
||||
batch_id bigint NOT NULL REFERENCES visual_batches (id),
|
||||
wo_base text NOT NULL,
|
||||
lot text NOT NULL DEFAULT '',
|
||||
split text NOT NULL DEFAULT '',
|
||||
sub text NOT NULL DEFAULT '',
|
||||
seq int,
|
||||
piece int,
|
||||
kind text NOT NULL CHECK (kind IN ('added', 'changed', 'gone', 'returned')),
|
||||
before jsonb,
|
||||
after jsonb,
|
||||
at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX visual_material_changes_batch ON visual_material_changes (batch_id);
|
||||
CREATE INDEX visual_material_changes_at_brin ON visual_material_changes USING brin (at);
|
||||
|
||||
-- Work orders released before the go-live date. A row pulled in by hand becomes tracked.
|
||||
CREATE TABLE visual_ignored_wos (
|
||||
id bigserial PRIMARY KEY,
|
||||
wo_base text NOT NULL,
|
||||
lot text NOT NULL DEFAULT '',
|
||||
split text NOT NULL DEFAULT '',
|
||||
sub text NOT NULL DEFAULT '',
|
||||
rel date,
|
||||
project text,
|
||||
unit_number text,
|
||||
res text,
|
||||
dwg text,
|
||||
first_seen_batch bigint,
|
||||
last_seen_at timestamptz NOT NULL DEFAULT now(),
|
||||
pulled_in_at timestamptz,
|
||||
pulled_in_by uuid REFERENCES users (id),
|
||||
pulled_in_reason text,
|
||||
UNIQUE (wo_base, lot, split, sub)
|
||||
);
|
||||
|
||||
-- VISUAL resource to shop department, or fold (an operation folded into the packet) or ignore.
|
||||
CREATE TABLE visual_resource_map (
|
||||
resource text PRIMARY KEY,
|
||||
target text NOT NULL CHECK (target = 'fold' OR target = 'ignore' OR target ~ '^[0-9]{2}$'),
|
||||
confirmed bool NOT NULL DEFAULT false,
|
||||
updated_at timestamptz,
|
||||
updated_by uuid
|
||||
);
|
||||
|
||||
-- Seeded from the old build's sync/visual_map.json. "Electical" is spelled that way in VISUAL.
|
||||
INSERT INTO visual_resource_map (resource, target) VALUES
|
||||
('LAYOUT', '01'),
|
||||
('LAYOUT-BT', '01'),
|
||||
('LAYOUT-SM BAND', '01'),
|
||||
('LAYOUT-BEVELER', '01'),
|
||||
('LAYOUT-ATT', '01'),
|
||||
('LAYOUT-ROLL', '01'),
|
||||
('LAYOUT-PRESS', '01'),
|
||||
('LAYOUT-THREADER', '01'),
|
||||
('LAYOUT-HAND CUT', '01'),
|
||||
('LAYOUT-WEST', '02'),
|
||||
('LT-SS', '02'),
|
||||
('WELD-PART', '03'),
|
||||
('NOVARC_1', '03'),
|
||||
('WELD-PART-WEST', '04'),
|
||||
('PART-WELD-WEST', '04'),
|
||||
('WP-SS', '05'),
|
||||
('WELD-CODE', '06'),
|
||||
('WELD-CODE-BT', '07'),
|
||||
('SUBASSEMBLY', '08'),
|
||||
('WEST-SA', '08'),
|
||||
('HYDROTEST', '09'),
|
||||
('WELD-ASY', '10'),
|
||||
('FINAL ASY-NORTH', '11'),
|
||||
('PREP-NORTH', '11'),
|
||||
('FINAL ASSEMBLY', '11'),
|
||||
('FINAL ASY-SOUTH', '12'),
|
||||
('PREP-SOUTH', '12'),
|
||||
('WEST-FA', '12'),
|
||||
('PAINT', '13'),
|
||||
('SANDBLAST', '13'),
|
||||
('FINAL-ASY-ELEC', '14'),
|
||||
('Electical', '14'),
|
||||
('MFG', 'fold'),
|
||||
('INSPECTION', 'fold'),
|
||||
('REWORK', 'fold'),
|
||||
('REWORK-CUSTOMER', 'fold'),
|
||||
('OUTSIDE SERVICE', 'fold'),
|
||||
('PTU', 'fold'),
|
||||
('FORKLIFT', 'ignore'),
|
||||
('STEAM', 'ignore'),
|
||||
('WIPLOAD', 'ignore'),
|
||||
('FOREMAN', 'ignore'),
|
||||
('ENGINEERING', 'ignore'),
|
||||
('TRAINING', 'ignore'),
|
||||
('F.S. TECH.', 'ignore'),
|
||||
('F.S. TECH. ASST', 'ignore'),
|
||||
('F.S. STEAM', 'ignore'),
|
||||
('F.S. WELDER', 'ignore'),
|
||||
('F.S. SANDBLAST', 'ignore'),
|
||||
('F.S. PAINT', 'ignore'),
|
||||
('F.S. SAFETY', 'ignore'),
|
||||
('REP. TECH. A', 'ignore'),
|
||||
('REP. TECH. B', 'ignore'),
|
||||
('REP. WELDER A', 'ignore'),
|
||||
('REP. WELDER B', 'ignore');
|
||||
|
||||
-- ---------------------------------------------------------------- drawing cache
|
||||
|
||||
-- One row per drawing number as VISUAL writes it (trimmed). asset_id is the last fetched copy.
|
||||
CREATE TABLE drawing_lookups (
|
||||
dwg text PRIMARY KEY,
|
||||
checked_at timestamptz,
|
||||
ok bool NOT NULL DEFAULT true,
|
||||
rev int,
|
||||
name text,
|
||||
obj_id int,
|
||||
obj_version int,
|
||||
file_id int,
|
||||
file_version int,
|
||||
file_size bigint,
|
||||
base text,
|
||||
draft_rev int,
|
||||
draft_where text,
|
||||
in_drafting bool NOT NULL DEFAULT false,
|
||||
note text,
|
||||
fail_count int NOT NULL DEFAULT 0,
|
||||
failing_since timestamptz,
|
||||
asset_id uuid REFERENCES assets (id),
|
||||
sha256 bytea,
|
||||
visual_rev_seen text,
|
||||
next_check_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX drawing_lookups_next_check ON drawing_lookups (next_check_at) WHERE next_check_at IS NOT NULL;
|
||||
|
||||
-- ---------------------------------------------------------------- worker jobs
|
||||
|
||||
-- "Run now": the app inserts a row and NOTIFYs ps_job_request; one run claims every pending row.
|
||||
CREATE TABLE job_requests (
|
||||
id bigserial PRIMARY KEY,
|
||||
job text NOT NULL,
|
||||
requested_at timestamptz NOT NULL DEFAULT now(),
|
||||
requested_by uuid REFERENCES users (id),
|
||||
note text,
|
||||
claimed_run_id bigint,
|
||||
claimed_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX job_requests_pending ON job_requests (job) WHERE claimed_at IS NULL;
|
||||
|
||||
CREATE TABLE job_runs (
|
||||
id bigserial PRIMARY KEY,
|
||||
job text NOT NULL,
|
||||
trigger text NOT NULL CHECK (trigger IN ('schedule', 'request', 'chain', 'cli')),
|
||||
requested_by uuid[] NOT NULL DEFAULT '{}',
|
||||
started_at timestamptz NOT NULL DEFAULT now(),
|
||||
finished_at timestamptz,
|
||||
status text NOT NULL DEFAULT 'running'
|
||||
CHECK (status IN ('running', 'ok', 'degraded', 'failed', 'skipped', 'abandoned')),
|
||||
counts jsonb NOT NULL DEFAULT '{}',
|
||||
errors jsonb NOT NULL DEFAULT '[]',
|
||||
build_version text,
|
||||
visual_target text,
|
||||
host text
|
||||
);
|
||||
|
||||
CREATE INDEX job_runs_job ON job_runs (job, id DESC);
|
||||
|
||||
-- The next run per job as the worker last scheduled it, for the Admin and Clerical screens.
|
||||
CREATE TABLE job_schedule (
|
||||
job text PRIMARY KEY,
|
||||
next_run_at timestamptz,
|
||||
interval_seconds int,
|
||||
paused bool NOT NULL DEFAULT false,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
INSERT INTO job_schedule (job) VALUES ('visual-feed'), ('drawing-lookup'), ('housekeeping');
|
||||
|
||||
-- One row, written every 30 seconds by the running worker.
|
||||
CREATE TABLE worker_heartbeat (
|
||||
id int PRIMARY KEY CHECK (id = 1),
|
||||
at timestamptz NOT NULL,
|
||||
build text,
|
||||
host text,
|
||||
pid int,
|
||||
running text[] NOT NULL DEFAULT '{}'
|
||||
);
|
||||
|
||||
-- ---------------------------------------------------------------- column additions
|
||||
|
||||
ALTER TABLE packets
|
||||
ADD COLUMN visual_cancelled_at timestamptz,
|
||||
ADD COLUMN wo_gone_at timestamptz;
|
||||
|
||||
-- The leg's own drawing, rev and planner instructions as VISUAL carries them.
|
||||
ALTER TABLE packet_work_orders
|
||||
ADD COLUMN partspecs text,
|
||||
ADD COLUMN rev text,
|
||||
ADD COLUMN dwg text;
|
||||
|
||||
-- One row per leg: two operations of one work order may sit on one packet. Hand rows keep seq NULL,
|
||||
-- and NULLS NOT DISTINCT still allows only one hand row per key.
|
||||
DO $$
|
||||
DECLARE
|
||||
c text;
|
||||
BEGIN
|
||||
SELECT con.conname INTO c
|
||||
FROM pg_constraint con
|
||||
WHERE con.conrelid = 'packet_work_orders'::regclass
|
||||
AND con.contype = 'u'
|
||||
AND (
|
||||
SELECT array_agg(a.attname::text ORDER BY a.attname)
|
||||
FROM unnest(con.conkey) k
|
||||
JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = k
|
||||
) = ARRAY['lot', 'packet_id', 'split', 'sub', 'wo_base'];
|
||||
IF c IS NULL THEN
|
||||
RAISE EXCEPTION 'packet_work_orders unique key (packet_id, wo_base, lot, split, sub) not found';
|
||||
END IF;
|
||||
EXECUTE format('ALTER TABLE packet_work_orders DROP CONSTRAINT %I', c);
|
||||
END;
|
||||
$$;
|
||||
|
||||
ALTER TABLE packet_work_orders
|
||||
ADD CONSTRAINT packet_work_orders_leg_key UNIQUE NULLS NOT DISTINCT (packet_id, wo_base, lot, split, sub, seq);
|
||||
|
||||
-- One pending feed change order per packet, whether VISUAL or M-Files opened it.
|
||||
DROP INDEX change_orders_visual_pending;
|
||||
CREATE UNIQUE INDEX change_orders_feed_pending ON change_orders (packet_id)
|
||||
WHERE origin IN ('visual', 'drawing') AND review = 'pending';
|
||||
|
||||
-- order_ref is the order the feed chose; order_pin is one Clerical pinned (with a reason).
|
||||
ALTER TABLE projects
|
||||
ADD COLUMN order_date_visual date,
|
||||
ADD COLUMN ship_date_visual date,
|
||||
ADD COLUMN order_ref jsonb,
|
||||
ADD COLUMN order_pin text,
|
||||
ADD COLUMN order_pin_by uuid REFERENCES users (id),
|
||||
ADD COLUMN order_pin_at timestamptz,
|
||||
ADD COLUMN order_pin_reason text;
|
||||
|
||||
ALTER TABLE units
|
||||
ADD COLUMN customer text,
|
||||
ADD COLUMN po text,
|
||||
ADD COLUMN customer_visual text,
|
||||
ADD COLUMN po_visual text,
|
||||
ADD COLUMN order_date_visual date,
|
||||
ADD COLUMN ship_date_visual date,
|
||||
ADD COLUMN order_ref jsonb,
|
||||
ADD COLUMN order_pin text,
|
||||
ADD COLUMN order_pin_by uuid REFERENCES users (id),
|
||||
ADD COLUMN order_pin_at timestamptz,
|
||||
ADD COLUMN order_pin_reason text;
|
||||
|
||||
-- v_units expanded u.* when it was made, so it is rebuilt to carry the new unit columns.
|
||||
DROP VIEW v_units;
|
||||
CREATE VIEW v_units AS
|
||||
SELECT u.*,
|
||||
p.number || u.lot AS number,
|
||||
p.number AS project_number,
|
||||
p.name AS project_name,
|
||||
p.status AS project_status,
|
||||
coalesce(u.due, p.due) AS due_effective
|
||||
FROM units u
|
||||
JOIN projects p ON p.id = u.project_id;
|
||||
|
||||
-- Explicit, after the default privileges: the view is read-only to the app.
|
||||
REVOKE INSERT, UPDATE, DELETE, TRUNCATE ON v_units FROM ps_app;
|
||||
GRANT SELECT ON v_units TO ps_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ps_app;
|
||||
|
||||
-- The feed COPYs each batch into temp tables before the MERGE. ps_migrate owns the database (initdb and
|
||||
-- bootstrap), so it may grant TEMP; stacks made before this migration get the grant here.
|
||||
DO $$
|
||||
BEGIN
|
||||
EXECUTE format('GRANT TEMPORARY ON DATABASE %I TO ps_app', current_database());
|
||||
END;
|
||||
$$;
|
||||
Reference in New Issue
Block a user