PacketStation/server/packetstation/migrations/0004_feed.sql

-- 0004_feed: the VISUAL and drawing caches, the go-live ignored list, the resource map, the worker's
-- job tables and the order-derived columns on projects and units. Runs as ps_migrate; 0001's default
-- privileges give ps_app SELECT, INSERT, UPDATE, DELETE on every table here. Every work order ID part
-- (base, lot, split, sub) and every part ID is text: leading zeros are kept and lot 7A occurs.

-- ---------------------------------------------------------------- VISUAL cache

-- One row per feed read. suspect batches skip the "gone" marking.
CREATE TABLE visual_batches (
    id           bigserial PRIMARY KEY,
    run_id       bigint,
    started_at   timestamptz NOT NULL DEFAULT now(),
    finished_at  timestamptz,
    server       text,
    database     text,
    site         text,
    since        date,
    until        date,
    window_rows  int,
    tracked_keys int,
    missing_keys int,
    suspect      bool NOT NULL DEFAULT false,
    counts       jsonb NOT NULL DEFAULT '{}'
);

-- One row per TYPE W operation as VISUAL last showed it. st is the status code (F R U C X) or ''.
CREATE TABLE visual_rows (
    id          bigserial PRIMARY KEY,
    wo_base     text NOT NULL,
    lot         text NOT NULL DEFAULT '',
    split       text NOT NULL DEFAULT '',
    sub         text NOT NULL DEFAULT '',
    seq         int NOT NULL,
    res         text NOT NULL DEFAULT '',
    dwg         text NOT NULL DEFAULT '',
    rev         text NOT NULL DEFAULT '',
    ps          text NOT NULL DEFAULT '',
    descr       text NOT NULL DEFAULT '',
    qty         numeric,
    tgt         numeric,
    rel         date,
    want        date,
    st          text NOT NULL DEFAULT '',
    part        text NOT NULL DEFAULT '',
    barcode     text,
    row_hash    bytea NOT NULL,
    first_batch bigint NOT NULL REFERENCES visual_batches (id),
    last_batch  bigint NOT NULL REFERENCES visual_batches (id),
    gone_batch  bigint REFERENCES visual_batches (id),
    UNIQUE (wo_base, lot, split, sub, seq)
);

CREATE INDEX visual_rows_wo ON visual_rows (wo_base, lot);
CREATE INDEX visual_rows_dwg ON visual_rows (dwg);
CREATE INDEX visual_rows_rel_live ON visual_rows (rel) WHERE gone_batch IS NULL;

-- Material requirement lines. part is NULL on a line with no part (a group header).
CREATE TABLE visual_materials (
    id          bigserial PRIMARY KEY,
    wo_base     text NOT NULL,
    lot         text NOT NULL DEFAULT '',
    split       text NOT NULL DEFAULT '',
    sub         text NOT NULL DEFAULT '',
    seq         int NOT NULL,
    piece       int NOT NULL,
    part        text,
    qty         numeric,
    issued      numeric,
    whs         text NOT NULL DEFAULT '',
    loc         text NOT NULL DEFAULT '',
    st          text NOT NULL DEFAULT '',
    istat       int,
    dwg         text NOT NULL DEFAULT '',
    descr       text NOT NULL DEFAULT '',
    comm        text NOT NULL DEFAULT '',
    cons        text NOT NULL DEFAULT '',
    mat_hash    bytea NOT NULL,
    first_batch bigint NOT NULL REFERENCES visual_batches (id),
    last_batch  bigint NOT NULL REFERENCES visual_batches (id),
    gone_batch  bigint REFERENCES visual_batches (id),
    UNIQUE (wo_base, lot, split, sub, seq, piece)
);

CREATE INDEX visual_materials_wo ON visual_materials (wo_base, lot);

CREATE TABLE visual_orders (
    cust_order_id     text PRIMARY KEY,
    status            text,
    customer_id       text,
    customer_name     text,
    customer_po_ref   text,
    order_date        date,
    desired_ship_date date,
    promise_date      date,
    last_batch        bigint
);

-- kind base: key is a work order base. kind project: key is a project number.
CREATE TABLE visual_order_links (
    id            bigserial PRIMARY KEY,
    kind          text NOT NULL CHECK (kind IN ('base', 'project')),
    key           text NOT NULL,
    cust_order_id text NOT NULL REFERENCES visual_orders (cust_order_id),
    rank          int NOT NULL,
    lines         int[] NOT NULL DEFAULT '{}',
    top_jobs      text[] NOT NULL DEFAULT '{}',
    last_batch    bigint,
    UNIQUE (kind, key, cust_order_id)
);

CREATE INDEX visual_order_links_key ON visual_order_links (kind, key);

-- Hours and ticket counts per operation. No cost columns, ever.
CREATE TABLE visual_labor (
    id          bigserial PRIMARY KEY,
    wo_base     text NOT NULL,
    lot         text NOT NULL DEFAULT '',
    split       text NOT NULL DEFAULT '',
    sub         text NOT NULL DEFAULT '',
    seq         int NOT NULL,
    hours       numeric NOT NULL DEFAULT 0,
    tickets     int NOT NULL DEFAULT 0,
    last_ticket date,
    observed_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (wo_base, lot, split, sub, seq)
);

-- What VISUAL changed, per batch, for the Admin view. Pruned at 90 days.
CREATE TABLE visual_row_changes (
    id       bigserial PRIMARY KEY,
    batch_id bigint NOT NULL REFERENCES visual_batches (id),
    wo_base  text NOT NULL,
    lot      text NOT NULL DEFAULT '',
    split    text NOT NULL DEFAULT '',
    sub      text NOT NULL DEFAULT '',
    seq      int,
    kind     text NOT NULL CHECK (kind IN ('added', 'changed', 'gone', 'returned')),
    before   jsonb,
    after    jsonb,
    at       timestamptz NOT NULL DEFAULT now()
);

CREATE INDEX visual_row_changes_batch ON visual_row_changes (batch_id);
CREATE INDEX visual_row_changes_at_brin ON visual_row_changes USING brin (at);

CREATE TABLE visual_material_changes (
    id       bigserial PRIMARY KEY,
    batch_id bigint NOT NULL REFERENCES visual_batches (id),
    wo_base  text NOT NULL,
    lot      text NOT NULL DEFAULT '',
    split    text NOT NULL DEFAULT '',
    sub      text NOT NULL DEFAULT '',
    seq      int,
    piece    int,
    kind     text NOT NULL CHECK (kind IN ('added', 'changed', 'gone', 'returned')),
    before   jsonb,
    after    jsonb,
    at       timestamptz NOT NULL DEFAULT now()
);

CREATE INDEX visual_material_changes_batch ON visual_material_changes (batch_id);
CREATE INDEX visual_material_changes_at_brin ON visual_material_changes USING brin (at);

-- Work orders released before the go-live date. A row pulled in by hand becomes tracked.
CREATE TABLE visual_ignored_wos (
    id               bigserial PRIMARY KEY,
    wo_base          text NOT NULL,
    lot              text NOT NULL DEFAULT '',
    split            text NOT NULL DEFAULT '',
    sub              text NOT NULL DEFAULT '',
    rel              date,
    project          text,
    unit_number      text,
    res              text,
    dwg              text,
    first_seen_batch bigint,
    last_seen_at     timestamptz NOT NULL DEFAULT now(),
    pulled_in_at     timestamptz,
    pulled_in_by     uuid REFERENCES users (id),
    pulled_in_reason text,
    UNIQUE (wo_base, lot, split, sub)
);

-- VISUAL resource to shop department, or fold (an operation folded into the packet) or ignore.
CREATE TABLE visual_resource_map (
    resource   text PRIMARY KEY,
    target     text NOT NULL CHECK (target = 'fold' OR target = 'ignore' OR target ~ '^[0-9]{2}$'),
    confirmed  bool NOT NULL DEFAULT false,
    updated_at timestamptz,
    updated_by uuid
);

-- Seeded from the old build's sync/visual_map.json. "Electical" is spelled that way in VISUAL.
INSERT INTO visual_resource_map (resource, target) VALUES
    ('LAYOUT', '01'),
    ('LAYOUT-BT', '01'),
    ('LAYOUT-SM BAND', '01'),
    ('LAYOUT-BEVELER', '01'),
    ('LAYOUT-ATT', '01'),
    ('LAYOUT-ROLL', '01'),
    ('LAYOUT-PRESS', '01'),
    ('LAYOUT-THREADER', '01'),
    ('LAYOUT-HAND CUT', '01'),
    ('LAYOUT-WEST', '02'),
    ('LT-SS', '02'),
    ('WELD-PART', '03'),
    ('NOVARC_1', '03'),
    ('WELD-PART-WEST', '04'),
    ('PART-WELD-WEST', '04'),
    ('WP-SS', '05'),
    ('WELD-CODE', '06'),
    ('WELD-CODE-BT', '07'),
    ('SUBASSEMBLY', '08'),
    ('WEST-SA', '08'),
    ('HYDROTEST', '09'),
    ('WELD-ASY', '10'),
    ('FINAL ASY-NORTH', '11'),
    ('PREP-NORTH', '11'),
    ('FINAL ASSEMBLY', '11'),
    ('FINAL ASY-SOUTH', '12'),
    ('PREP-SOUTH', '12'),
    ('WEST-FA', '12'),
    ('PAINT', '13'),
    ('SANDBLAST', '13'),
    ('FINAL-ASY-ELEC', '14'),
    ('Electical', '14'),
    ('MFG', 'fold'),
    ('INSPECTION', 'fold'),
    ('REWORK', 'fold'),
    ('REWORK-CUSTOMER', 'fold'),
    ('OUTSIDE SERVICE', 'fold'),
    ('PTU', 'fold'),
    ('FORKLIFT', 'ignore'),
    ('STEAM', 'ignore'),
    ('WIPLOAD', 'ignore'),
    ('FOREMAN', 'ignore'),
    ('ENGINEERING', 'ignore'),
    ('TRAINING', 'ignore'),
    ('F.S. TECH.', 'ignore'),
    ('F.S. TECH. ASST', 'ignore'),
    ('F.S. STEAM', 'ignore'),
    ('F.S. WELDER', 'ignore'),
    ('F.S. SANDBLAST', 'ignore'),
    ('F.S. PAINT', 'ignore'),
    ('F.S. SAFETY', 'ignore'),
    ('REP. TECH. A', 'ignore'),
    ('REP. TECH. B', 'ignore'),
    ('REP. WELDER A', 'ignore'),
    ('REP. WELDER B', 'ignore');

-- ---------------------------------------------------------------- drawing cache

-- One row per drawing number as VISUAL writes it (trimmed). asset_id is the last fetched copy.
CREATE TABLE drawing_lookups (
    dwg             text PRIMARY KEY,
    checked_at      timestamptz,
    ok              bool NOT NULL DEFAULT true,
    rev             int,
    name            text,
    obj_id          int,
    obj_version     int,
    file_id         int,
    file_version    int,
    file_size       bigint,
    base            text,
    draft_rev       int,
    draft_where     text,
    in_drafting     bool NOT NULL DEFAULT false,
    note            text,
    fail_count      int NOT NULL DEFAULT 0,
    failing_since   timestamptz,
    asset_id        uuid REFERENCES assets (id),
    sha256          bytea,
    visual_rev_seen text,
    next_check_at   timestamptz
);

CREATE INDEX drawing_lookups_next_check ON drawing_lookups (next_check_at) WHERE next_check_at IS NOT NULL;

-- ---------------------------------------------------------------- worker jobs

-- "Run now": the app inserts a row and NOTIFYs ps_job_request; one run claims every pending row.
CREATE TABLE job_requests (
    id             bigserial PRIMARY KEY,
    job            text NOT NULL,
    requested_at   timestamptz NOT NULL DEFAULT now(),
    requested_by   uuid REFERENCES users (id),
    note           text,
    claimed_run_id bigint,
    claimed_at     timestamptz
);

CREATE INDEX job_requests_pending ON job_requests (job) WHERE claimed_at IS NULL;

CREATE TABLE job_runs (
    id            bigserial PRIMARY KEY,
    job           text NOT NULL,
    trigger       text NOT NULL CHECK (trigger IN ('schedule', 'request', 'chain', 'cli')),
    requested_by  uuid[] NOT NULL DEFAULT '{}',
    started_at    timestamptz NOT NULL DEFAULT now(),
    finished_at   timestamptz,
    status        text NOT NULL DEFAULT 'running'
                  CHECK (status IN ('running', 'ok', 'degraded', 'failed', 'skipped', 'abandoned')),
    counts        jsonb NOT NULL DEFAULT '{}',
    errors        jsonb NOT NULL DEFAULT '[]',
    build_version text,
    visual_target text,
    host          text
);

CREATE INDEX job_runs_job ON job_runs (job, id DESC);

-- The next run per job as the worker last scheduled it, for the Admin and Clerical screens.
CREATE TABLE job_schedule (
    job              text PRIMARY KEY,
    next_run_at      timestamptz,
    interval_seconds int,
    paused           bool NOT NULL DEFAULT false,
    updated_at       timestamptz NOT NULL DEFAULT now()
);

INSERT INTO job_schedule (job) VALUES ('visual-feed'), ('drawing-lookup'), ('housekeeping');

-- One row, written every 30 seconds by the running worker.
CREATE TABLE worker_heartbeat (
    id      int PRIMARY KEY CHECK (id = 1),
    at      timestamptz NOT NULL,
    build   text,
    host    text,
    pid     int,
    running text[] NOT NULL DEFAULT '{}'
);

-- ---------------------------------------------------------------- column additions

ALTER TABLE packets
    ADD COLUMN visual_cancelled_at timestamptz,
    ADD COLUMN wo_gone_at timestamptz;

-- The leg's own drawing, rev and planner instructions as VISUAL carries them.
ALTER TABLE packet_work_orders
    ADD COLUMN partspecs text,
    ADD COLUMN rev text,
    ADD COLUMN dwg text;

-- One row per leg: two operations of one work order may sit on one packet. Hand rows keep seq NULL,
-- and NULLS NOT DISTINCT still allows only one hand row per key.
DO $$
DECLARE
    c text;
BEGIN
    SELECT con.conname INTO c
    FROM pg_constraint con
    WHERE con.conrelid = 'packet_work_orders'::regclass
      AND con.contype = 'u'
      AND (
          SELECT array_agg(a.attname::text ORDER BY a.attname)
          FROM unnest(con.conkey) k
          JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = k
      ) = ARRAY['lot', 'packet_id', 'split', 'sub', 'wo_base'];
    IF c IS NULL THEN
        RAISE EXCEPTION 'packet_work_orders unique key (packet_id, wo_base, lot, split, sub) not found';
    END IF;
    EXECUTE format('ALTER TABLE packet_work_orders DROP CONSTRAINT %I', c);
END;
$$;

ALTER TABLE packet_work_orders
    ADD CONSTRAINT packet_work_orders_leg_key UNIQUE NULLS NOT DISTINCT (packet_id, wo_base, lot, split, sub, seq);

-- One pending feed change order per packet, whether VISUAL or M-Files opened it.
DROP INDEX change_orders_visual_pending;
CREATE UNIQUE INDEX change_orders_feed_pending ON change_orders (packet_id)
    WHERE origin IN ('visual', 'drawing') AND review = 'pending';

-- order_ref is the order the feed chose; order_pin is one Clerical pinned (with a reason).
ALTER TABLE projects
    ADD COLUMN order_date_visual date,
    ADD COLUMN ship_date_visual date,
    ADD COLUMN order_ref jsonb,
    ADD COLUMN order_pin text,
    ADD COLUMN order_pin_by uuid REFERENCES users (id),
    ADD COLUMN order_pin_at timestamptz,
    ADD COLUMN order_pin_reason text;

ALTER TABLE units
    ADD COLUMN customer text,
    ADD COLUMN po text,
    ADD COLUMN customer_visual text,
    ADD COLUMN po_visual text,
    ADD COLUMN order_date_visual date,
    ADD COLUMN ship_date_visual date,
    ADD COLUMN order_ref jsonb,
    ADD COLUMN order_pin text,
    ADD COLUMN order_pin_by uuid REFERENCES users (id),
    ADD COLUMN order_pin_at timestamptz,
    ADD COLUMN order_pin_reason text;

-- v_units expanded u.* when it was made, so it is rebuilt to carry the new unit columns.
DROP VIEW v_units;
CREATE VIEW v_units AS
SELECT u.*,
       p.number || u.lot AS number,
       p.number AS project_number,
       p.name AS project_name,
       p.status AS project_status,
       coalesce(u.due, p.due) AS due_effective
FROM units u
JOIN projects p ON p.id = u.project_id;

-- Explicit, after the default privileges: the view is read-only to the app.
REVOKE INSERT, UPDATE, DELETE, TRUNCATE ON v_units FROM ps_app;
GRANT SELECT ON v_units TO ps_app;
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ps_app;

-- The feed COPYs each batch into temp tables before the MERGE. ps_migrate owns the database (initdb and
-- bootstrap), so it may grant TEMP; stacks made before this migration get the grant here.
DO $$
BEGIN
    EXECUTE format('GRANT TEMPORARY ON DATABASE %I TO ps_app', current_database());
END;
$$;