diff --git a/.env.example b/.env.example index ab51050..d869a22 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,9 @@ # deploy/ci/ci.env -> CI and local end-to-end runs (committed, no secrets) # Passwords and keys are never put in an env file. They are files in SECRETS_DIR, # mounted as compose secrets. See docs/DEPLOY.md. +# +# Worker sources (phase 3). The defaults read VISUAL through VisualKit and M-Files through MFilesKit: +# FEED_SOURCE=visualkit visualkit | fixture +# DRAWING_SOURCE=mfiles mfiles | fixture +# FIXTURE_DIR=/fixtures where the fixture sources read feed..json, drawings..json and state +# fixture reads synthetic files from tests/fixtures/generator.py. STACK=prod refuses it. diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8772bbb..2ce2322 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -150,6 +150,20 @@ jobs: - name: Throwaway secrets run: bash deploy/ci/make-secrets.sh + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # The worker reads these through FEED_SOURCE=fixture and DRAWING_SOURCE=fixture (deploy/ci/ci.env). + # The generator needs pypdf only, so nothing else is installed here. + - name: Generate feed fixtures + run: | + set -euo pipefail + python -m pip install pypdf==6.21.0 + python -m tests.fixtures.generator --out tests/fixtures/generated + # The Playwright container writes state; the worker reads as uid 10001. + chmod -R a+rwX tests/fixtures/generated + - name: Stack up (app, worker, postgres, Samba DC) run: $COMPOSE up -d --build --wait --wait-timeout 300 diff --git a/.gitignore b/.gitignore index 6a9c5df..78b71d0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ secrets/ # Real job data stays out of git tests/fixtures/local/ +tests/fixtures/generated/ reference/ recordings/ diff --git a/CHANGELOG.md b/CHANGELOG.md index de8ec7b..0d2cc13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ All notable changes to this project are recorded here. The format follows [Keep ### Added +- Phase 3: the VISUAL feed and M-Files drawings run in the worker. Migration 0004 with the VISUAL cache (batches, rows, materials, orders and order links, labor, per-batch change rows), the go-live ignored list, the resource map seeded from the old build's map (fold and ignore lines included), drawing lookups, job requests, job runs, job schedule and the worker heartbeat, plus the order-derived columns with provenance on projects and units. The work order key on a packet now includes the operation sequence, so one packet can hold two operations of the same work order. +- The real worker: APScheduler with a settings-driven trigger that re-reads the feed settings on every fire, shop-hours and off-hours intervals, pause, LISTEN on `ps_settings` and `ps_job_request`, one advisory lock per job, `job_runs` rows with counts and redacted errors, a heartbeat row and file, the stack guard before every VISUAL read, and a housekeeping job that prunes the caches at 90 days and the runs at 180. +- The `visual-feed` job: five reads through VisualKit (released work orders in the window with `until` always passed, every tracked work order by key with no status or date filter, operations and part specs and parts and materials with `mat_known` per leg, order links by base and by project, labor tickets at most every 30 minutes), COPY and MERGE into the caches with change rows, the suspect-batch guard, and the apply in one transaction: candidates by the old rules (one packet is drawing plus lot plus operation; fold and ignore; project from the base prefix; unit is project plus the three-digit lot), the go-live boundary with the ignored list and pull-in, matching on the VISUAL key across live and removed packets (a removed packet is never recreated), new drafts, drafts updated in place, change orders opened, re-aimed, withdrawn or followed on for packets past draft (minor when only target hours or the want date changed), left-behind drafts retired, work order gone after two misses, closed and cancelled observed per leg, projects and units created with customer, PO and dates from the order links and the several-open-orders rule in one function with the provenance stored, auto-release of clean drafts up to 25 per run, audit rows and NOTIFY. +- `feed/fingerprint.py`: a canonical hash over legs, quantity, rev, drawing and BOM (major) and target hours and want date (minor), with per-field tests and invariance tests. +- The `drawing-lookup` job: a prioritised queue, a per-run budget, a new MFilesKit lookup per run, download only when the M-Files object or file version changed, sha256 into assets, pages through pypdfium2, the decision table (attach, rev mismatch staged for Clerical Supervisor approval, not released with the drafting status, not found, lookup failed with backoff, non-numeric rev), the H-sheet note, drawing-origin change orders for a re-released file or a newer rev on an in-shop packet, and the hand drawing taken off on a new VISUAL rev. +- Clerical: the From VISUAL panel (run now, status, last ten runs, ignored work orders with pull in, unmapped resources, projects without names), the Changes to approve column and KPI, flag kinds in the Flagged column and the find bar, rev-mismatch approval with reason and signature, hand upload of a drawing or a BOM (XLSX or CSV parsed on the server) on flagged drafts with reason and signature, change-order approve, reject and hold with signature, and the project's VISUAL orders with a pin. +- Admin: the Feed settings tab (intervals, shop hours, window, go-live date, departments, drawing recheck and budget, pause, next run, last ten runs) and the VISUAL resources editor. Every change is audited. +- `/api/feed/*`, `/api/jobs/*`, `/api/admin/feed/*`, and under `/api/packets/{id}`: `rev-approve`, `hand/drawing`, `hand/bom`. `GET /api/packets` gains `co=pending` and `flag=`. +- `packetstation job run ` and `packetstation fixtures use N`. Settings `FEED_SOURCE`, `DRAWING_SOURCE` and `FIXTURE_DIR`; a production stack refuses a fixture source. +- `tests/fixtures/generator.py` writes a synthetic feed in the exact VisualKit contract shape with the edge cases from the plan; CI feeds the worker from it and the Playwright scenario drives a feed run, a drawing attach and a change order on an in-shop packet. +- docs/FEED.md written in full; docs/SCHEMA.md, DEPLOY.md, POLICY.md and SECURITY.md updated. +- Integration run from a developer machine against VISUAL DEV and the vault (read-only, 2026-10-16): `visual-feed` ok against `VSQL\DEV / PESCOINCDEV`: 994 firmed work orders in the window, 4,090 operation rows, 13,465 material lines, 100 customer orders with 102 links, 474 new drafts, 47 units and 18 projects created, 232 lettered-lot rows skipped (lots such as 6A make no unit, as in the old build), no unmapped resources, no errors. `drawing-lookup` ok against the vault: 60 drawings looked up and fetched (the per-run budget), 213 drafts got their drawing attached, 68 flagged rev mismatch, none not found or failed. Nothing was written to VISUAL or M-Files. + - Phase 2: core domain. Migration 0003 with projects, project aliases and renumber history, units, packets and every child table (work orders, pages, BOM lines, shorts and receipts, welds, heat tags, heat table placements, stages, warehouse stage rows, transitions, Q.C. inspections and history, seals, packet log, VISUAL state), change orders with items, required departments and verifications, assets, packet documents, Q.C. documents, ITPs, user station limits and the department weld flag. Append-only triggers, soft-delete columns with partial unique indexes, server-assigned counters on packets. - The policy engine in Python (`server/packetstation/policy/`): Principal, PacketState, the Action enum, one rule per action, custody, named preconditions, the change-order hold, and the order of checks from the plan. Every packet read returns `can`, `blocked` with reasons, and `holder`. docs/POLICY.md lists the rules and the tightenings against the old build for Seth. - The write pipeline: one transaction per write with the packet row locked, the policy decision, compare-and-set on the expected status and department (409 "someone else already took this step" with the current state), If-Match row versions on child rows, server-assigned weld, heat tag, page and change order numbers, field-level compare-and-set on header scalars, packet version bump, packet log and audit rows, NOTIFY. Signatures through the phase 1 helper on every action the plan marks. Admin edits need an IT reason and are marked in the packet log and the audit. Filed packets are locked; filing writes an HMAC seal and Admin reopen breaks it with a reason. diff --git a/README.md b/README.md index 3c9aab1..842159f 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ This repo is the rebuild of Packet Station as a server application: Docker Compo ## Status -**Rebuild in progress. Phase 2 done: core domain, policy engine, packet API, Clerical screen.** +**Rebuild in progress. Phase 3 done: the VISUAL feed and M-Files drawing lookups run in the worker.** -People can sign in, set up projects and units, make hand drafts, attach a drawing, and send a packet to the warehouse, a department or Q.C. Every packet write goes through the server policy. The warehouse, shop and Q.C. screens are still placeholders; their API exists. The old build on `M:\Production\Packet Station` stays the live system until go-live. The new system starts empty; nothing is migrated from M:. +People can sign in, set up projects and units, make hand drafts, attach a drawing, and send a packet to the warehouse, a department or Q.C. The worker reads released work orders from VISUAL on a schedule, makes and updates drafts, opens change orders on packets already in the shop, looks drawings up in M-Files and attaches them at the rev VISUAL calls. Clerical sees it all on the From VISUAL panel and in the Flagged and Changes to approve columns; Admin sets the feed up on the Feed tab. Every packet write goes through the server policy. The warehouse, shop and Q.C. screens are still placeholders; their API exists. The old build on `M:\Production\Packet Station` stays the live system until go-live. The new system starts empty; nothing is migrated from M:. | Area | Exists yet? | |---|---| @@ -16,7 +16,7 @@ People can sign in, set up projects and units, make hand drafts, attach a drawin | Image, compose, CI, `doctor`, migration runner, security headers, build version endpoint | Yes (phase 0, image and stack verified on docker-1) | | Sign-in (AD password, PIN on registered devices), sessions, lock screen, Account settings, Admin shell, audit, SSE | Yes (phase 1, verified against the Samba DC in the CI stack on docker-1) | | Schema, policy engine, packet API, projects and units, assets, Clerical screen, packet page | Yes (phase 2, verified against PostgreSQL and the Samba DC in the CI stack on docker-1) | -| VISUAL feed and drawings | No (phase 3) | +| VISUAL feed, M-Files drawings, change orders from the feed, From VISUAL panel, Admin Feed tab and resource map | Yes (phase 3, run once against VISUAL DEV and the vault from a developer machine; fixture-driven in CI) | | Warehouse, shop, Q.C., close and file, PDFs | API only (phase 2); screens in phases 4 to 7 | | Project Management, Admin, notifications, backups | No (phases 8 and 9) | | Regression bench and go-live | No (phase 10) | @@ -78,7 +78,9 @@ docker compose -p packetstation-test --env-file /etc/packetstation/test/stack.en Phase 2 adds `/api/projects`, `/api/units`, `/api/packets` (header, transitions, work orders, pages, documents, BOM, shorts, welds, heat tags, heat table, stage, checklist, clock, warehouse stage, Q.C., change orders, log, seal), `/api/assets` and `/_blob/{id}`. Every packet read carries `can`, `blocked` and `holder` from the policy. See [docs/POLICY.md](docs/POLICY.md) and [docs/SCHEMA.md](docs/SCHEMA.md). -The `packetstation` command has `doctor`, `migrate`, `app`, `worker`, `ad-groups --list` (the 50 AD group names the app expects) and `ad-refresh` (create or refresh the users from the groups' members). +The `packetstation` command has `doctor`, `migrate`, `app`, `worker`, `job run ` (one job in the foreground), `fixtures use N` (switch the fixture state, never on prod), `ad-groups --list` (the 50 AD group names the app expects) and `ad-refresh` (create or refresh the users from the groups' members). + +Phase 3 adds the worker (`packetstation worker`: APScheduler, the `visual-feed`, `drawing-lookup` and `housekeeping` jobs, settings re-read on every fire, run now through `job_requests`), the VISUAL and drawing cache tables (migration 0004), `/api/feed/*`, `/api/jobs/*`, `/api/admin/feed/*`, rev approval and hand uploads under `/api/packets/{id}`, and the synthetic fixture generator (`python -m tests.fixtures.generator --out DIR`) that CI feeds the worker with. See [docs/FEED.md](docs/FEED.md). CI stack with the Samba domain controller and the Playwright scenario, on a Docker host: @@ -97,9 +99,9 @@ Server tests that need PostgreSQL run when `TEST_PG_SUPERUSER_URL` is set; the o - [docs/PLAN.md](docs/PLAN.md): the approved phased plan and open questions. - [docs/SECURITY.md](docs/SECURITY.md): old layers and where they live now, the new controls, the requirement ID table. - [docs/DEPLOY.md](docs/DEPLOY.md): host, stacks, secrets, NPM proxy host, doctor, backups. -- [docs/FEED.md](docs/FEED.md): the VISUAL feed and drawing jobs (phase 3). +- [docs/FEED.md](docs/FEED.md): the worker, the VISUAL feed, the packet rules, the fingerprint, the drawing decision table, the flags and the feed settings. - [docs/POLICY.md](docs/POLICY.md): who may do what, the preconditions, the order of checks, concurrency, and the tightenings against the old build. -- [docs/SCHEMA.md](docs/SCHEMA.md): the database schema (phases 1 and 2; phase 3 adds the caches). +- [docs/SCHEMA.md](docs/SCHEMA.md): the database schema (phases 1 to 3). - [docs/GO-LIVE.md](docs/GO-LIVE.md): go-live procedure (phase 10). - [docs/PAPER-FALLBACK.md](docs/PAPER-FALLBACK.md): what the shop does when the server is down (phase 10). - [CHANGELOG.md](CHANGELOG.md): what changed. diff --git a/deploy/ci/ci.env b/deploy/ci/ci.env index 2f4ff83..51cee16 100644 --- a/deploy/ci/ci.env +++ b/deploy/ci/ci.env @@ -94,6 +94,15 @@ SMTP_CA_BUNDLE=/etc/ssl/pesco/ca.pem # true logs notifications and sends nothing. NOTIFY_DRY_RUN=true +# ---- Worker sources ---- +# CI reads the synthetic fixtures, never VISUAL or M-Files. The e2e job generates them into +# tests/fixtures/generated, which compose.ci.yaml mounts at /fixtures. +FEED_SOURCE=fixture +# fixture: drawings..json and the PDFs it names, under FIXTURE_DIR. +DRAWING_SOURCE=fixture +# Fixture folder inside the containers. The scenario writes /fixtures/state to switch states. +FIXTURE_DIR=/fixtures + # ---- VisualKit (VISUAL ERP, read only) ---- # SQL Server instance. Full name: containers have no search domain. VISUAL_SERVER= diff --git a/deploy/compose.ci.yaml b/deploy/compose.ci.yaml index c400d32..9996580 100644 --- a/deploy/compose.ci.yaml +++ b/deploy/compose.ci.yaml @@ -10,6 +10,10 @@ # docker compose -p packetstation-ci --env-file deploy/ci/ci.env \ # -f deploy/compose.yaml -f deploy/compose.ci.yaml down -v # +# The worker reads the synthetic feed and drawings fixtures (FEED_SOURCE=fixture, DRAWING_SOURCE=fixture in +# ci.env) from ../tests/fixtures/generated, mounted at /fixtures. Generate them first: +# python -m tests.fixtures.generator --out tests/fixtures/generated +# # Never combine this overlay with compose.proxy.yaml. x-logging: &logging @@ -55,6 +59,7 @@ services: pull_policy: build volumes: - ci-dc-export:/etc/ssl/pstest:ro + - ../tests/fixtures/generated:/fixtures:ro # synthetic feed and drawings; the scenario switches state # Throwaway Samba AD DC (PSTEST.LOCAL) so LDAP sign-in runs over real TLS in CI. # Built from deploy/samba/. Runs as root, as Samba needs. No published port. @@ -101,9 +106,11 @@ services: environment: E2E_BASE_URL: http://ps-app:8000 E2E_DC_USERS_FILE: /export/users.json + E2E_FIXTURE_DIR: /fixtures CI: "true" volumes: - ../tests/e2e:/work/tests/e2e + - ../tests/fixtures/generated:/fixtures # read-write: the scenario writes /fixtures/state - /work/tests/e2e/node_modules # anonymous volume: node_modules stays out of the checkout - ci-dc-export:/export:ro networks: diff --git a/deploy/env/prod.env.example b/deploy/env/prod.env.example index 8788557..7c5aa62 100644 --- a/deploy/env/prod.env.example +++ b/deploy/env/prod.env.example @@ -36,6 +36,12 @@ TRUSTED_PROXY_HOST=npm_app_1 LOG_LEVEL=INFO # Worker heartbeat file read by its healthcheck. /tmp is a tmpfs. WORKER_BEAT_FILE=/tmp/packetstation-worker.beat +# Where the worker reads work orders: visualkit (the real VISUAL through VisualKit) or fixture (CI only; prod refuses it). +FEED_SOURCE=visualkit +# Where the worker looks up drawings: mfiles (the vault through MFilesKit) or fixture (CI only; prod refuses it). +DRAWING_SOURCE=mfiles +# Fixture folder inside the containers. Only read when a source is fixture. +FIXTURE_DIR=/fixtures # ---- PostgreSQL (roles made by deploy/postgres/initdb/01-roles.sh) ---- # postgres alias on the internal backend network. diff --git a/deploy/env/test.env.example b/deploy/env/test.env.example index 5a88109..d825912 100644 --- a/deploy/env/test.env.example +++ b/deploy/env/test.env.example @@ -36,6 +36,12 @@ TRUSTED_PROXY_HOST=npm_app_1 LOG_LEVEL=INFO # Worker heartbeat file read by its healthcheck. /tmp is a tmpfs. WORKER_BEAT_FILE=/tmp/packetstation-worker.beat +# Where the worker reads work orders: visualkit (the real VISUAL through VisualKit) or fixture (CI only; prod refuses it). +FEED_SOURCE=visualkit +# Where the worker looks up drawings: mfiles (the vault through MFilesKit) or fixture (CI only; prod refuses it). +DRAWING_SOURCE=mfiles +# Fixture folder inside the containers. Only read when a source is fixture. +FIXTURE_DIR=/fixtures # ---- PostgreSQL (roles made by deploy/postgres/initdb/01-roles.sh) ---- # postgres alias on the internal backend network. diff --git a/deploy/postgres/initdb/01-roles.sh b/deploy/postgres/initdb/01-roles.sh index ff16454..8803972 100755 --- a/deploy/postgres/initdb/01-roles.sh +++ b/deploy/postgres/initdb/01-roles.sh @@ -39,7 +39,7 @@ CREATE ROLE ps_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION P CREATE DATABASE packetstation OWNER ps_migrate ENCODING 'UTF8' TEMPLATE template0; REVOKE ALL ON DATABASE packetstation FROM PUBLIC; -GRANT CONNECT ON DATABASE packetstation TO ps_app; +GRANT CONNECT, TEMPORARY ON DATABASE packetstation TO ps_app; -- TEMP: the feed COPYs into temp tables \connect packetstation -- PostgreSQL 15+ already denies CREATE on public to PUBLIC. Make it explicit and give diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7dff644..621c706 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Container entrypoint. tini runs this as PID 1's child. -# Usage: app | worker | migrate [--bootstrap-roles] | doctor | version | healthcheck app|worker +# Usage: app | worker | job run | fixtures use [--dir DIR] | migrate [--bootstrap-roles] | doctor +# | version | healthcheck app|worker set -euo pipefail # MFilesKit reads its password from the environment only, so export it from the secret file. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 1a02e98..04986c6 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -32,7 +32,7 @@ Both stacks run from the same image as separate compose projects. - `postgres`: PostgreSQL 16, named volume, `backend` network only. Its init creates the database roles from secrets. - `migrate`: one-shot, runs `packetstation migrate` as the migrate role. - `app`: `packetstation app` (uvicorn). `backend` network plus egress plus `npm_default` through the proxy overlay. Assets volume, CA bundle bind, secrets, healthcheck. -- `worker`: `packetstation worker`. Same image. Assets volume, backup folder read-only (planned, phase 9). +- `worker`: `packetstation worker`. Same image. Runs the VISUAL feed, the drawing lookup and housekeeping (see [FEED.md](FEED.md)). `backend` network plus egress (VISUAL, M-Files), no `npm_default`, no published port. Assets volume (fetched drawings land there), CA bundle bind, backup folder read-only (used from phase 9). Its healthcheck reads the heartbeat file. - All services restart unless stopped, log to stdout (json-file), and have healthchecks. ## Secrets @@ -52,6 +52,8 @@ Docker secrets, one file each, under `/etc/packetstation//secrets/`. Owne Never put a secret in the repo, the env file, a log or chat. Settings read secrets through `*_PASSWORD_FILE` paths. +The worker gets `postgres_app_password` (it logs in as the app role, with application name `ps-worker`), `visualkit_password`, `mfileskit_password`, `ldap_bind_password` and `seal_key`. The worker never logs a password: job errors are redacted before they are stored, and the redaction also strips the VISUAL and M-Files passwords themselves. + The PESCO CA chain is mounted into the containers once and referenced by `LDAP_CA_BUNDLE`, `MFILES_CA_BUNDLE` and `SMTP_CA_BUNDLE`. ## Env files @@ -66,8 +68,14 @@ Key settings: - `SMTP_HOST=app-03.pesco.com`, `SMTP_PORT=25`, `SMTP_STARTTLS=true`, `SMTP_FROM=noreply@pescoinc.biz`, `SMTP_CA_BUNDLE`. - `APP_ALIAS` per stack. - Identity timing and limits, all optional with these defaults: `SESSION_IDLE_MINUTES=15`, `SESSION_LOCK_MINUTES=3`, `SESSION_HIDDEN_LOCK_SECONDS=60`, `SESSION_PASSWORD_HOURS=12`, `SESSION_PIN_HOURS=10`, `ROLES_REFRESH_MINUTES=15`, `SIGNIN_ACCOUNT_LIMIT=5`, `SIGNIN_ACCOUNT_WINDOW_MINUTES=5`, `SIGNIN_ACCOUNT_COOLDOWN_MINUTES=10`, `SIGNIN_IP_LIMIT=30`, `SIGNIN_IP_WINDOW_MINUTES=5`, `PIN_MAX_FAILURES=5`, `STEPUP_MAX_FAILURES=3`, `DEVICE_COOKIE_DAYS=365`, `PIN_MIN_DIGITS=6`, `PIN_MAX_DIGITS=12`. +- Worker sources, all optional: `FEED_SOURCE` (`visualkit`, the default, or `fixture`), `DRAWING_SOURCE` (`mfiles`, the default, or `fixture`) and `FIXTURE_DIR` (default `/fixtures`). A fixture source reads files instead of VISUAL or M-Files (see [FEED.md](FEED.md), "Fixture sources"). `STACK=prod` refuses a fixture source at start. The test and production stacks leave all three unset. Only the CI env file sets them. +- `SITE_ID` (default `PESCO`) and `SITE_TZ` (default `America/Denver`): the VISUAL site in every call, and the time zone for shop hours and the schedule. - The session cookie is marked `Secure` when the request arrived over https (as reported by NPM through `X-Forwarded-Proto`). Behind the proxy that is always the case. +## Feed settings live in the database + +The feed's schedule and rules are not env settings. Intervals, shop hours, the window, the go-live date, departments, the drawing recheck and budget, labor tickets, auto-release and the pause switch are one `settings` row (key `feed`), edited by an Admin on the Admin page, Feed tab. The resource map is the `visual_resource_map` table, edited on the "VISUAL resources" tab. A save notifies the worker, which applies it without a restart. A new stack starts with the defaults in [FEED.md](FEED.md), "Settings". Set the go-live date before the production feed's first run. + ## Deploy command Test stack: @@ -158,6 +166,17 @@ The DC runs with `cap_add: [SYS_ADMIN]`, which Samba needs to set the sysvol ACL The real PESCO domain controllers are never used by CI. The test and production stacks do not run the DC. +### CI feed fixtures (phase 3) + +CI never reads VISUAL or M-Files. `deploy/ci/ci.env` sets `FEED_SOURCE=fixture`, `DRAWING_SOURCE=fixture` and `FIXTURE_DIR=/fixtures`. The image does not contain `tests/`, so the e2e job generates the fixtures on the runner before the stack starts: + +```bash +python -m pip install pypdf==6.21.0 +python -m tests.fixtures.generator --out tests/fixtures/generated +``` + +The generator is deterministic and writes `feed.1.json`, `feed.2.json`, `drawings.1.json`, `drawings.2.json`, the PDFs they name and `state` (`1`). `tests/fixtures/generated/` is ignored by git. `deploy/compose.ci.yaml` binds it at `/fixtures` into the worker (read-only) and the Playwright container (read-write, so the scenario can write `state` = `2` and press Run now again). `packetstation fixtures use N --dir DIR` switches the state by hand. The CI feed runs at the default intervals; the scenario does not wait for them, it presses Run now. + ## Gitea Actions runner (phase 2) `.gitea/workflows/ci.yml` runs on a Gitea Actions runner named `docker-1`, installed on docker-1 at diff --git a/docs/FEED.md b/docs/FEED.md index d018d34..4bf39ee 100644 --- a/docs/FEED.md +++ b/docs/FEED.md @@ -1,43 +1,426 @@ # Packet Station feed and drawings -Status: planned, phase 3. This is a stub. It lists what the approved plan says this document will contain. See [PLAN.md](PLAN.md), "Worker, feed and drawings". +Status: phase 3. This file says how the worker reads VISUAL and M-Files and what it does with what it reads. The design is in [PLAN.md](PLAN.md), "Worker, feed and drawings". When this file and the code disagree, the code is the rule and this file is out of date. ## Purpose -The worker reads VISUAL through VisualKit and M-Files through MFilesKit on a schedule. It caches the rows, applies the packet rules and works the drawing queue. Screens read PostgreSQL only. +The worker is the only part of Packet Station that talks to VISUAL and M-Files. It reads VISUAL through VisualKit and M-Files through MFilesKit, both read-only, on a schedule. It stores what it read in cache tables, turns the rows into projects, units, draft packets and change orders, and works the drawing queue. Screens read PostgreSQL only. + +There are three reasons for a scheduled job instead of reads per screen: + +- VISUAL production must not be queried on page loads. One modest query every few minutes from one place is fine. One per user per screen is not. +- Turning VISUAL rows into packets is a state change. It runs in one place, in one transaction, with an audit row. +- Drawing lookups and downloads are slow. They belong in the background. + +The worker never writes to VISUAL or M-Files. ## Worker process -`packetstation worker` runs APScheduler. One run per job at a time, guarded by a PostgreSQL advisory lock per job. Settings are re-read on every tick, so an Admin change applies without a restart. The pause switch stops scheduled feed and drawing runs. "Run now" still works. +`packetstation worker` starts the worker. It runs the stack guard, builds the scheduler and blocks. The main thread writes the heartbeat and waits for a stop signal. + +### Scheduler + +- APScheduler 3 with a pool of 4 threads. Each job has `coalesce` on, `max_instances` 1 and a misfire grace of 300 seconds. +- Times are scheduled in site time (`SITE_TZ`, default `America/Denver`) and stored in UTC. +- Jobs are registered in `worker/jobs/__init__.py`. Each job has a name, a run function, a cadence function and a `pausable` flag. A job can name others to chain after it: `visual-feed` chains `drawing-lookup`, so every feed run is followed by a drawing run. +- The cadence function returns the seconds to the next run, from the feed settings and the current site time. + +### Settings re-read + +The feed settings live in the database (see "Settings"), not in the env file. The worker keeps one connection that listens on two channels: + +- `ps_settings`: the app sends it when an Admin saves the Feed tab. The worker reloads the settings and recomputes the next run of every job. A new interval or the pause switch applies at once, without a restart. +- `ps_job_request`: see "Run now" below. + +The listening connection reconnects with backoff when it drops. + +### Pause + +When `paused` is on, scheduled fires of the pausable jobs (`visual-feed`, `drawing-lookup`) are skipped. Each skip is recorded as a `job_runs` row with status `skipped` and counts `{"reason": "paused"}`. Auto-release does not run while paused. "Run now" still works while paused. Housekeeping is not paused. + +### Run now and job_requests + +1. Clerical presses **Run now** on the From VISUAL panel (or an Admin on the Feed tab). The app inserts a `job_requests` row and sends `NOTIFY ps_job_request` with `{"job": ""}` in the same transaction. +2. The worker moves that job's next run to now. +3. The run claims every pending request for the job at its start. Ten clicks make one run. The run's trigger is `request` when it claimed any. +4. A request that arrives while the job is running is left pending, and the job runs again straight after. + +### Advisory lock + +Each run opens its own connection and takes `pg_try_advisory_lock(7310, hashtext())`. When the lock is held, the run records status `skipped` with counts `{"reason": "already running"}` and stops. One run per job at a time, whether the trigger was the schedule, a request, a chain or the command line. + +### job_runs + +Every run writes one `job_runs` row: + +- `trigger`: `schedule`, `request`, `chain` or `cli`. +- `requested_by`: the users whose requests the run claimed. +- `started_at`, `finished_at`. +- `status`: `running`, then `ok`, `degraded` (finished but part of it failed, for example the order read), `failed` (an exception), `skipped` or `abandoned`. On start the worker marks its own host's rows still `running` as `abandoned`. +- `counts`: a JSON object (below). +- `errors`: at most 20 strings of at most 500 characters each, redacted. Redaction strips anything that looks like `password=...`, `pwd=...` or an `Authorization:` header, and the VISUAL and M-Files passwords themselves when they are in the environment. +- `build_version`, `host`, and `visual_target` (`" / "`) on feed runs. + +Feed run counts, all integers: `wos_window`, `wos_tracked`, `rows`, `materials`, `orders`, `labor_legs`, `new_drafts`, `drafts_updated`, `cos_opened`, `cos_reaimed`, `cos_withdrawn`, `retired`, `wo_gone_flagged`, `closed_observed`, `ignored_golive`, `projects_created`, `units_created`, `auto_released`, `unmapped_resources`, `skipped_removed`, `skipped_closed` and `suspect` (0 or 1). `counts["unmapped"]` holds `{resource: rows}` for the top 30 unmapped resources. + +Drawing run counts: `checked`, `fetched`, `unchanged`, `attached`, `flagged` (`{kind: n}`), `lookup_failed`, `cos_opened`, `staged` and `budget_left`. + +`packetstation job run ` runs one job once in the foreground with trigger `cli`, prints its `job_runs` row as JSON, and exits 1 when the run failed. + +### Heartbeat + +The worker writes the `worker_heartbeat` row every 30 seconds (time, build, host, pid and the jobs running now), and touches the beat file the container healthcheck reads. The From VISUAL panel shows "Worker not running" when the heartbeat is stale. + +### Stack guard + +At start the worker runs the same stack guard as the app: the test stack refuses a production VISUAL target, and the production stack refuses a VISUAL target that was not set explicitly. Before every feed run the guard runs again with fresh VisualKit settings. A failed guard fails the run. The run records the target it read. ## Jobs and cadence -visual-feed (default every 5 minutes inside shop hours, 30 outside), drawing-lookup (after every feed run, plus its own cadence), session-refresh (15 minutes), ad-reconcile (nightly), integrity, backup-check, notify-send, notify-digest and housekeeping. Each run is recorded in `job_runs` with its counts and errors. +| Job | Cadence | Pausable | What it does | +|---|---|---|---| +| `visual-feed` | `interval_shop_minutes` inside shop hours (default 5), `interval_off_minutes` outside (default 30) | yes | The five reads, the cache, the packet rules, auto-release. Chains `drawing-lookup`. | +| `drawing-lookup` | After every feed run, plus every `drawing_interval_minutes` (default 30) | yes | The drawing queue (see "Drawings"). | +| `housekeeping` | Daily at 03:30 site time | no | Prunes old rows. The plan keeps the caches 90 days and `job_runs` 180 days. | + +Planned, not built in phase 3: + +- `session-refresh` (every 15 minutes): re-resolve groups and the account state for every live session. +- `ad-reconcile` (nightly 01:30 site time, plus "refresh from AD now"), phase 9. +- `integrity` (04:00): seals, asset hashes, missing and orphan files, the audit hash chain. Phase 9. +- `backup-check` (07:15): the newest dump is under 26 hours old. Phase 9. +- `notify-send` (every 60 seconds) and `notify-digest` (daily at the digest time). Phase 9. + +`job_schedule` holds each job's next run, interval and pause state for the Admin tab. Migration 0004 seeds rows for `visual-feed`, `drawing-lookup` and `housekeeping`. ## Feed reads -Five reads per run: released work orders in the window (firmed only, `until` always passed), every unfiled packet's work orders by key, materials for both sets, order links, and labor tickets if the timer decision keeps them. A work order that leaves the window is not a removal. +Each feed run opens one VisualKit connection (`Visual.from_env()`), does five reads outside any database transaction, and closes it. Every value goes through one parser, so VisualKit and the fixture source produce the same data. The site is `SITE_ID` (default `PESCO`) in every call. + +The window comes from today in site time: `since` is today minus `window_back_days` (default 14), `until` is today plus `window_ahead_days` (default 30). + +1. **Released work orders in the window.** `v.work_orders.released(site, since, until, status=("F",))`, paged. Firmed only. `until` is always passed. This read creates and updates drafts. +2. **Tracked work orders by key.** `v.work_orders.get_many(keys)` for every work order key on a live packet that is not filed, plus every ignored work order Clerical pulled in. No status filter and no date filter, so a packet already in the shop keeps getting changes and its closed state. Keys VISUAL no longer returns are collected as missing. +3. **Operations, text, parts and materials** for the union of reads 1 and 2: `operations_for`, `operation_text_for` (the planner instructions, called partspecs), `parts.get_many` for descriptions. Materials: `work_orders.for_bases(bases, type="W", status=("F", "R"))`, `requirements_for`, `requirement_text_for`, `parts.get_many` and `parts.sites_for`. Materials are known per leg: the legs `for_bases` returned are the legs whose BOM the run knows (`mat_known`). A closed leg's materials are not read, and that does not look like a removed BOM. +4. **Order links** by work order base and by project, through `v.links.orders_for_bases`, `v.links.orders_for_projects`, `v.orders.get_many` and `v.orders.customers`. When this read fails, the run is degraded and the apply keeps using the last cached orders. +5. **Labor tickets**, when `labor_tickets` is on and the last labor read is more than 30 minutes old: `v.labor.tickets_for_work_orders(keys, since=...)`, summed to hours and ticket count per leg and operation. Employee 9999 is left out (VisualKit's default). No cost column is ever read. + +A work order that leaves the window is not a removal. Once a packet tracks it, read 2 keeps reading it whatever its date or status. + +### Suspect batch + +A batch is suspect when: + +- the window read returned zero rows and the previous batch had some, or +- more than 5 tracked keys are missing and they are more than 10% of the tracked keys. + +A suspect batch is still cached and applied, but without the "gone" marking in the cache and without the "work order gone" step. The run is degraded and the reason is in its counts. + +## Cache + +`feed/cache.py` writes one batch per run, in one transaction: + +- **Batch.** A `visual_batches` row: run, source (server and database, or the fixture state), site, window, window rows, tracked keys, missing keys, suspect, counts. +- **Rows.** The rows are copied into a temporary table and merged into `visual_rows`, keyed on the typed key (base, lot, split, sub, seq). A new row gets `first_batch` and `last_batch`. A row whose hash changed is updated. A row seen again moves `last_batch`. `row_hash` is a sha256 of the canonical row (the same canonical form as the fingerprint). +- **Materials.** The same for `visual_materials`, keyed on the typed key plus piece, with `mat_hash`. +- **Change rows.** Each insert, change, return and disappearance writes a `visual_row_changes` or `visual_material_changes` row for the batch, with the row before and after. Kinds: `added`, `changed`, `gone`, `returned`. Admin reads these as "what VISUAL changed". +- **Gone marking.** A cached row that is in play (its key is tracked, or it is firmed with its release date inside the window) and was not in this batch gets `gone_batch` and a `gone` change row. Materials are in play when their base is. A row that comes back clears `gone_batch` and gets a `returned` change row. A suspect batch marks nothing gone. +- **Orders.** `visual_orders` is upserted and `visual_order_links` is replaced for the bases and projects read. +- **Labor.** `visual_labor` is upserted per leg and operation. + +The apply then reads the rows back from the cache, so it works on one consistent snapshot. ## Packet rules -One packet is drawing + lot + operation. The project comes from the work order prefix. The unit is project + 3-digit lot. New drafts are created, drafts are updated in place, packets past draft get change orders, removed packets are never recreated, and untouched drafts whose legs all moved are retired. A work order gone from VISUAL is flagged, not removed. +### One packet + +One packet is one drawing, one lot and one operation (resource). Its match key, `packets.visual_key`, is `||`, with `(no drawing)` when the drawing is blank. Several work order operations (legs) can share one packet. + +### Resource map + +Each VISUAL resource maps to a shop department code, `fold` or `ignore` (`visual_resource_map`, seeded from the old build's `visual_map.json`, edited on the Admin "VISUAL resources" tab). + +- A department code makes packets in that department. +- `fold` makes no packet. The row's operation is listed on the packets of the same work order and lot as a folded operation. A resource starting `MAT_` with no map entry is fold. +- `ignore` makes nothing. +- A resource with no entry makes nothing and is counted under `unmapped`, so IT can assign it. +- When `departments` is set in the feed settings, rows routed to other departments are skipped silently. + +### Project and unit + +- The project number is the work order base's prefix as the old build took it. +- The unit is the project number plus the lot zero-padded to 3 digits (`L900` and lot `1` make `L900001`). +- A lot that is not all digits (lot `7A`) makes no candidate. The old build dropped them the same way. Hand units may still use such a lot. +- A project or unit that does not exist is created (see the apply, step 1). + +### Which rows take part + +A row takes part when it routes to a department and either its key is tracked (any status, any date), or it is firmed with a release date inside the window. A row that is not tracked and was released before the go-live date is set aside (see "Go-live boundary"). + +### The candidate + +Rows are grouped by match key. For each group: + +- Legs are deduplicated on (key, seq). +- `rel` is the earliest release date, `want` the latest want date. +- `rev` is the highest numeric rev among the legs. A non-numeric rev only wins when there is no rev yet. +- `title` is the first leg's description. +- `folded` lists the fold rows of the same work order and lot. `partspecs` lists the distinct planner instructions of every row of the same work order and lot, in first-seen order. +- The BOM is built from the materials (the old `matBom`): + - A line belongs to a leg when its drawing is blank or matches the leg's drawing, ignoring spaces and case. + - Per piece the best status wins: R, then F, then U, then anything else. + - Lines without a part are dropped, except as group headers: a piece that is a multiple of 10 heads the next 9 pieces. + - The item is the piece number, with `-` added on a repeat. + - A line is a sub-assembly and n/a when its commodity is WIP or its warehouse is ASY. + - The location is the warehouse (unless MAIN) and the location, joined by a space. + - When no leg of the candidate has known materials, the BOM is unknown and the packet's BOM is left alone. + +### The apply + +The apply runs in one transaction with a 5-second lock timeout. A packet row locked by someone else is skipped and picked up next run. Steps, in order: + +1. **Projects and units.** Find each candidate's project through the alias table, so renames resolve. Create a missing project (source `visual`, no name, `needs_details` on, due = the latest want date) and its alias. Create a missing unit the same way. Fill the order-derived fields (see "Several open orders"). Candidates under a closed project or unit are skipped (`skipped_closed`). +2. **Match.** Look up every candidate key among packets, removed ones included. A key whose only packets are removed makes nothing, ever (`skipped_removed`). A key with a live packet is existing. Anything else is new. +3. **New drafts.** Apply the go-live boundary first. Then insert the draft: header, work orders per leg (qty, target hours, op, status, partspecs, drawing, rev), BOM lines (source `visual`), the visual state row, `drawing_state` `needed` and flag `drawing_needed` (or `none` and flag `no_drawing` when the drawing is blank), the fingerprint. Log "Packet created from VISUAL" by "VISUAL feed". Audit `packet.created` with role `system`. +4. **Drafts updated in place** when the fingerprint or the status facts changed (action `vis_update_draft`): + - Work orders are replaced, keeping rows whose key and seq match. + - The BOM is replaced unless Clerical uploaded one by hand. + - When the drawing or the rev moved: `drawing_state` goes back to `needed`, every drawing page is taken off (the file stays in assets), a hand drawing is taken off, any approved rev is cleared, the drawing flags are cleared and `drawing_needed` is added. + - The fingerprint, want and release dates, folded operations and partspecs are stored. + - Log "Updated from VISUAL" with the lines that changed ("work order X added", "X target 1.5 h -> 2 h", "drawing rev 1 -> 2", want date, material lines). +5. **Past draft: change orders** (action `vis_open_co`). See "Change orders from the feed". Filed packets get nothing. +6. **Retire moved drafts** (action `vis_retire_draft`). A live draft made by the feed, with no drawing pages and no hand uploads, that is not among this run's candidates, and whose every leg now sits on another live candidate, is soft-removed with reason `moved`. `visual.moved` records where the legs went. Log "Retired: work orders moved to another packet". It is hidden from the boards and, being removed, is never recreated. +7. **Work order gone.** Skipped on a suspect batch. Each tracked leg VISUAL did not return gets `missing_runs + 1`. At 2 the leg gets `gone_at`, the packet gets flag `wo_gone` and `wo_gone_at`, and the log says "Work order no longer in VISUAL". A leg that comes back resets to 0. When no leg is gone any more the flag goes and the log says "Work order back in VISUAL". The feed never removes a packet for this. +8. **Closed and cancelled.** Each leg's status code is stored, with ticket hours and the time observed. When every leg is C, `visual_closed_at` is set and the log says "Closed in VISUAL (observed)". Any leg X sets flag `cancelled` and `visual_cancelled_at`. The flag goes when no leg is X. A cancelled draft is not removed. +9. **Auto-release** when `auto_release` is on and the feed is not paused: up to `auto_release_max` (default 25) live drafts made by the feed with the drawing attached, no flags at all, and the rev confirmed by the drawing job. A draft whose heat map is unanswered gets "yes" when it has a real material line (not a sub-assembly, not n/a, not consumable) and "no" otherwise. Then action `vis_auto_release` moves it to the warehouse with an automatic warehouse stage row and a transition row. The policy's preconditions (heat map answered, drawing attached) still apply. +10. **Summary.** One audit row `feed.run` with the counts, and one `feed` event to the screens after commit. + +Every packet write in the apply writes a `packet_log` row (by "VISUAL feed", role `system`), an audit row (`packet.`, role `system`) and a change event, through the same rules as a person's write (see [POLICY.md](POLICY.md), "Worker writes"). ## Fingerprint -A canonical hash over the fields a packet is built from. Major parts (legs, qty, rev, drawing, BOM) open a change order. Minor parts (tgt, want) need Clerical approval only. Tests show what each field does. +The fingerprint is a canonical hash over the fields a packet is built from. `feed/fingerprint.py` computes it per candidate. `packets.visual_fp_applied` stores the full hash, and `packets.visual.fp_parts` stores the hash of each part, so the next run can say which parts changed. + +Canonical form: text in NFC, line ends as LF, trimmed; null and empty text alike; IDs (work order base, part, drawing, lot, split, sub) upper-cased; lot, split and sub kept as text; seq and piece as integers; numbers normalised (`2` and `2.0` are the same); dates in ISO form; lists sorted (legs by base, lot, split, sub, seq; BOM by item). JSON with sorted keys, then sha256. + +| Field | Part | What a change does past draft | +|---|---|---| +| Legs (work order keys and operations on the packet) | major | Opens or re-aims a change order | +| Quantity per leg | major | Opens or re-aims a change order | +| Rev | major | Opens or re-aims a change order, waiting for the drawing at the new rev | +| Drawing number (upper-cased, spaces removed) | major | Opens or re-aims a change order, waiting for the new drawing | +| BOM (item, part, qty per line) | major | Opens or re-aims a change order | +| Target hours per leg | minor | Opens a minor change order: Clerical approval only, no department verifies | +| Want date | minor | Opens a minor change order | +| Release date | never | Nothing | +| Folded operations | never | Nothing | +| Partspecs (planner instructions) | never | Nothing | +| Title | never | Nothing | +| Work order status | never | Nothing (status is observed separately: closed, cancelled) | +| Issued quantity, warehouse, location | never | Nothing | +| Orders, labor, department, operation | never | Nothing | + +On a draft any fingerprint change updates the draft in place, major or minor. A change order is only for packets past draft. Tests in `tests/server/test_fingerprint.py` show each row. ## Go-live boundary -Work orders released before the go-live date are ignored and listed. Clerical can pull one in by hand. +With no migration from the M: install, the first run would make drafts for every firmed work order in the window, including jobs already running on paper or on M:. The `golive_date` setting keeps them out: + +- A row that is not tracked and was released before the go-live date makes no candidate. +- A candidate none of whose legs is tracked and whose earliest release is before the go-live date is not created. Its legs go into `visual_ignored_wos` (`ignored_golive`). +- The From VISUAL panel lists the ignored work orders, newest release first. **Pull in** asks a reason, records who and when, and requests a feed run. A pulled-in work order is tracked from then on, so the next run makes its draft whatever its release date. + +Jobs already open on M: finish there. + +## Change orders from the feed + +A packet past draft (warehouse to complete) is never changed silently. A fingerprint change opens a change order instead. + +- **Open.** A changed fingerprint opens a change order with origin `visual`, review `pending`, ECN `VISUAL`, issued by nobody. It carries `vis` (from and to drawing and rev, the list of changes in words, the legs, the BOM, want and release dates, folded operations, partspecs and the fingerprint parts), `target_fp`, `minor`, and `waiting` (`{dwg, rev, since}`) when the drawing or rev changed and the new drawing is still needed. +- **One at a time.** At most one pending feed change order per packet (origin `visual` or `drawing`). A further change re-aims it: reason, `vis`, `waiting`, `target_fp` and `minor` are updated, the move is appended to `vis.moved`, and a staged drawing is kept when the target drawing and rev did not change. +- **Unchanged.** A pending change order with the same target does nothing. A rejected one with the same target does nothing either: Clerical rejected exactly this. +- **Withdraw.** When VISUAL goes back to what the packet has, the pending change order is withdrawn: review `rejected`, rejected by nobody, with the reason in the log. +- **Follow on.** An approved change order with the same target does nothing. A change after an approval opens a fresh change order. +- **Minor.** When only target hours or the want date changed, the change order is minor. It needs Clerical approval only. On approval no department is asked to verify. +- **Hold.** A pending change order holds nothing unless Clerical puts it on hold. While held it blocks the packet's held steps (see `v_open_cos` in [SCHEMA.md](SCHEMA.md)). +- **Approval.** Clerical approves with a signature. When the drawing changed, approval needs the new drawing staged first ("The new drawing has not been fetched from M-Files yet."). Approval applies `vis`: work orders from `vis.legs`, the BOM unless it was uploaded by hand, the header drawing and rev, the fingerprint, want and release dates. A staged drawing replaces the old drawing pages: the old pages are voided (never deleted) and the new pages added, with the page changes in `co_items`. A major change order then asks the departments it touches to verify, and holds the packet until they have. A minor one asks nobody. +- **Reject.** Clerical rejects with a reason and a signature. + +A packet at `complete` still gets a change order. A filed packet gets nothing. ## Drawings -Drawing lookups go through MFilesKit with the matching rules in MFilesKit's `docs/packet-station.md`. Those docs are stale in two lines (drafting rev precedence and the project-folder rule). The code is the rule, and this document will state the code's behaviour. +The `drawing-lookup` job finds each packet's drawing in M-Files through MFilesKit, downloads it, and attaches it to drafts. + +### How MFilesKit finds a drawing + +This is what `mfileskit/drawings.py` does. MFilesKit's own `docs/packet-station.md` is stale in two places, noted below. + +- **Key.** Hyphens and spaces removed, upper-cased: `L073-76-2` becomes `L073762`. +- **Search by Name property, not folders.** The released PDF is searched in class "Drawing Production (PDF)" by the Name property starting with the key, and a hit counts only when its name is the key or the key plus `X`. There is no project-folder rule any more (the `P-521` to `H777` mapping in the MFilesKit docs is gone). +- **Released rev.** Only a document whose Controlled Status (or workflow state) is Effective counts. The rev is its Controlled Version Label, else the number after `R` in its title. The highest released rev wins. +- **Drafting rev.** Working copies are searched by Controlled Version Title. Copies in a done status (released, effective, retired, obsolete) or a done state (drawing released to shop, drawing already exists, cancel drawing) are skipped. The drafting rev is the highest label among the rest. Its workflow state maps to "still being drawn (unapproved)" or "in the drafting check cycle". (The MFilesKit docs still say "the higher of the Unapproved rev and the Check Cycle rev"; that is stale.) +- **In drafting.** A Drafting Assignment named for the key means the drawing is in drafting. +- **H sheets.** A drawing like `K978-01-6H13` has no object of its own. When nothing is found for it (no released PDF, no working copy, no assignment), the lookup tries the base drawing `K978-01-6` and reports `base`. The packet note then says "sheet of drawing K978-01-6". +- A search that hits the 500-result cap is a failed lookup, so a wrong rev is never reported. + +### Queue and priorities + +The queue covers live packets made by the feed, not filed, with a drawing number, in the departments in scope. It is distinct by drawing number, and the first item's priority wins: + +1. The VISUAL rev changed since the last lookup of that drawing, or a pending change order waits for a drawing at a target rev. +2. Drafts with `drawing_state` `needed`. +3. Failed lookups whose backoff is over (15 minutes doubled per failure, at most 24 hours). +4. Rechecks due: last checked more than `drawing_recheck_hours` ago (default 4). + +### Budget + +At most `drawing_budget` lookups per run (default 60). The rest wait for the next run. The worker opens one M-Files vault per process and a new `DrawingLookup` per run, so its cache never outlives a run. When the vault cannot be opened, the run is skipped and degraded with one error. + +### Download + +The file is downloaded only when the object id and version or the file id and version differ from the last lookup, or nothing is stored yet. The download goes to a temporary file, is checked for the PDF magic bytes and the upload size limit, hashed with sha256 while streaming, then fsynced and renamed into the assets volume. A file with the same sha256 and size reuses the existing asset. Pages are counted with pypdfium2 (pypdf as a fallback). Each fetch writes an audit row `drawing.fetched`. + +### Decision table + +`drawings/decide.py` decides per packet sharing the drawing. Rows are checked top to bottom. + +| Case | Decision | Flag | What happens | +|---|---|---|---| +| The lookup failed | `lookup_failed` | `lookup_failed` | What is attached stays. `drawing_state` becomes `lookup_failed` only if it was `needed`. Retried with backoff. | +| No released rev, but a drafting rev or a drafting assignment | `not_released` | `not_released` | Note "Not released in M-Files yet (sheet of ) - drafting status: rev N ". | +| No released rev and nothing in drafting | `not_found` | `not_found` | Note "Drawing not in M-Files - check the drawing number in VISUAL". | +| VISUAL's rev is not a whole number | `bad_rev` | `bad_rev` | Not attached. | +| Revs differ, and no approved rev matches M-Files | `rev_mismatch` | `rev_mismatch` | Note "Rev mismatch: VISUAL calls rev V, M-Files has rev N", plus the drafting status. The M-Files copy is fetched and staged for Clerical Supervisor approval. No change order: VISUAL did not change. | +| Revs match (or the approved rev matches), draft | `attach` | none | Drawing pages attached, `drawing_state` `attached`, drawing flags cleared, rev confirmed. Log "Drawing attached from M-Files". | +| Revs match, draft with a hand drawing at the same rev | `unchanged` | none | The hand drawing stays. | +| Revs match, past draft, same file (same sha256) | `unchanged` | none | Nothing. | +| Revs match, past draft, same rev but a new file | `open_co` | none | Drawing-origin change order: "M-Files re-released rev N of (new file)". | +| Past draft, M-Files released a higher rev than the one attached | `open_co` | none | Drawing-origin change order: "M-Files released rev N of ; this packet has rev M". Never a silent swap. | +| A pending feed change order waits for this drawing at the rev M-Files now has | `stage_for_co` | none | The fetched file is staged on the change order and `waiting` is cleared. Log "Drawing for change order N ready for Clerical". | + +A flag decision sets `drawing_state`, replaces the drawing flags with the decision's flag, stores the M-Files answer in `visual.appr`, and clears the rev confirmation. On a draft it is a `vis_update_draft` write. Past draft it is a `vis_open_co` write logged "Drawing status from M-Files", since that is the only system action allowed past draft. + +"Drawing needed" blocks the warehouse step through the `drawing_attached` precondition. + +### Rev mismatch staging and approval + +When M-Files has a different released rev from VISUAL, the M-Files copy is fetched and its asset is kept on the `drawing_lookups` row. A Clerical Supervisor can approve it for one draft with a reason and a signature (action `rev_approve`). The pages are attached, the drawing flags are cleared, and `visual.rev_ok` records the rev, the VISUAL rev, who, when and why. The log says "Different rev approved by hand" with "rev N in place of VISUAL rev M". Without a staged copy the answer is "nothing staged". + +### Hand drawing taken off on a new rev + +A draft whose VISUAL drawing or rev moves loses every drawing page, a hand drawing included (step 4 of the apply). The drawing job then looks up the new rev. A hand drawing at the same rev is left alone. + +### Drawing-origin change orders + +A drawing change on a packet past draft opens a change order with origin `drawing`, ECN `M-FILES`, the new file already staged, and no target fingerprint. It counts as the packet's one pending feed change order. Approval voids the old drawing pages and adds the new ones, as for a feed change order. ## Flags -What each flag means and what Clerical does about it: rev mismatch, not released, not found, lookup failed, non-numeric rev, work order gone, cancelled, closed in VISUAL. +Flags live in `packets.flags`. A draft with any flag shows in Clerical's Flagged column. Auto-release never takes a packet with a flag. The find bar filters by flag kind. + +| Flag | Text shown | What Clerical does | +|---|---|---| +| `drawing_needed` | Made from VISUAL - drawing not attached yet | Usually nothing: the drawing job attaches it. Run now to hurry it. Upload by hand only if it stays stuck. | +| `rev_mismatch` | The M-Files note, for example "Rev mismatch: VISUAL calls rev 2, M-Files has rev 3" | Get the rev corrected in VISUAL, or a Clerical Supervisor approves the staged M-Files rev with a reason and a signature. | +| `not_released` | The note: "Not released in M-Files yet ... drafting status: rev N " | Wait for drafting to release it, or upload the drawing by hand with a reason and a signature. | +| `not_found` | Drawing not in M-Files - check the drawing number in VISUAL | Check the drawing number in VISUAL and have it fixed there. Or upload by hand. | +| `lookup_failed` | M-Files lookup failed: (the reason) | Nothing at first: the job retries with backoff. If it persists, tell IT. | +| `bad_rev` | VISUAL rev is not a number - drawing not attached | Have the rev fixed in VISUAL, or upload by hand. | +| `no_drawing` | (the draft has no drawing number) | Have the drawing number added in VISUAL, or upload by hand. | +| `wo_gone` | Work order no longer in VISUAL | Check VISUAL. If the job was deleted on purpose, remove the draft by hand (or handle the in-shop packet). The flag clears itself if the work order comes back. | +| `cancelled` | Work order cancelled in VISUAL | Decide: remove the draft by hand, or leave it if the cancel was a mistake. The feed never removes it. | +| `hand_drawing` | Drawing uploaded by hand - check it and send it on | Check the drawing and send the draft on by hand. | +| `hand_bom` | BOM uploaded by hand - check it and send it on | Check the BOM and send it on. "Use VISUAL's BOM again" puts the feed's BOM back. | + +With both hand flags the text is "Drawing and BOM uploaded by hand - check it and send it on". Hand drafts keep their phase 2 text. + +"Closed in VISUAL" is not a flag. It is an observed state (`visual_closed_at`), shown on the packet. Filing a Data Book packet needs it, as the `vclosed_if_databook` precondition says. + +### Hand uploads + +On a flagged draft made by the feed (or one already uploaded by hand), Clerical can: + +- **Upload a drawing** (PDF) with a reason and a signature. It replaces the drawing pages, sets `drawing_state` `hand`, clears the drawing flags and adds `hand_drawing`. +- **Upload a BOM** (XLSX or CSV) with a reason and a signature. The server parses it: the header row is found in the first 20 rows by the old build's column patterns (item, qty, part, description, length, material, drawing reference); qty and a description or part are needed; empty rows are skipped; items default to the row number. It replaces the BOM lines (source `hand`) and adds `hand_bom`. The feed then leaves the BOM alone. +- **Use VISUAL's BOM again**: rebuilds the BOM from the cache and removes `hand_bom`. + +An unflagged draft refuses a hand upload ("not flagged"). ## Several open orders -Open question 5 in the plan. Proposed: the open order whose top job is the packet's work order base, else the newest open order, with the source shown and a Clerical override. +A project can have several customer orders in VISUAL. One function, `feed/orders.choose`, picks the order for a project or a unit. The first rule that finds one wins: + +1. **Pin.** Clerical pinned an order (a customer order ID among the project's orders) on the project or unit, with a reason. Via `pin`. +2. **Base.** The order linked to the work order base itself. Via `base`. +3. **Unit top job.** The project order whose line's top job is the unit number (open orders first). Via `unit_top_job`. +4. **Project open.** The first open order of the project (status F, H or R), newest first. Via `project_open`. +5. **Project newest.** The first order of the project, open or not. Via `project_newest`. + +When nothing matches there is no order. + +### Provenance + +The chosen order is stored as `order_ref` on the project and the unit: order ID, `via`, status, customer name, customer PO, order, desired ship and promise dates, and when it was chosen. The fields derived from it: + +| Field | From the order | +|---|---| +| customer | customer name | +| po | customer PO reference | +| order date | order date | +| ship date | promise date, else desired ship date | + +For each field the feed always writes `_visual`. It writes the field itself, with `_src = 'visual'`, only when `_src` is empty or already `visual`. A value Clerical typed (`_src = 'hand'`) is never overwritten. Each change is audited as `project.updated` with `src: visual`. + +### The pin + +Clerical can pin an order on a project or a unit (`order_pin`, with a reason, who and when) from the project page, which lists every cached order of the project. The feed re-derives on its next run. Clearing the pin goes back to the rules. + +## Settings + +The feed settings are one `settings` row, key `feed`, a JSON object. An Admin edits them on the Admin page, Feed tab (password session only). Unknown keys are ignored and missing keys take the default. Every save is validated, bumps the row version, is audited as `feed.settings_changed` with each changed key's before and after, and notifies the worker. + +| Setting | Default | Range | Meaning | +|---|---|---|---| +| `interval_shop_minutes` | 5 | 1 to 1440 | Feed interval inside shop hours | +| `interval_off_minutes` | 30 | 1 to 1440 | Feed interval outside shop hours | +| `shop_days` | Monday to Friday (`[0,1,2,3,4]`) | 0 (Monday) to 6 | Shop days, site time | +| `shop_start` | `06:00` | HH:MM | Start of shop hours, site time | +| `shop_end` | `18:00` | HH:MM | End of shop hours, site time | +| `window_back_days` | 14 | 0 to 120 | Window start, days before today | +| `window_ahead_days` | 30 | 0 to 365 | Window end, days after today | +| `golive_date` | none | a date | Work orders released before it are ignored | +| `departments` | all (`[]`) | shop codes | Departments to make packets for. Empty means every department the resource map routes to. | +| `drawing_recheck_hours` | 4 | 1 to 168 | Recheck each drawing after this long | +| `drawing_budget` | 60 | 1 to 500 | Drawing lookups per run | +| `drawing_interval_minutes` | 30 | not limited | The drawing job's own cadence | +| `labor_tickets` | on | on or off | Read labor tickets, at most every 30 minutes | +| `auto_release` | on | on or off | Release ready drafts to the warehouse | +| `auto_release_max` | 25 | not limited | Most drafts auto-released per run | +| `paused` | off | on or off | Stops scheduled feed and drawing runs and auto-release. Run now still works. | + +The Feed tab also shows the next scheduled run and the last ten runs. The "VISUAL resources" tab edits the resource map: each resource with its rows in the last batch, target (a shop department, Fold, Ignore or not assigned) and a confirmed tick. Every map change is audited. + +## Fixture sources + +For CI and local runs the worker can read files instead of VISUAL and M-Files: + +- `FEED_SOURCE=fixture` reads `FIXTURE_DIR/feed..json`, a file in the exact shape of VisualKit's `examples/packet_station_feed.py` output (`meta`, `rows`, `materials`, `orders_by_base`, `orders_by_project`) plus an optional `labor` list. The state is read from `FIXTURE_DIR/state` (default `1`). It returns the firmed rows released inside the window, plus every row whose key is tracked; tracked keys with no rows are missing. +- `DRAWING_SOURCE=fixture` reads `FIXTURE_DIR/drawings..json`, a map from drawing number to its rev, file name, drafting state and ids. A drawing not in the file is not found. A download copies `FIXTURE_DIR/`. +- `FIXTURE_DIR` defaults to `/fixtures`. +- The production stack refuses a fixture source at start. + +`tests/fixtures/generator.py` writes the files: `python -m tests.fixtures.generator --out DIR` makes `feed.1.json`, `feed.2.json`, `drawings.1.json`, `drawings.2.json`, the one-page PDFs they name, and `state` = `1`. It is deterministic. State 1 covers the packet rules (fold, ignore, an unmapped resource, a lot `7A`, two operations on one work order, no drawing, a non-numeric rev, the BOM rules, a work order before go-live, closed and cancelled legs, several open orders). State 2 changes a rev, a quantity and a target, removes one leg and moves another, so the change order, minor, gone and retire rules run. `packetstation fixtures use N --dir DIR` switches the state. The Playwright scenario writes the state file itself. See [DEPLOY.md](DEPLOY.md) for the CI step and [tests/fixtures/README.md](../tests/fixtures/README.md). + +Real job data never enters git or CI. Recordings from VISUAL DEV stay local, in `tests/fixtures/local/` and `recordings/`. + +## What the screens read + +Screens read PostgreSQL only, never VISUAL or M-Files per request: + +- Packets, flags, work order status, closed and cancelled state, ticket hours and the chosen order come from the packet tables and `packet_visual_state`. +- The From VISUAL panel reads `/api/feed/overview`: pause state, next run, the heartbeat, the last ten feed and drawing runs, the ignored work orders, unmapped resources and projects without names. +- Admin reads the cache tables (`/api/feed/batches`, `/api/feed/changes`) for "what VISUAL changed". +- The project page reads the cached orders from `visual_order_links` and `visual_orders`. + +Scheduled runs come from the worker's own clock, not from an open browser. diff --git a/docs/PLAN.md b/docs/PLAN.md index b43bc57..15f8a55 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -219,7 +219,7 @@ Need an answer before the named phase. Defaults apply if none arrives in time. 2. **PDFs on the server** (confirm before phase 7): reportlab + pypdf + pypdfium2. Default: yes. 3. **Frontend stack** (confirm before phase 1): TypeScript + React + Vite. Default: yes. Plain JavaScript would not be faster for parity: the old 7,000 lines are screen code that must be rewritten either way, and typed API models cut rework. 4. **PIN defaults** (confirm): 6 to 12 digits, five attempts, 10-hour shop session, device cookie. Genuinely open (Seth and Q.C., before phase 6): whether Q.C. accept and filing require the domain password even with a PIN set. Default: PIN is enough. IT: device certificate later. -5. **Several open orders on a project** (before phase 3): which order a packet shows. Proposed rule: the open order whose line's top job is the packet's work order base, else the newest open order, else the newest closed one; the packet shows "via base" or "via project" and Clerical can override. Needs Seth's view. +5. **Several open orders on a project** (before phase 3): which order a packet shows. Proposed rule: the open order whose line's top job is the packet's work order base, else the newest open order, else the newest closed one; the packet shows "via base" or "via project" and Clerical can override. Needs Seth's view. Phase 3 (2026-10-16) built the default in one function (`feed/orders.py`): a Clerical pin, else the base's own linked order, else the project order whose top job is the unit, else the project's first open order, else its newest; the source (`via`) is stored on the project and unit and shown on the project page, which lists every open order. Seth may change the rule later in that one place. 6. **VISUAL_ENCRYPT** (IT, before production): yes once vsql has a trusted certificate; `no` until then. 7. **Policy tightenings** listed under "Challenges" 1 to 3 (Seth, before phase 2). 8. **Repo location**: PacketStation2 as proposed, or another name. diff --git a/docs/POLICY.md b/docs/POLICY.md index 9683d14..8da3607 100644 --- a/docs/POLICY.md +++ b/docs/POLICY.md @@ -199,6 +199,49 @@ Every packet read returns: `can(principal, state, assume_it_reason=True) -> (list[Action], dict[Action, list[str]])`. For an admin the IT reason is assumed, so the list shows what the admin may do once they give one. `evaluate(principal, state, action)` returns the read-mode decision with `admin_bypass` set, so the screen knows to ask for the reason. System-only actions are never listed. CO_VERIFY needs a department in the request, so it is not listed; the change order list says which departments the person can verify. +## Worker writes (phase 3) + +The worker has no session. It writes through the same policy as a person, as the system principal `Principal(user_id=None, system="visual-feed")`. Each write locks the packet row (`FOR UPDATE SKIP LOCKED`; a row locked by someone else is left for the next run), builds the packet state, runs `policy.check()`, makes its change, bumps `version`, and writes a `packet_log` row (by nobody, by name "VISUAL feed", role `system`), an audit row (`packet.`, role `system`) and a change event. See [FEED.md](FEED.md) for when each write happens. + +| Action | From | What the worker does | +|---|---|---| +| vis_update_draft | draft | Updates a draft made by the feed: work orders, BOM (unless uploaded by hand), drawing and rev, dates, flags, drawing pages from M-Files. | +| vis_retire_draft | draft | Soft-removes an untouched draft whose work orders all moved to another packet, reason `moved`. | +| vis_open_co | any but draft and filed | Opens, re-aims or withdraws the packet's one pending feed change order, stages a drawing on it, and records drawing flags on a packet past draft. | +| vis_auto_release | draft to warehouse | Sends a ready draft to the warehouse. Same preconditions as send_to_warehouse (heat map answered, drawing attached). The feed only picks drafts with no flags and the rev confirmed by the drawing job. | + +What the feed may write: + +- Drafts it made, in place, through vis_update_draft and vis_retire_draft. +- New drafts, projects and units (`created_by` null, audit role `system`). +- The order-derived fields of projects and units, only where the value came from VISUAL. A value Clerical typed is never overwritten. +- On packets past draft: only a change order (vis_open_co), the observed VISUAL state (leg statuses, closed, cancelled, work order gone, ticket hours) and the drawing flags. +- Its own tables: the caches, `drawing_lookups`, the job tables. + +What the feed may never write: + +- Shop-side tables: stages, welds, heat tags, checklists, the clock, warehouse pulls and shorts, Q.C. records, seals. In-shop packets only get a change order. +- A removed packet. A key whose packets are all removed is never made again. +- A filed packet. +- Any person's action. The system principal passes only the vis_* actions, and people can never take them. +- VISUAL or M-Files. Both libraries are read-only. + +### Rev approval + +When M-Files has a different released rev from VISUAL, the drawing job stages the M-Files copy. A Clerical Supervisor (PS-Clerical-Supervisor) can approve it on a draft with rev_approve: a reason of at least 8 characters and a signature ("Approve the M-Files revision"). The pages are attached and the approval is kept on the packet (`visual.rev_ok`). With nothing staged the answer is 409 `nothing_staged`. + +### Hand uploads on flagged drafts + +On a draft made by the feed that is flagged (or already has a hand upload), Clerical can upload a drawing (edit_document_set) or a BOM (edit_bom) by hand. Both need a reason and a signature. A hand drawing sets `drawing_state` `hand` and flag `hand_drawing`; a hand BOM sets flag `hand_bom` and the feed stops replacing the BOM. An unflagged draft refuses with 409 `not_flagged`. Auto-release never takes a hand-flagged draft, so a person sends it on. "Use VISUAL's BOM again" removes the hand BOM. + +### Change orders from the feed + +- A pending feed change order holds nothing, unless Clerical puts it on hold (co_hold_toggle). Then it blocks the held steps until released or decided. +- Approval (co_approve, Clerical, signature) applies the staged change to the packet: work orders, BOM, drawing and rev. When the drawing changed, the new drawing must be staged first (`co_staged_if_dwg_change`, 409 `co_not_staged`). The old drawing pages are voided, not deleted. +- After approval a major change order asks the departments it touches to verify (co_verify), and holds the packet until they have. +- A minor change order (only target hours or the want date changed) needs Clerical approval only. No department is asked to verify, so it holds nothing after approval. +- Rejection (co_reject) needs a reason and a signature. The feed does not reopen a change order Clerical rejected for the same target. + ## Tightenings against the old build The old policy was looser than the old screens in places. The rebuild follows the screens where they were stricter, and the README where it states the intent. Each row is a change Seth can veto. @@ -217,3 +260,7 @@ The old policy was looser than the old screens in places. The rebuild follows th | Attaching drawing pages by hand needed no signature. | Attaching a drawing by hand needs a signature ("Attach a drawing by hand"). | It replaces what M-Files would supply, so a person stands behind it. | Yes | | Admin could write anything on a filed packet with an IT reason. | Admin cannot write a filed packet except through ADMIN_REOPEN (with a reason and a signature), and SET_DATABOOK_AT. | A filed packet is sealed. A change breaks the seal on purpose, on record. | No | | Admin passed every check after the change-order hold, so an admin could make a step from any status. | Admin goes through the same status check. The bypass covers who and custody only. Preconditions still apply. | Admin fixes data, not the workflow order. | Yes | +| A hand drawing or BOM upload on a flagged VISUAL draft took a reason of 3 characters and a PIN sign-off checked in the browser. | Both need a reason of at least 8 characters and a signature verified on the server, kept on the packet with who and when. | A hand upload stands in for VISUAL or M-Files, so a person stands behind it. | Yes | +| Approving a different M-Files rev was open to a Clerical lead. | rev_approve needs Clerical Supervisor, with a reason and a signature. | There is no Clerical lead AD group, and the approval overrides VISUAL. | Yes | +| The Sync on a Clerical PC skipped removed packets ("removed here, not re-made"), but only as the browser saw them. | A removed packet's key is never recreated, whatever VISUAL says. The match runs on the server across live and removed rows, in the feed's transaction. | A removal is a decision. The feed does not undo it. | No | +| The Sync saved through the same packet document that held the shop's work, so nothing but the code kept it off the shop's fields. | The feed never writes shop-side tables. Packets past draft only get a change order, observed VISUAL state and drawing flags. | The shop's record is the shop's. | No | diff --git a/docs/SCHEMA.md b/docs/SCHEMA.md index d523b1f..4ab6fc0 100644 --- a/docs/SCHEMA.md +++ b/docs/SCHEMA.md @@ -1,6 +1,6 @@ # Packet Station schema -Status: phases 1 and 2 built (identity, organisation, projects, packets, change orders, documents). Phase 3 adds the job and VISUAL tables, phase 9 the notification tables. The DDL is in `server/packetstation/migrations/`; this file explains it. See [PLAN.md](PLAN.md), "Schema". +Status: phases 1 to 3 built (identity, organisation, projects, packets, change orders, documents, jobs, VISUAL and drawing caches). Phase 9 adds the notification tables. The DDL is in `server/packetstation/migrations/`; this file explains it. See [PLAN.md](PLAN.md), "Schema". ## Conventions @@ -8,7 +8,7 @@ Identity bigints, except users, keyed on the AD objectGUID. Postgres enums for w ## Database roles -A superuser for bootstrap, the migrate role `ps_migrate` (owns the schema, runs migrations) and the app role `ps_app`. The plan's deployment section also names a worker role `ps_worker`. Phase 0 creates only the first three; whether the worker gets its own role is still to settle. +A superuser for bootstrap, the migrate role `ps_migrate` (owns the database and the schema, runs migrations) and the app role `ps_app`. The worker shares `ps_app` and sets `application_name` to `ps-worker`, so the two are told apart in `pg_stat_activity`; a separate `ps_worker` role was not needed (decided in phase 3). `ps_app` holds CONNECT and TEMPORARY on the database (the feed COPYs each batch into temp tables) and no CREATE on the schema. ## Identity (phase 1, migration 0002) @@ -77,7 +77,7 @@ The trigger `ps_packet_touch` sets `updated_at` on every UPDATE. It does not tou ### Children -- `packet_work_orders`: `wo_base`, `lot`, `split`, `sub` (all text; the last three default to `''` so the key has no nulls), `seq`, `qty`, `target_hrs`, `op`, `description`, `visual_status`, `missing_runs` and `gone_at` (the feed's "work order gone" check), `sort`, `row_version`. Unique on (`packet_id`, `wo_base`, `lot`, `split`, `sub`). +- `packet_work_orders`: `wo_base`, `lot`, `split`, `sub` (all text; the last three default to `''` so the key has no nulls), `seq`, `qty`, `target_hrs`, `op`, `description`, `visual_status`, `missing_runs` and `gone_at` (the feed's "work order gone" check), `sort`, `row_version`. Phase 3 adds `partspecs`, `rev` and `dwg`. Unique on (`packet_id`, `wo_base`, `lot`, `split`, `sub`, `seq`) with nulls not distinct, since migration 0004 (see "VISUAL and drawing caches"). - `packet_pages`: `n` (the packet page number, from `next_page_no`), `type`, `asset_id`, `pn` (page within the asset), `dwg`, `rev`, `co_id` (the change order that added it), `void_co_id`, `void_at`, `void_by`, `void_new_n` (a replaced page is voided, never deleted), `row_version`. Unique on (`packet_id`, `n`). - `bom_lines`: `item`, `qty`, `part_id` (text), `description`, `dwg_ref`, `cl_item`, `length`, `material`, `heat`, `na`, `issued`, `loc`, `pulled`, `pulled_at`, `pulled_by`, `mtr_asset_id`, `src`, `vkey`, `whs`, `comm`, `cons`, `sub`, `grp`, `sort`, `row_version`. Unique on (`packet_id`, `item`). - `bom_shorts`: `bom_line_id` (cascades with the line), `qty`, `qty_open`, `reason`, `ref`, `eta`, `note`, `opened_at`, `opened_by`, `closed_at`, `row_version`. At most one open short per line (unique on `bom_line_id` where `closed_at` is null). @@ -124,7 +124,15 @@ The app role is refused by privilege first. The migrate role owns the tables, so ## Audit, settings, jobs (phases 1 and 3) -audit_events with a hash chain, settings, job_requests, job_runs, job_schedule, worker_heartbeat. +- `audit_events` (migration 0001): append-only. `at`, `user_id`, `session_id`, `role_snapshot` (`system` for the worker), `action`, `subject_type`, `subject_id`, `packet_id`, `detail` jsonb, `it_reason`, `ip`. A hash chain: `row_hash` is sha256 over `prev_hash` and the canonical JSON of the row, and `prev_hash` is the row before it. BRIN index on `at`. +- `settings` (migration 0001): `key` (text PK), `value` jsonb, `version`, `updated_at`, `updated_by`. Phase 3 adds the row `feed`, the feed settings as one JSON object (see [FEED.md](FEED.md), "Settings"). A missing key takes its default. A save bumps `version` and sends `ps_notify('ps_settings', ...)` so the worker reschedules. + +Phase 3, migration 0004: + +- `job_requests`: one row per "Run now". `job`, `requested_at`, `requested_by`, `note`, `claimed_run_id` and `claimed_at` (set by the run that took it). Index on `job` where `claimed_at` is null. The app inserts the row and NOTIFYs `ps_job_request` in the same transaction. +- `job_runs`: one row per run. `job`, `trigger` (schedule, request, chain, cli), `requested_by` uuid[] (the requests claimed), `started_at`, `finished_at`, `status` (running, ok, degraded, failed, skipped, abandoned), `counts` jsonb, `errors` jsonb (a list of redacted strings), `build_version`, `visual_target` (`" / "`), `host`. Index on (`job`, `id DESC`). +- `job_schedule`: one row per job (`job` PK). `next_run_at`, `interval_seconds`, `paused`, `updated_at`. Written by the worker after each run for the Admin tab. Seeded for `visual-feed`, `drawing-lookup` and `housekeeping`. +- `worker_heartbeat`: a single row (`id` = 1 by CHECK). `at`, `build`, `host`, `pid`, `running` text[] (the jobs running now). Upserted every 30 seconds. ## Notifications (phase 9) @@ -132,7 +140,26 @@ Rules, recipients, exclusions, outbox, sent log, state. ## VISUAL and drawing caches (phase 3) -visual_batches, visual_rows, visual_materials, order links, labor, change rows, ignored work orders, resource map, drawing_lookups. +Migration 0004. Every ID part is text (`wo_base`, `lot`, `split`, `sub`, part IDs, order IDs), with `''` in place of null in a key. `seq` and `piece` are int. Closed sets are text with a CHECK. The worker fills these tables. Screens read them, never VISUAL. [FEED.md](FEED.md) says how each is written. + +- `visual_batches`: one row per feed run. `run_id`, `started_at`, `finished_at`, `server`, `database`, `site`, `since` and `until` (the window), `window_rows`, `tracked_keys`, `missing_keys`, `suspect`, `counts` jsonb. +- `visual_rows`: one row per work order operation (TYPE W), unique on (`wo_base`, `lot`, `split`, `sub`, `seq`). The contract fields: `res`, `dwg`, `rev`, `ps` (partspecs), `descr`, `qty`, `tgt`, `rel`, `want`, `st` (the status code F, R, U, C, X or `''`), `part`, `barcode`. `row_hash` (sha256 of the canonical row), `first_batch`, `last_batch` and `gone_batch` (set when a row in play was not read; cleared when it returns). Indexes on (`wo_base`, `lot`), on `dwg`, and on `rel` for rows not gone. +- `visual_materials`: one row per material line, unique on (`wo_base`, `lot`, `split`, `sub`, `seq`, `piece`). `part` (null for a line with no part), `qty`, `issued`, `whs`, `loc`, `st`, `istat`, `dwg`, `descr`, `comm`, `cons`, `mat_hash`, `first_batch`, `last_batch`, `gone_batch`. Index on (`wo_base`, `lot`). +- `visual_orders`: one row per customer order (`cust_order_id` PK). `status`, `customer_id`, `customer_name`, `customer_po_ref`, `order_date`, `desired_ship_date`, `promise_date`, `last_batch`. +- `visual_order_links`: `kind` (base or project), `key` (the base or the project number), `cust_order_id`, `rank` (open first, newest first), `lines` int[], `top_jobs` text[], `last_batch`. Unique on (`kind`, `key`, `cust_order_id`). Replaced for every key read. +- `visual_labor`: ticket hours per leg, unique on (`wo_base`, `lot`, `split`, `sub`, `seq`). `hours`, `tickets`, `last_ticket`, `observed_at`. No cost or pay column. +- `visual_row_changes`: what VISUAL changed, per batch. `batch_id`, the key and `seq`, `kind` (added, changed, gone, returned), `before` and `after` jsonb (the row as text values), `at`. Index on `batch_id`. `visual_material_changes` is the same plus `piece`. +- `visual_ignored_wos`: work orders kept out by the go-live date, unique on (`wo_base`, `lot`, `split`, `sub`). `rel`, `project`, `unit_number`, `res`, `dwg`, `first_seen_batch`, `last_seen_at`, and `pulled_in_at`, `pulled_in_by`, `pulled_in_reason` when Clerical pulls one in. +- `visual_resource_map`: `resource` (PK), `target` (`fold`, `ignore` or a two-digit department code, by CHECK), `confirmed`, `updated_at`, `updated_by`. Seeded from the old build's `visual_map.json`, every entry unconfirmed. "Electical" is spelled that way in VISUAL. +- `drawing_lookups`: one row per drawing number as VISUAL writes it (`dwg` PK, trimmed). The last answer: `checked_at`, `ok`, `rev` (released), `name`, `obj_id`, `obj_version`, `file_id`, `file_version`, `file_size`, `base` (H sheet), `draft_rev`, `draft_where`, `in_drafting`, `note`. Retry state: `fail_count`, `failing_since`, `next_check_at`. The fetched file: `asset_id`, `sha256`. `visual_rev_seen` is the VISUAL rev at the last lookup, so a rev change moves the drawing to the front of the queue. A rev mismatch keeps the staged M-Files copy here for approval. + +Column additions in 0004: + +- `packets`: `visual_cancelled_at` and `wo_gone_at`, so boards can filter on cancelled and gone without reading `flags`. +- `packet_work_orders`: `partspecs`, `rev` and `dwg`, the leg's own planner instructions, rev and drawing as VISUAL has them. The header keeps the highest rev. The unique key now includes `seq`: 0004 drops the 0003 unique on (`packet_id`, `wo_base`, `lot`, `split`, `sub`) and adds `UNIQUE NULLS NOT DISTINCT (packet_id, wo_base, lot, split, sub, seq)`, because the feed puts two operations of one work order on one packet. Hand drafts keep `seq` null, so they still have one row per key. +- `projects`: `order_date_visual`, `ship_date_visual`, `order_ref` jsonb (the chosen order and how it was chosen), and the Clerical pin `order_pin` (a customer order ID), `order_pin_by`, `order_pin_at`, `order_pin_reason`. +- `units`: `customer`, `po`, `customer_visual`, `po_visual`, `order_date_visual`, `ship_date_visual`, `order_ref`, and the same pin columns. +- `change_orders`: nothing new. `vis`, `waiting`, `staged`, `target_fp` and `minor` came in 0003. ## Versioning diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 0274657..ba9cb6e 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2,7 +2,7 @@ This document is rewritten for the server build. The old browser build's security document is in the old repo (`C:\Users\spencerm\Coding\PacketStation\docs\SECURITY.md`). The design comes from [BRIEF.md](BRIEF.md) and [PLAN.md](PLAN.md). -Status: phase 2. The identity controls (sessions, PIN, rate limits, audit, Origin check, derived group map) and the policy controls (server policy on every write, workflow steps as compare-and-set, server-side validation and limits, IT edit reasons, the filed lock, the upload pipeline for drawings and documents) are built and tested. Every other control names the phase that builds it. +Status: phase 3. The identity controls (sessions, PIN, rate limits, audit, Origin check, derived group map) and the policy controls (server policy on every write, workflow steps as compare-and-set, server-side validation and limits, IT edit reasons, the filed lock, the upload pipeline for drawings and documents) are built and tested. Phase 3 adds the worker controls (read-only VISUAL and M-Files reads, stack guard, advisory locks, redacted errors, audited system writes). No requirement ID closes in phase 3. Every other control names the phase that builds it. Contents: @@ -82,6 +82,16 @@ The M: drive build had 20 layers of protection, most of them enforced in the bro - The filed lock: a filed packet accepts only the data book date and the Admin reopen. Filing writes an HMAC-SHA256 seal with the server seal key; a reopen keeps the old seal with who broke it and why. - Details in [POLICY.md](POLICY.md). +### Worker (phase 3) + +- The worker publishes no port and joins no proxy network. It reaches out to VISUAL and M-Files only; nothing reaches in. Admin and Clerical reach it only through rows in PostgreSQL (`job_requests`, the `feed` settings row) and NOTIFY. +- It reads VISUAL through VisualKit as `visualkit_ro` and M-Files through MFilesKit as `mfileskitRO`. Both libraries are read-only in their own code. Nothing in Packet Station writes to either. +- Stack guard at start and again before every feed run: the test stack refuses a production VISUAL target, production refuses an implicit one, and production refuses a fixture source. +- One run per job at a time, by a PostgreSQL advisory lock per job name. Feed writes run in one transaction with a lock timeout and skip rows a person has locked. +- Job errors are redacted (connection strings, `password=`, `Authorization:` and the VISUAL and M-Files passwords themselves) and cut to 20 entries of 500 characters before they are stored or shown. +- Every feed write goes through the server policy as the system principal, which can take only the `vis_*` actions. Each one writes a packet log row and an audit row with role `system`. Each run writes a `feed.run` audit row with its counts, and each drawing fetch a `drawing.fetched` row with the sha256. Feed settings and resource map changes are audited with before and after. +- Downloaded drawings go through the same checks as uploads: PDF magic, size limit, streamed sha256, stored by id. + ### Audit (phase 1) - Append-only `audit_events`, with a hash chain (`prev_hash`, `row_hash`). diff --git a/frontend/src/api/feed.ts b/frontend/src/api/feed.ts new file mode 100644 index 0000000..a1daf2b --- /dev/null +++ b/frontend/src/api/feed.ts @@ -0,0 +1,15 @@ +// The VISUAL feed endpoints for Clerical (phase 3 contract, api/feed.py). +import { api } from "./client"; +import { asList, qs } from "./query"; +import type { FeedOverview, IgnoredWo } from "./types"; + +export type FeedJob = "visual-feed" | "drawing-lookup"; + +export const feedApi = { + overview: (signal?: AbortSignal) => api.get("/api/feed/overview", signal), + run: (job?: FeedJob) => api.post<{ requested: boolean; pending: number }>("/api/feed/run", job ? { job } : {}), + pullIn: (id: number, reason: string) => + api.post(`/api/feed/ignored/${encodeURIComponent(String(id))}/pull-in`, { reason }), + changes: async (opts: { batch_id?: number; limit?: number } = {}, signal?: AbortSignal): Promise => + asList(await api.get(`/api/feed/changes${qs({ batch_id: opts.batch_id, limit: opts.limit ?? 200 })}`, signal), "changes"), +}; diff --git a/frontend/src/api/jobs.ts b/frontend/src/api/jobs.ts new file mode 100644 index 0000000..78bdf01 --- /dev/null +++ b/frontend/src/api/jobs.ts @@ -0,0 +1,30 @@ +// The worker's jobs (phase 3 contract, api/jobs.py) and the Admin feed settings (api/feed_settings.py). +import { api } from "./client"; +import { asList, qs } from "./query"; +import type { FeedSettings, FeedSettingsAnswer, JobInfo, JobRun, JobsAnswer, ResourceMapRow, WorkerStatus } from "./types"; + +const enc = encodeURIComponent; + +/** GET /api/jobs may come as {jobs, heartbeat} or as a bare list. Both give the same answer. */ +function asJobs(raw: unknown): JobsAnswer { + const jobs = asList(raw, "jobs"); + const hb = raw && typeof raw === "object" && !Array.isArray(raw) ? ((raw as { heartbeat?: WorkerStatus | null }).heartbeat ?? null) : null; + return { jobs, heartbeat: hb }; +} + +export const jobsApi = { + list: async (signal?: AbortSignal): Promise => asJobs(await api.get("/api/jobs", signal)), + runs: async (job: string, limit = 10, signal?: AbortSignal): Promise => + asList(await api.get(`/api/jobs/${enc(job)}/runs${qs({ limit })}`, signal), "runs"), + run: (job: string) => api.post<{ requested: boolean; pending: number }>(`/api/jobs/${enc(job)}/run`, {}), +}; + +export const feedAdminApi = { + settings: (signal?: AbortSignal) => api.get("/api/admin/feed/settings", signal), + saveSettings: (patch: Partial) => api.patch("/api/admin/feed/settings", patch), + resourceMap: async (signal?: AbortSignal): Promise => + asList(await api.get("/api/admin/feed/resource-map", signal), "resources"), + mapResource: (resource: string, target: string) => + api.put(`/api/admin/feed/resource-map/${enc(resource)}`, { target }), + unmapResource: (resource: string) => api.del(`/api/admin/feed/resource-map/${enc(resource)}`), +}; diff --git a/frontend/src/api/packets.ts b/frontend/src/api/packets.ts index 3197264..614a2c4 100644 --- a/frontend/src/api/packets.ts +++ b/frontend/src/api/packets.ts @@ -43,4 +43,29 @@ export const packetsApi = { setVclosed: (id: number, closed: boolean, extra: Guarded = {}) => api.post(`${base(id)}/vclosed`, { closed, ...extra }), databook: (id: number) => api.post(`${base(id)}/databook`, {}), + + /* Phase 3: VISUAL packets. */ + revApprove: (id: number, body: { reason: string } & Guarded) => api.post(`${base(id)}/rev-approve`, body), + handDrawing: (id: number, body: { asset_id: string; reason: string } & Guarded) => + api.post(`${base(id)}/hand/drawing`, body), + /** Multipart cannot carry JSON, so the signature rides as sign_factor and sign_secret. */ + handBom: (id: number, file: File, reason: string, extra: Guarded = {}) => { + const fd = new FormData(); + fd.append("file", file, file.name); + fd.append("reason", reason); + if (extra.sign) { + fd.append("sign_factor", extra.sign.factor); + fd.append("sign_secret", extra.sign.secret); + } + if (extra.it_reason) fd.append("it_reason", extra.it_reason); + return request(`${base(id)}/hand/bom`, { method: "POST", body: fd }); + }, + removeHandBom: (id: number, extra: Guarded = {}) => api.delWithBody(`${base(id)}/hand/bom`, extra), + + /* Change orders. */ + coApprove: (id: number, no: number, extra: Guarded = {}) => api.post(`${base(id)}/change-orders/${no}/approve`, extra), + coReject: (id: number, no: number, body: { reason: string } & Guarded) => + api.post(`${base(id)}/change-orders/${no}/reject`, body), + coHold: (id: number, no: number, hold: boolean, extra: Guarded = {}) => + api.post(`${base(id)}/change-orders/${no}/hold`, { hold, ...extra }), }; diff --git a/frontend/src/api/projects.ts b/frontend/src/api/projects.ts index 1d0b0e2..c24cb48 100644 --- a/frontend/src/api/projects.ts +++ b/frontend/src/api/projects.ts @@ -12,6 +12,8 @@ export interface ProjectFields { due: string | null; status: Project["status"]; notes: string | null; + order_pin?: string | null; + order_pin_reason?: string | null; } export interface UnitFields { @@ -19,6 +21,8 @@ export interface UnitFields { due: string | null; status: Unit["status"]; notes: string | null; + order_pin?: string | null; + order_pin_reason?: string | null; } const pid = (id: number) => `/api/projects/${encodeURIComponent(String(id))}`; diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 30a5abf..a241ba1 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -72,6 +72,9 @@ export interface Unit { packets?: StageCounts; version: number; project?: ProjectRef; + order_ref?: OrderRef | null; + order_pin?: string | null; + order_pin_reason?: string | null; } /** One row of GET /api/projects. GET /api/projects/{id} adds units and aliases. */ @@ -91,6 +94,11 @@ export interface Project { packets?: StageCounts; aliases?: Array; version: number; + /** Phase 3: cached VISUAL orders for the project and its bases. */ + orders?: VisualOrder[]; + order_ref?: OrderRef | null; + order_pin?: string | null; + order_pin_reason?: string | null; } export interface ProjectCreate { @@ -161,6 +169,12 @@ export interface PacketCard { blocked: Record; /** Not in the contract's card shape. Shown when the server sends it. */ wo_count?: number; + /** Phase 3: the raw flag kinds (drawing_needed, rev_mismatch, ...). */ + flag_kinds?: string[]; + /** Phase 3: a feed change order waits for Clerical. */ + co_pending?: boolean; + /** Phase 3: every leg is closed in VISUAL (observed). */ + visual_closed?: boolean; } export interface PacketList { @@ -174,6 +188,8 @@ export interface PacketListQuery { project_id?: number; q?: string; flag?: string; + /** "pending": packets with a pending feed change order. */ + co?: string; limit?: number; } @@ -282,7 +298,9 @@ export interface PacketRead { source: Record; drawing_state: string; flags: string[]; - visual: Record; + visual: VisualInfo; + /** Phase 3: what the feed last observed in VISUAL. */ + visual_state?: VisualState | null; removed: { at: string; by: WhoRef | null; reason: string | null } | null; work_orders: WorkOrder[]; pages: PacketPage[]; @@ -295,7 +313,7 @@ export interface PacketRead { wh: Record | null; qc: Record | null; qc_history: unknown[]; - change_orders: unknown[]; + change_orders: ChangeOrder[]; open_cos: number[]; open_shorts: number; docs: PacketDoc[]; @@ -394,3 +412,245 @@ export interface Guarded { it_reason?: string; sign?: SignBody; } + +/* ---------- Phase 3: VISUAL feed on the packet ---------- */ + +/** visual.appr: the last M-Files answer for the drawing. */ +export interface VisualAppr { + ok?: boolean; + rev?: number | string | null; + vrev?: string | null; + name?: string | null; + note?: string | null; + base?: string | null; + draft_rev?: number | null; + draft_where?: string | null; + in_drafting?: boolean; + at?: string | null; +} + +/** visual.manual_dwg and visual.manual_bom. `by` may be a who reference or a name. */ +export interface HandUpload { + asset_id?: string; + name?: string | null; + reason?: string | null; + by?: WhoRef | string | null; + at?: string | null; + vis_rev?: string | null; + lines?: number; +} + +/** One folded operation (MFG, inspection and the like) kept on the packet for reference. */ +export interface FoldedOp { + key?: string | { base?: string; lot?: string; split?: string; sub?: string }; + wo?: string; + wo_base?: string; + lot?: string; + seq?: number; + op?: string; + tgt?: number | string; +} + +/** The packet's `visual` jsonb. Every key is optional; hand drafts have {}. */ +export interface VisualInfo { + appr?: VisualAppr; + /** Set by the read model from drawing_lookups when M-Files has a staged copy. */ + staged_rev?: { rev: number | string; name: string | null }; + partspecs?: string[]; + folded?: FoldedOp[]; + manual_dwg?: HandUpload; + manual_bom?: HandUpload; + rev_ok?: { rev: string | number; vis_rev?: string | null; by?: WhoRef | string | null; at?: string | null; reason?: string | null }; + rev_confirmed?: boolean; + [key: string]: unknown; +} + +export interface VisualLeg { + key?: string; + seq?: number; + st?: string; +} + +export interface VisualState { + legs: VisualLeg[]; + ticket_hours: number | string | null; + order_ref: OrderRef | null; + observed_at: string | null; + closed_at: string | null; + cancelled_at: string | null; +} + +/** change_orders[].vis for feed change orders. */ +export interface CoVis { + from_dwg?: string | null; + from_rev?: string | null; + to_dwg?: string | null; + to_rev?: string | null; + dwg_change?: boolean; + changes?: string[]; + [key: string]: unknown; +} + +export interface ChangeOrderItem { + old_page: number | null; + old_dwg: string | null; + old_rev: string | null; + new_page: number | null; + new_dwg: string | null; + new_rev: string | null; + added?: boolean; +} + +export interface ChangeOrder { + id: number; + co_no: number; + origin: string; + review: string; + minor: boolean; + hold: boolean; + ecn: string | null; + reason: string | null; + issued_at: string | null; + issued_by: WhoRef | null; + approved_at?: string | null; + approved_by?: WhoRef | null; + rejected_at?: string | null; + rejected_by?: WhoRef | null; + rejected_reason?: string | null; + items: ChangeOrderItem[]; + required: string[]; + verified: Record; + vis?: CoVis | null; + waiting?: { dwg: string; rev: string; since?: string | null; err?: string | null } | null; + staged?: { asset_id: string; name: string | null; rev: string; dwg: string; sha256?: string; pages?: number | null } | null; + target_fp?: string | null; + row_version?: number; +} + +/* ---------- Phase 3: orders ---------- */ + +/** One cached VISUAL customer order. */ +export interface VisualOrder { + cust_order_id: string; + status: string | null; + customer_id?: string | null; + customer_name: string | null; + customer_po_ref: string | null; + order_date: string | null; + desired_ship_date: string | null; + promise_date: string | null; + /** "project" or "base" link, when the server says. */ + kind?: string; + key?: string; + top_jobs?: string[]; +} + +/** The chosen order stored on projects and units. */ +export interface OrderRef { + cust_order_id: string; + via: string; + status: string | null; + customer_name: string | null; + customer_po_ref: string | null; + order_date: string | null; + desired_ship_date: string | null; + promise_date: string | null; + chosen_at?: string | null; +} + +/* ---------- Phase 3: jobs and the feed ---------- */ + +export interface JobRun { + id: number; + job: string; + trigger: string; + requested_by?: string[]; + started_at: string; + finished_at: string | null; + status: string; + counts: Record; + errors: string[]; + build_version?: string | null; + visual_target?: string | null; + host?: string | null; +} + +export interface WorkerStatus { + at: string | null; + build: string | null; + running: string[]; + stale: boolean; +} + +export interface IgnoredWo { + id: number; + wo_base: string; + lot: string; + split: string; + sub: string; + rel: string | null; + project: string | null; + unit_number: string | null; + res: string | null; + dwg: string | null; + pulled_in_at: string | null; +} + +export interface FeedOverview { + status: { paused: boolean; next_run_at: string | null; worker: WorkerStatus | null }; + last_run: JobRun | null; + runs: JobRun[]; + drawing_runs: JobRun[]; + ignored: IgnoredWo[]; + unmapped: Array<{ resource: string; rows: number }>; + projects_without_names: Array<{ id: number; number: string; units: number; packets: number }>; + golive_date: string | null; + departments: string[]; +} + +/** One row of GET /api/jobs. The contract leaves the shape open. */ +export interface JobInfo { + job: string; + next_run_at: string | null; + interval_seconds: number | null; + paused: boolean; + last_run: JobRun | null; +} + +export interface JobsAnswer { + jobs: JobInfo[]; + heartbeat: WorkerStatus | null; +} + +/** The feed settings row (feed.config.FeedSettings). */ +export interface FeedSettings { + interval_shop_minutes: number; + interval_off_minutes: number; + shop_days: number[]; + shop_start: string; + shop_end: string; + window_back_days: number; + window_ahead_days: number; + golive_date: string | null; + departments: string[]; + drawing_recheck_hours: number; + drawing_budget: number; + drawing_interval_minutes: number; + labor_tickets: boolean; + auto_release: boolean; + auto_release_max: number; + paused: boolean; +} + +export interface FeedSettingsAnswer extends FeedSettings { + next_run_at?: string | null; + runs?: JobRun[]; +} + +export interface ResourceMapRow { + resource: string; + target: string | null; + confirmed: boolean; + rows_last_run: number | null; + updated_at: string | null; +} diff --git a/frontend/src/auth/useEvents.ts b/frontend/src/auth/useEvents.ts index 778a80e..22ec9a3 100644 --- a/frontend/src/auth/useEvents.ts +++ b/frontend/src/auth/useEvents.ts @@ -49,6 +49,11 @@ export function useEvents(sessionId: string | null, handlers: EventHandlers): Pr es.addEventListener("changed", (ev) => { ref.current.onChanged?.(parse((ev as MessageEvent).data)); }); + // Phase 3: the worker finished a feed run. Fanned out with the changed events, marked event "feed". + es.addEventListener("feed", (ev) => { + const p = parse((ev as MessageEvent).data); + ref.current.onChanged?.({ ...(p && typeof p === "object" ? p : {}), event: "feed" }); + }); return () => { es.close(); diff --git a/frontend/src/components/Card.tsx b/frontend/src/components/Card.tsx index 1305e24..93ab490 100644 --- a/frontend/src/components/Card.tsx +++ b/frontend/src/components/Card.tsx @@ -40,6 +40,8 @@ export function Card({ p, extra }: { p: PacketCard; extra?: ReactNode }) { {cos ? CO {cos} : null} {p.flag ?
⚑ {p.flag}
: null} + {p.co_pending ?
Change from VISUAL to approve
: null} + {p.visual_closed && p.status !== "filed" ?
Closed in VISUAL
: null} {extra} ); diff --git a/frontend/src/components/FindBar.tsx b/frontend/src/components/FindBar.tsx index bff6c0b..33c4204 100644 --- a/frontend/src/components/FindBar.tsx +++ b/frontend/src/components/FindBar.tsx @@ -1,3 +1,4 @@ +import { FLAG_FILTERS } from "../lib/packets"; import { PickUnit, type UnitPick } from "./PickUnit"; export type BoardView = "board" | "list"; @@ -11,10 +12,13 @@ interface FindBarProps { onView: (v: BoardView) => void; /** "Showing n packets". */ count: number | null; + /** The flag kind filter. Shown when onFlag is given. */ + flag?: string; + onFlag?: (v: string) => void; } /** Find a job: the search box, the project then unit picker, and the Board | List toggle. */ -export function FindBar({ q, onQ, pick, onPick, view, onView, count }: FindBarProps) { +export function FindBar({ q, onQ, pick, onPick, view, onView, count, flag = "", onFlag }: FindBarProps) { return (
@@ -39,6 +43,18 @@ export function FindBar({ q, onQ, pick, onPick, view, onView, count }: FindBarPr
+ {onFlag ? ( + + ) : null}
) : null} + {review && (me?.roles ?? []).some((r) => r === "clerical" || r === "clerical_supervisor" || r === "admin") ? ( +
+ Change order {review.co_no}: a change from VISUAL is waiting for your approval - {coSummary(review)}. Nothing has gone to the shop packet yet. + {review.hold ? Shop packet on hold. : null}{" "} + +
+ ) : null} + + setMsg({ ok: false, text })} /> + {editing ? setEditing(false)} /> : null} {hasDraftControls ? ( @@ -522,6 +538,7 @@ export function PacketPage() { ) : null} {shownTab === "materials" ? : null} {shownTab === "workorders" ? : null} + {shownTab === "changes" ? : null} {shownTab === "docs" ? : null} {shownTab === "history" ? : null}
diff --git a/frontend/src/packet/visual.test.tsx b/frontend/src/packet/visual.test.tsx new file mode 100644 index 0000000..a286e2c --- /dev/null +++ b/frontend/src/packet/visual.test.tsx @@ -0,0 +1,227 @@ +import { fireEvent, screen, waitFor, within } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import type { ChangeOrder, PacketRead } from "../api/types"; +import { DEPARTMENTS, makeMe, renderApp } from "../test/auth"; +import { callsTo, mockFetch } from "../test/fetchMock"; +import { makePacket } from "../test/packets"; + +const CLARA = makeMe(); +const LEO = makeMe({ user: { display_name: "Leo Lead", sam_account: "clerk.leo" }, roles: ["clerical_supervisor"] }); + +function visualDraft(over: Partial = {}): PacketRead { + return makePacket({ + source_kind: "visual", + source: { kind: "visual", batch: 1 }, + drawing: "M00543", + rev: "2", + drawing_state: "needed", + flags: ["rev_mismatch"], + flag: "Rev mismatch: VISUAL calls rev 2, M-Files has rev 3", + visual: { + appr: { ok: true, rev: 3, vrev: "2", name: "M00543 r3.pdf", note: "Rev mismatch: VISUAL calls rev 2, M-Files has rev 3" }, + staged_rev: { rev: 3, name: "M00543 r3.pdf" }, + partspecs: ["Grind welds flush.", "Paint after test."], + folded: [{ wo: "L900001", seq: 30, op: "MFG" }], + }, + can: ["edit_header", "edit_document_set", "edit_bom", "remove"], + blocked: {}, + ...over, + }); +} + +function feedCo(over: Partial = {}): ChangeOrder { + return { + id: 51, + co_no: 2, + origin: "visual", + review: "pending", + minor: false, + hold: false, + ecn: "VISUAL", + reason: "VISUAL changed", + issued_at: "2026-10-16T08:00:00Z", + issued_by: null, + items: [], + required: [], + verified: {}, + vis: { from_dwg: "M00543", from_rev: "1", to_dwg: "M00543", to_rev: "2", dwg_change: true, changes: ["L900001 qty 2 -> 3"] }, + waiting: null, + staged: { asset_id: "as-2", name: "M00543 r2.pdf", rev: "2", dwg: "M00543", pages: 1 }, + target_fp: "fp", + ...over, + }; +} + +function inShop(co: ChangeOrder): PacketRead { + return makePacket({ + status: "in_progress", + source_kind: "visual", + source: { kind: "visual" }, + drawing_state: "attached", + flags: [], + change_orders: [co], + open_cos: [co.co_no], + can: [], + blocked: {}, + holder: "03 Weld Parts", + }); +} + +function base(packet: PacketRead, me = CLARA) { + return { + "/api/auth/me": { body: me }, + "/api/departments": { body: DEPARTMENTS }, + "/api/packets/1": { body: packet }, + "/api/packets/1/log": { body: [] }, + }; +} + +async function sign(secret = "pw-1") { + const sheet = await screen.findByRole("dialog", { name: "Sign" }); + fireEvent.change(within(sheet).getByLabelText("Password"), { target: { value: secret } }); + fireEvent.click(within(sheet).getByRole("button", { name: "Sign" })); + await waitFor(() => expect(screen.queryByRole("dialog", { name: "Sign" })).not.toBeInTheDocument()); +} + +describe("Packet page, VISUAL box", () => { + it("shows the flag, the planner instructions and folded operations; Clerical sees no rev approve", async () => { + mockFetch(base(visualDraft())); + renderApp("/packets/1"); + expect(await screen.findByText("Made from VISUAL - drawing not attached yet.")).toBeInTheDocument(); + expect(screen.getByText(/M-Files: Rev mismatch: VISUAL calls rev 2/)).toBeInTheDocument(); + expect(screen.getByText("A Clerical supervisor can approve rev 3 for this packet.")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Approve rev 3 for this packet" })).not.toBeInTheDocument(); + expect(screen.getByText("Planner instructions from VISUAL (2)")).toBeInTheDocument(); + expect(screen.getByText("Grind welds flush.")).toBeInTheDocument(); + expect(screen.getByText("Also on these work orders: L900001 seq 30 MFG")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Upload drawing" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Use VISUAL's BOM again" })).not.toBeInTheDocument(); + }); + + it("a Clerical supervisor approves rev N with a reason and a signature", async () => { + const fetch = mockFetch({ + ...base(visualDraft(), LEO), + "POST /api/packets/1/rev-approve": { body: visualDraft({ version: 4, flags: [], flag: null, drawing_state: "attached" }) }, + }); + renderApp("/packets/1"); + fireEvent.click(await screen.findByRole("button", { name: "Approve rev 3 for this packet" })); + const ask = await screen.findByRole("dialog", { name: "Approve rev 3 for M00543" }); + fireEvent.change(within(ask).getByLabelText("Reason"), { target: { value: "Build to rev 3 per Drafting" } }); + fireEvent.click(within(ask).getByRole("button", { name: "Approve and attach" })); + expect(await within(await screen.findByRole("dialog", { name: "Sign" })).findByText("Approve the M-Files revision")).toBeInTheDocument(); + await sign(); + expect(callsTo(fetch, "POST", "/api/packets/1/rev-approve")).toEqual([ + { reason: "Build to rev 3 per Drafting", sign: { factor: "password", secret: "pw-1" } }, + ]); + expect(await screen.findByText("Rev 3 approved and attached.")).toBeInTheDocument(); + }); + + it("uploads a hand BOM as multipart with the signature fields", async () => { + const fetch = mockFetch({ + ...base(visualDraft()), + "POST /api/packets/1/hand/bom": { body: visualDraft({ version: 4, visual: { manual_bom: { name: "bom.csv", reason: "No materials yet", lines: 3 } } }) }, + }); + renderApp("/packets/1"); + fireEvent.click(await screen.findByRole("button", { name: "Upload BOM" })); + const sheet = await screen.findByRole("dialog", { name: "Upload the BOM" }); + const file = new File(["Item,Qty,Description\n1,2,Plate\n"], "bom.csv", { type: "text/csv" }); + fireEvent.change(within(sheet).getByLabelText("File (XLSX or CSV, up to 25 MB)"), { target: { files: [file] } }); + fireEvent.change(within(sheet).getByLabelText("Why by hand (kept on the packet)"), { target: { value: "No materials yet" } }); + fireEvent.click(within(sheet).getByRole("button", { name: "Upload and sign" })); + await sign("pw-9"); + const call = fetch.mock.calls.find(([u, init]) => String(u) === "/api/packets/1/hand/bom" && init?.method === "POST"); + const fd = call?.[1]?.body as FormData; + expect(fd).toBeInstanceOf(FormData); + expect(fd.get("reason")).toBe("No materials yet"); + expect(fd.get("sign_factor")).toBe("password"); + expect(fd.get("sign_secret")).toBe("pw-9"); + expect((fd.get("file") as File).name).toBe("bom.csv"); + expect(await screen.findByRole("button", { name: "Use VISUAL's BOM again" })).toBeInTheDocument(); + expect(screen.getByText("BOM uploaded by hand:", { exact: false })).toBeInTheDocument(); + }); + + it("uploads a hand drawing: the asset first, then the attach with reason and sign", async () => { + const fetch = mockFetch({ + ...base(visualDraft()), + "POST /api/assets": { status: 201, body: { id: "as-9", mime: "application/pdf", size: 9, sha256: "00", pages: 1, original_name: "d.pdf", url: "/_blob/as-9" } }, + "POST /api/packets/1/hand/drawing": { body: visualDraft({ version: 4, flags: ["hand_drawing"] }) }, + }); + renderApp("/packets/1"); + fireEvent.click(await screen.findByRole("button", { name: "Upload drawing" })); + const sheet = await screen.findByRole("dialog", { name: "Upload the drawing" }); + fireEvent.change(within(sheet).getByLabelText("File (PDF, up to 25 MB)"), { target: { files: [new File(["%PDF-1.4"], "d.pdf", { type: "application/pdf" })] } }); + fireEvent.change(within(sheet).getByLabelText("Why by hand (kept on the packet)"), { target: { value: "Drafting sent it" } }); + fireEvent.click(within(sheet).getByRole("button", { name: "Upload and sign" })); + await sign(); + expect(callsTo(fetch, "POST", "/api/packets/1/hand/drawing")).toEqual([ + { asset_id: "as-9", reason: "Drafting sent it", sign: { factor: "password", secret: "pw-1" } }, + ]); + }); +}); + +describe("Packet page, Changes tab", () => { + it("Clerical approves a staged feed change order with a signature", async () => { + const co = feedCo(); + const fetch = mockFetch({ + ...base(inShop(co)), + "POST /api/packets/1/change-orders/2/approve": { body: inShop({ ...co, review: "approved", approved_by: { id: "u", display_name: "Clara Clerk" }, required: ["03"] }) }, + }); + renderApp("/packets/1"); + expect(await screen.findByText(/Change order 2: a change from VISUAL is waiting for your approval/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Review change" })); + const sec = await screen.findByRole("region", { name: "Change order 2" }); + expect(within(sec).getByText("M00543 Rev 1 -> Rev 2 + 1 other change")).toBeInTheDocument(); + expect(within(sec).getByText("L900001 qty 2 -> 3")).toBeInTheDocument(); + expect(within(sec).getByRole("link", { name: "Open the new drawing" })).toHaveAttribute("href", "/_blob/as-2"); + fireEvent.click(within(sec).getByRole("button", { name: "Approve - update the shop packet" })); + expect(await within(await screen.findByRole("dialog", { name: "Sign" })).findByText("Approve a change order")).toBeInTheDocument(); + await sign(); + expect(callsTo(fetch, "POST", "/api/packets/1/change-orders/2/approve")).toEqual([{ sign: { factor: "password", secret: "pw-1" } }]); + expect(await screen.findByText("Change order 2 approved. The shop packet is updated.")).toBeInTheDocument(); + expect(await screen.findByRole("table", { name: "Change order 2 verification" })).toBeInTheDocument(); + }); + + it("Approve is disabled with the reason while the drawing waits; reject and hold post", async () => { + const co = feedCo({ staged: null, waiting: { dwg: "M00543", rev: "2" } }); + const fetch = mockFetch({ + ...base(inShop(co)), + "POST /api/packets/1/change-orders/2/hold": { body: inShop({ ...co, hold: true }) }, + "POST /api/packets/1/change-orders/2/reject": { body: inShop({ ...co, hold: true, review: "rejected", rejected_reason: "Not for this unit" }) }, + }); + renderApp("/packets/1"); + fireEvent.click(await screen.findByRole("tab", { name: /Changes/ })); + const sec = await screen.findByRole("region", { name: "Change order 2" }); + expect(within(sec).getByRole("button", { name: "Approve - update the shop packet" })).toBeDisabled(); + expect(within(sec).getByText("Waits on that rev in M-Files.")).toHaveClass("why"); + expect(within(sec).getByText(/That rev isn't released in M-Files yet/)).toBeInTheDocument(); + + fireEvent.click(within(sec).getByRole("button", { name: "Hold the shop packet now" })); + expect(await within(sec).findByRole("button", { name: "Release the hold" })).toBeInTheDocument(); + expect(callsTo(fetch, "POST", "/api/packets/1/change-orders/2/hold")).toEqual([{ hold: true }]); + + fireEvent.click(within(sec).getByRole("button", { name: "Reject" })); + const ask = await screen.findByRole("dialog", { name: "Reject change order 2" }); + fireEvent.change(within(ask).getByLabelText("Reason"), { target: { value: "Not for this unit" } }); + fireEvent.click(within(ask).getByRole("button", { name: "Reject" })); + await sign(); + expect(callsTo(fetch, "POST", "/api/packets/1/change-orders/2/reject")).toEqual([ + { reason: "Not for this unit", sign: { factor: "password", secret: "pw-1" } }, + ]); + }); + + it("shows the co_not_staged answer as the callout text", async () => { + const co = feedCo(); + mockFetch({ + ...base(inShop(co)), + "POST /api/packets/1/change-orders/2/approve": { + status: 409, + body: { detail: "The new drawing has not been fetched from M-Files yet.", code: "co_not_staged" }, + }, + }); + renderApp("/packets/1"); + fireEvent.click(await screen.findByRole("button", { name: "Review change" })); + fireEvent.click(await screen.findByRole("button", { name: "Approve - update the shop packet" })); + await sign(); + expect(await screen.findByRole("alert")).toHaveTextContent("The new drawing has not been fetched from M-Files yet."); + }); +}); diff --git a/frontend/src/packet/visual.tsx b/frontend/src/packet/visual.tsx new file mode 100644 index 0000000..45040e6 --- /dev/null +++ b/frontend/src/packet/visual.tsx @@ -0,0 +1,562 @@ +// The VISUAL box and the Changes tab on the packet page (phase 3). +import { useState, type FormEvent } from "react"; +import { assetsApi, blobUrl } from "../api/assets"; +import { packetsApi } from "../api/packets"; +import type { ChangeOrder, FoldedOp, HandUpload, PacketRead, WhoRef } from "../api/types"; +import type { ReasonOptions } from "../components/ReasonSheet"; +import { Sheet } from "../components/Sheet"; +import { errorText, fmtTime } from "../lib/format"; +import { coSummary, feedOrigin } from "../lib/changeOrders"; +import { DRAWING_FLAGS } from "../lib/packets"; +import type { Mutate } from "./tabs"; + +/** Signature meanings, as the rules state them. */ +export const SIGN_REV_APPROVE = "Approve the M-Files revision"; +export const SIGN_HAND_DRAWING = "Attach a drawing by hand"; +export const SIGN_HAND_BOM = "Upload the BOM by hand"; +export const SIGN_CO_APPROVE = "Approve a change order"; +export const SIGN_CO_REJECT = "Reject a change order"; + +type AskReason = (opts: ReasonOptions) => Promise; + +const isCl = (roles: readonly string[]) => roles.some((r) => r === "clerical" || r === "clerical_supervisor" || r === "admin"); +const isClSup = (roles: readonly string[]) => roles.some((r) => r === "clerical_supervisor" || r === "admin"); + +function whoName(by: WhoRef | string | null | undefined): string { + if (!by) return ""; + return typeof by === "string" ? by : by.display_name; +} + +function foldedText(f: FoldedOp): string { + const k = f.key; + const wo = f.wo ?? f.wo_base ?? (k && typeof k === "object" ? [k.base, k.lot].filter(Boolean).join("/") : typeof k === "string" ? k : ""); + return [wo, f.seq !== undefined ? `seq ${f.seq}` : "", f.op ?? ""].filter(Boolean).join(" "); +} + +/* ---------- Hand upload sheet ---------- */ + +function HandSheet({ + p, + kind, + sub, + onCancel, + onSubmit, +}: { + p: PacketRead; + kind: "drawing" | "bom"; + sub: string; + onCancel: () => void; + onSubmit: (file: File, reason: string) => void; +}) { + const dwg = kind === "drawing"; + const [file, setFile] = useState(null); + const [reason, setReason] = useState(""); + const [error, setError] = useState(null); + + const submit = (e: FormEvent) => { + e.preventDefault(); + if (!file) { + setError("Choose the file."); + return; + } + if (reason.trim().length < 3) { + setError("Say why it is uploaded by hand."); + return; + } + if (dwg && !/\.pdf$/i.test(file.name) && file.type !== "application/pdf") { + setError("The drawing must be a PDF."); + return; + } + onSubmit(file, reason.trim()); + }; + + return ( + +
+ {dwg ? ( + <> + VISUAL calls{" "} + + {p.drawing || "-"} rev {p.rev || "-"} + + . Upload that drawing as a PDF. It replaces any drawing on this draft. + + ) : ( + <>Excel or CSV with a header row (Item, Qty, Part ID, Description, Length, Material, any of those names). It replaces the BOM on this draft. The feed will not overwrite it. + )} +
+
+ + + {error ? ( +
+ {error} +
+ ) : null} +
+ + +
+
+ + ); +} + +function HandLine({ what, m }: { what: string; m: HandUpload }) { + return ( +
+ {what} uploaded by hand: {m.name ?? ""} + {m.by ? ` - ${whoName(m.by)}` : ""} {m.at ? fmtTime(m.at) : ""} + {m.reason ? ` · ${m.reason}` : ""} + {typeof m.lines === "number" ? ` · ${m.lines} line${m.lines === 1 ? "" : "s"}` : ""} +
+ ); +} + +/* ---------- The VISUAL box ---------- */ + +export function VisualBox({ + p, + roles, + sub, + mutate, + busy, + askReason, + onError, +}: { + p: PacketRead; + roles: readonly string[]; + sub: string; + mutate: Mutate; + busy: boolean; + askReason: AskReason; + onError: (text: string) => void; +}) { + const [hand, setHand] = useState<"drawing" | "bom" | null>(null); + const [uploading, setUploading] = useState(false); + if (p.source_kind !== "visual") return null; + + const v = p.visual ?? {}; + const flags = p.flags ?? []; + const draft = p.status === "draft"; + const hasDwg = p.pages.some((x) => x.type === "drawing" && !x.void); + const drawingFlag = flags.some((f) => (DRAWING_FLAGS as readonly string[]).includes(f)); + const appr = v.appr; + const staged = v.staged_rev; + const mismatch = flags.includes("rev_mismatch"); + const md = v.manual_dwg; + const mb = v.manual_bom; + const canHand = draft && !p.removed && (isCl(roles) || p.can.includes("edit_document_set")) && (flags.length > 0 || !!md || !!mb); + const canApproveRev = draft && !p.removed && (p.can.includes("rev_approve") || isClSup(roles)); + const partspecs = v.partspecs ?? []; + const folded = v.folded ?? []; + const vs = p.visual_state; + + const approveRev = async () => { + if (!staged) return; + const reason = await askReason({ + title: `Approve rev ${staged.rev} for ${p.drawing ?? "this drawing"}`, + sub, + note: `VISUAL calls rev ${p.rev || "-"}. The drawing in M-Files is rev ${staged.rev}${staged.name ? `: ${staged.name}` : ""}. That drawing is attached to this packet. It is recorded with your name, signature and the reason. If VISUAL's rev changes, the packet is flagged again.`, + placeholder: "e.g. rev 2 not released yet - build to rev 1 per Drafting", + min: 3, + max: 400, + submitLabel: "Approve and attach", + }); + if (reason === null) return; + await mutate((g) => packetsApi.revApprove(p.id, { reason, ...g }), { sign: SIGN_REV_APPROVE, ok: `Rev ${staged.rev} approved and attached.` }); + }; + + const upload = async (kind: "drawing" | "bom", file: File, reason: string) => { + setHand(null); + if (kind === "bom") { + await mutate((g) => packetsApi.handBom(p.id, file, reason, g), { sign: SIGN_HAND_BOM, ok: "BOM uploaded. Check it, then send it on." }); + return; + } + setUploading(true); + let asset; + try { + asset = await assetsApi.upload(file, { packet_id: p.id, purpose: "drawing" }); + } catch (err) { + onError(errorText(err)); + return; + } finally { + setUploading(false); + } + if (asset.mime !== "application/pdf") { + onError("The drawing must be a PDF."); + return; + } + await mutate((g) => packetsApi.handDrawing(p.id, { asset_id: asset.id, reason, ...g }), { + sign: SIGN_HAND_DRAWING, + ok: "Drawing attached. Check it, then send it on.", + }); + }; + + const bomOff = () => mutate((g) => packetsApi.removeHandBom(p.id, g), { ok: "Hand BOM taken off. VISUAL's BOM is back." }); + + const working = busy || uploading; + + return ( +
+ {v.rev_ok && hasDwg ? ( +
+ Rev {String(v.rev_ok.rev)} approved by hand in place of VISUAL rev {v.rev_ok.vis_rev || "-"} + {v.rev_ok.by ? ` - ${whoName(v.rev_ok.by)}` : ""} {v.rev_ok.at ? fmtTime(v.rev_ok.at) : ""} + {v.rev_ok.reason ? `: ${v.rev_ok.reason}` : ""}. +
+ ) : null} + + {!hasDwg || drawingFlag ? ( +
+ {hasDwg ? "Made from VISUAL - drawing needs a look." : "Made from VISUAL - drawing not attached yet."} {p.drawing || "No drawing number"} rev {p.rev || "-"}. The drawing + is attached from M-Files automatically when its rev matches the rev VISUAL calls. Only Clerical can upload one by hand, and only while the packet is flagged. + {appr?.note ?
M-Files: {appr.note}{appr.at ? ` (looked ${fmtTime(appr.at)})` : ""}
: null} + {mismatch ? ( +
+ {staged ? ( + <> + In M-Files: {staged.name ?? `rev ${staged.rev}`}.{" "} + {canApproveRev ? ( + + ) : draft ? ( + A Clerical supervisor can approve rev {String(staged.rev)} for this packet. + ) : null} + + ) : ( + The M-Files drawing is being copied. Check back after the next drawing run. + )} +
+ ) : null} +
+ ) : null} + + {canHand ? ( +
+ Flagged - Clerical can upload the drawing or the BOM by hand. Only if it cannot come from M-Files or VISUAL in time. It is recorded with your name, + signature and reason. The packet stays in Flagged until you send it on. +
+ + + {mb ? ( + + ) : null} +
+ {md ? : null} + {mb ? : null} +
+ ) : null} + + {vs && (vs.closed_at || vs.cancelled_at || vs.ticket_hours) ? ( +
+ {vs.closed_at ? Closed in VISUAL (seen {fmtTime(vs.closed_at)}). : null} + {vs.cancelled_at ? Cancelled in VISUAL (seen {fmtTime(vs.cancelled_at)}). : null} + {vs.ticket_hours !== null && vs.ticket_hours !== undefined && Number(vs.ticket_hours) > 0 ? ( + Labor tickets in VISUAL: {Number(vs.ticket_hours).toFixed(1)} h. + ) : null} +
+ ) : null} + + {partspecs.length || folded.length ? ( +
+ + {partspecs.length ? `Planner instructions from VISUAL (${partspecs.length})` : `Also on these work orders (${folded.length})`} + + {partspecs.map((s, i) => ( + 
+              {s}
+
+ ))} + {folded.length ?
Also on these work orders: {folded.map(foldedText).join(", ")}
: null} +
+ ) : null} + + {hand ? setHand(null)} onSubmit={(f, r) => void upload(hand, f, r)} /> : null} +
+ ); +} + +/* ---------- Change orders ---------- */ + +function coPill(co: ChangeOrder) { + if (co.review === "rejected") return Rejected; + if (co.review === "pending") return Waiting on Clerical; + const open = co.required.filter((d) => !co.verified[d]); + if (open.length) return Open - {open.length} to verify; + return Verified; +} + +function ReviewPanel({ + p, + co, + sub, + mutate, + busy, + askReason, + canReview, +}: { + p: PacketRead; + co: ChangeOrder; + sub: string; + mutate: Mutate; + busy: boolean; + askReason: AskReason; + canReview: boolean; +}) { + const v = co.vis ?? {}; + const needsDrawing = !!v.dwg_change || !!co.waiting; + const ready = !needsDrawing || !!co.staged; + const why = ready ? "" : "Waits on that rev in M-Files."; + const old = p.pages.find((x) => x.type === "drawing" && !x.void); + + const approve = () => + mutate((g) => packetsApi.coApprove(p.id, co.co_no, g), { sign: SIGN_CO_APPROVE, ok: `Change order ${co.co_no} approved. The shop packet is updated.` }); + + const reject = async () => { + const reason = await askReason({ + title: `Reject change order ${co.co_no}`, + sub, + note: "The shop packet is not changed. If VISUAL changes again, a new change order comes in.", + min: 3, + max: 400, + danger: true, + submitLabel: "Reject", + }); + if (reason === null) return; + await mutate((g) => packetsApi.coReject(p.id, co.co_no, { reason, ...g }), { sign: SIGN_CO_REJECT, ok: `Change order ${co.co_no} rejected.` }); + }; + + const hold = () => + mutate((g) => packetsApi.coHold(p.id, co.co_no, !co.hold, g), { ok: co.hold ? "Hold released." : "Shop packet held." }); + + return ( +
+

Clerical review - nothing has gone to the shop yet

+ {needsDrawing ? ( + co.staged ? ( +
+ NOW {old?.dwg || v.from_dwg || "sheet"} Rev {old?.rev || v.from_rev || "?"} ->{" "} + NEW {co.staged.dwg} Rev {co.staged.rev} {co.staged.name ?? ""}{" "} + + Open the new drawing + +
+ ) : ( +
+ VISUAL calls {co.waiting?.dwg ?? v.to_dwg ?? ""} Rev {co.waiting?.rev ?? v.to_rev ?? "?"}. That rev isn't released in M-Files yet. It shows up here when it is. + {co.waiting?.err ?
Last try: {co.waiting.err}
: null} +
+ ) + ) : null} + {v.changes?.length ? ( +
+ Also changing on approval: +
    + {v.changes.map((x, i) => ( +
  • {x}
    • + ))} +
+ BOM lines keep the heats, pulls and issues already recorded on them. +
+ ) : null} + {co.minor ?
A minor change (target hours or want date). No department needs to verify it.
: null} + {canReview ? ( + <> +
+
+ + {why ?
{why}
: null} +
+
+ +
+
+ +
+
+
+ {co.hold + ? "Held. Sign-offs on this packet are stopped until you approve or reject." + : "The shop keeps working on the current drawing until you approve. Hold it if they should stop now."} +
+ + ) : ( +
A Clerical user approves or rejects this change.
+ )} +
+ ); +} + +/** The Changes tab: every change order, newest first. Clerical reviews the ones from VISUAL and M-Files here. */ +export function ChangesTab({ + p, + roles, + sub, + mutate, + busy, + askReason, +}: { + p: PacketRead; + roles: readonly string[]; + sub: string; + mutate: Mutate; + busy: boolean; + askReason: AskReason; +}) { + const list = [...(p.change_orders ?? [])].sort((a, b) => b.co_no - a.co_no); + if (!list.length) return
No change orders on this packet.
; + const canReview = isCl(roles) || p.can.includes("co_approve"); + return ( +
+ {list.map((co) => { + const feed = feedOrigin(co); + return ( +
+
+

+ Change order {co.co_no} + {co.ecn ? ` · ${co.ecn}` : ""} +

+ {coPill(co)} + + {feed ? (co.origin === "drawing" ? "from M-Files" : "from VISUAL") : `issued by ${co.issued_by?.display_name ?? "-"}`} {fmtTime(co.issued_at)} + {co.hold ? " · shop packet on hold" : ""} + +
+
+ {coSummary(co)} +
+ {co.review === "pending" && feed ? ( + canReview ? ( + + ) : ( +
+ Clerical is reviewing a change from VISUAL. Keep working to the current drawing. If Clerical approves it, the new drawing and changes come onto + this packet and you will be asked to verify. +
+ ) + ) : null} + {co.review === "rejected" ? ( +
+ {co.rejected_by ? ( + <> + Rejected by Clerical - {co.rejected_by.display_name} {fmtTime(co.rejected_at)}: {co.rejected_reason ?? ""}. The shop packet was not changed. + + ) : ( + <> + Withdrawn {fmtTime(co.rejected_at)}: {co.rejected_reason ?? "VISUAL went back to what the packet has"}. The shop packet was not changed. + + )} +
+ ) : null} + {co.review === "approved" && co.approved_by ? ( +
+ Approved by Clerical: {co.approved_by.display_name} {fmtTime(co.approved_at)} +
+ ) : null} + {co.review === "approved" && feed && co.vis?.changes?.length ? ( +
+ Also changed from VISUAL: {co.vis.changes.join("; ")} +
+ ) : null} +
+ Reason: {co.reason || "-"} +
+ {co.items.length ? ( +
+ + + + + + + + + {co.items.map((it, i) => ( + + + + + ))} + +
Old sheetNew sheet
+ {it.added ? ( + Added + ) : ( + <> + VOID {it.old_dwg || "sheet"} Rev {it.old_rev || "?"} + {it.old_page ? · page {it.old_page} : null} + + )} + + {it.added ? "ADDED" : "NEW"} {it.new_dwg || "sheet"} Rev {it.new_rev || "?"} + {it.new_page ? · page {it.new_page} : null} +
+
+ ) : null} + {co.review === "approved" && co.required.length ? ( +
+ + + + + + + + + + {co.required.map((d) => { + const ver = co.verified[d]; + return ( + + + + + + ); + })} + +
DepartmentVerifiedNote
{d}{ver ? `${ver.by?.display_name ?? "-"} ${fmtTime(ver.at)}` : Pending}{ver?.note ?? ""}
+
+ ) : null} +
+ ); + })} +
+ ); +} diff --git a/frontend/src/screens/admin/AdminPage.tsx b/frontend/src/screens/admin/AdminPage.tsx index 8771f99..9e6ada6 100644 --- a/frontend/src/screens/admin/AdminPage.tsx +++ b/frontend/src/screens/admin/AdminPage.tsx @@ -5,8 +5,9 @@ import type { AdLookup, AdRefreshResult, AdminUser, GroupRow, PinState } from ". import { useAuth } from "../../auth/useAuth"; import { errorText, fmtTime } from "../../lib/format"; import { useLoad, type Load } from "../../lib/useLoad"; +import { FeedTab, ResourcesTab } from "./FeedTabs"; -type Tab = "users" | "lookup" | "devices" | "groups" | "attempts"; +type Tab = "users" | "lookup" | "devices" | "groups" | "attempts" | "feed" | "resources"; const TABS: ReadonlyArray<{ id: Tab; label: string }> = [ { id: "users", label: "Users" }, @@ -14,6 +15,8 @@ const TABS: ReadonlyArray<{ id: Tab; label: string }> = [ { id: "devices", label: "Devices" }, { id: "groups", label: "Group map" }, { id: "attempts", label: "Sign-in attempts" }, + { id: "feed", label: "Feed" }, + { id: "resources", label: "VISUAL resources" }, ]; const ROLES_NOTE = "Roles come from AD groups. Ask IT to change them."; @@ -696,6 +699,8 @@ export function AdminPage() { {tab === "devices" ? : null} {tab === "groups" ? : null} {tab === "attempts" ? : null} + {tab === "feed" ? : null} + {tab === "resources" ? : null}
); diff --git a/frontend/src/screens/admin/FeedTabs.test.tsx b/frontend/src/screens/admin/FeedTabs.test.tsx new file mode 100644 index 0000000..de598ea --- /dev/null +++ b/frontend/src/screens/admin/FeedTabs.test.tsx @@ -0,0 +1,113 @@ +import { fireEvent, screen, waitFor, within } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { DEPARTMENTS, makeMe, renderApp } from "../../test/auth"; +import { callsTo, mockFetch } from "../../test/fetchMock"; + +const ADMIN = makeMe({ + user: { display_name: "Ann Admin", sam_account: "admin.ann" }, + roles: ["admin"], + screens: ["clerical", "warehouse", "supervisor", "stall", "qc", "pm", "admin"], +}); + +const SETTINGS = { + interval_shop_minutes: 5, + interval_off_minutes: 30, + shop_days: [0, 1, 2, 3, 4], + shop_start: "06:00", + shop_end: "18:00", + window_back_days: 14, + window_ahead_days: 30, + golive_date: null, + departments: [], + drawing_recheck_hours: 4, + drawing_budget: 60, + drawing_interval_minutes: 30, + labor_tickets: true, + auto_release: true, + auto_release_max: 25, + paused: false, + next_run_at: "2026-10-16T08:05:00Z", +}; + +const RUN = { + id: 7, + job: "visual-feed", + trigger: "schedule", + started_at: "2026-10-16T07:58:00Z", + finished_at: "2026-10-16T07:58:10Z", + status: "ok", + counts: { new_drafts: 1 }, + errors: [], +}; + +function base() { + return { + "/api/auth/me": { body: ADMIN }, + "/api/departments": { body: DEPARTMENTS }, + "/api/admin/users": { body: [] }, + "/api/admin/feed/settings": { body: SETTINGS }, + "/api/jobs/visual-feed/runs": { body: [RUN] }, + "/api/jobs": { body: { jobs: [], heartbeat: { at: "2026-10-16T07:59:00Z", build: "1.0", running: [], stale: false } } }, + "/api/admin/feed/resource-map": { + body: [ + { resource: "WELD-PART", target: "03", confirmed: true, rows_last_run: 40, updated_at: null }, + { resource: "NEWRES", target: null, confirmed: false, rows_last_run: 3, updated_at: null }, + { resource: "MFG", target: "fold", confirmed: false, rows_last_run: 10, updated_at: null }, + ], + }, + }; +} + +describe("Admin feed tabs", () => { + it("the Feed tab PATCHes only the changed fields", async () => { + const fetch = mockFetch({ ...base(), "PATCH /api/admin/feed/settings": { body: { ...SETTINGS, paused: true, interval_shop_minutes: 10, shop_days: [0, 1, 2, 3, 4, 5], departments: ["03"] } } }); + renderApp("/admin"); + fireEvent.click(await screen.findByRole("tab", { name: "Feed" })); + const form = await screen.findByRole("form", { name: "Feed settings" }); + expect(screen.getByRole("table", { name: "Last feed runs" })).toBeInTheDocument(); + + fireEvent.change(within(form).getByLabelText("Every (min) in shop hours"), { target: { value: "10" } }); + fireEvent.click(within(within(form).getByRole("group", { name: "Shop days" })).getByRole("button", { name: "Sat" })); + fireEvent.click(await within(within(form).getByRole("group", { name: "Departments" })).findByRole("button", { name: "03 Weld Parts" })); + fireEvent.click(within(form).getByLabelText(/^Paused/)); + fireEvent.click(within(form).getByRole("button", { name: "Save" })); + + expect(await screen.findByText("Saved. The worker picks up the new settings at once.")).toBeInTheDocument(); + expect(callsTo(fetch, "PATCH", "/api/admin/feed/settings")).toEqual([ + { interval_shop_minutes: 10, shop_days: [0, 1, 2, 3, 4, 5], departments: ["03"], paused: true }, + ]); + }); + + it("the Feed tab refuses a number out of range and sends nothing", async () => { + const fetch = mockFetch(base()); + renderApp("/admin"); + fireEvent.click(await screen.findByRole("tab", { name: "Feed" })); + const form = await screen.findByRole("form", { name: "Feed settings" }); + fireEvent.change(within(form).getByLabelText("Drawing lookups per run"), { target: { value: "900" } }); + fireEvent.click(within(form).getByRole("button", { name: "Save" })); + expect(await screen.findByText("Drawing lookups per run: a whole number from 1 to 500.")).toBeInTheDocument(); + expect(callsTo(fetch, "PATCH", "/api/admin/feed/settings")).toHaveLength(0); + }); + + it("the VISUAL resources tab saves a row with PUT and an unassign with DELETE", async () => { + const fetch = mockFetch({ + ...base(), + "PUT /api/admin/feed/resource-map/NEWRES": { body: { resource: "NEWRES", target: "14", confirmed: true, rows_last_run: 3, updated_at: null } }, + "DELETE /api/admin/feed/resource-map/MFG": { status: 204 }, + }); + renderApp("/admin"); + fireEvent.click(await screen.findByRole("tab", { name: "VISUAL resources" })); + const table = await screen.findByRole("table", { name: "VISUAL resources" }); + expect(screen.getByText("1 resource is not assigned. Their work orders are left out of the feed.")).toBeInTheDocument(); + const rows = within(table).getAllByRole("row"); + expect(rows.map((r) => within(r).queryAllByRole("cell")[0]?.textContent ?? "")).toEqual(["", "WELD-PART", "MFG", "NEWRES"]); + + fireEvent.change(await within(table).findByLabelText("NEWRES goes to"), { target: { value: "14" } }); + fireEvent.click(within(table).getByRole("button", { name: "Save NEWRES" })); + await waitFor(() => expect(callsTo(fetch, "PUT", "/api/admin/feed/resource-map/NEWRES")).toEqual([{ target: "14" }])); + + fireEvent.change(within(table).getByLabelText("MFG goes to"), { target: { value: "" } }); + fireEvent.click(await screen.findByRole("button", { name: "Save 1 change" })); + await waitFor(() => expect(callsTo(fetch, "DELETE", "/api/admin/feed/resource-map/MFG")).toHaveLength(1)); + }); +}); diff --git a/frontend/src/screens/admin/FeedTabs.tsx b/frontend/src/screens/admin/FeedTabs.tsx new file mode 100644 index 0000000..95e2272 --- /dev/null +++ b/frontend/src/screens/admin/FeedTabs.tsx @@ -0,0 +1,350 @@ +// Admin: the VISUAL feed settings and the resource map (phase 3). +import { useEffect, useMemo, useState, type FormEvent } from "react"; +import { feedAdminApi, jobsApi } from "../../api/jobs"; +import type { ResourceMapRow } from "../../api/types"; +import { DAYS, feedPatch, NUMS, toForm, type Form, type NumKey } from "../../lib/feedSettings"; +import { RunsTable } from "../../components/RunsTable"; +import { useChanged } from "../../lib/changes"; +import { errorText, fmtAgo, fmtIn, fmtTime } from "../../lib/format"; +import { useShopDepartments } from "../../lib/useDepartments"; +import { useLoad } from "../../lib/useLoad"; + +const toggle = (list: T[], v: T) => (list.includes(v) ? list.filter((x) => x !== v) : [...list, v]); + +function Msg({ msg }: { msg: { ok: boolean; text: string } | null }) { + if (!msg) return null; + return ( +
+ {msg.text} +
+ ); +} + +/* ---------- Feed ---------- */ + +export function FeedTab() { + const depts = useShopDepartments(); + const [load, reload] = useLoad((s) => feedAdminApi.settings(s)); + const [runs, reloadRuns] = useLoad((s) => jobsApi.runs("visual-feed", 10, s)); + const [jobs, reloadJobs] = useLoad((s) => jobsApi.list(s)); + const [form, setForm] = useState
(null); + const [busy, setBusy] = useState(false); + const [msg, setMsg] = useState<{ ok: boolean; text: string } | null>(null); + + const settings = load.state === "ok" ? load.data : null; + useEffect(() => { + if (settings) setForm(toForm(settings)); + }, [settings]); + useChanged( + () => { + reloadRuns(); + reloadJobs(); + }, + { delayMs: 1000, filter: (ev) => ev.event === "feed" }, + ); + + if (load.state === "loading") return
Loading
; + if (load.state === "error") return
{load.message}
; + if (!settings || !form) return null; + + const setNum = (k: NumKey, v: string) => setForm((f) => (f ? { ...f, nums: { ...f.nums, [k]: v } } : f)); + const set = (k: K, v: Form[K]) => setForm((f) => (f ? { ...f, [k]: v } : f)); + + const save = async (e: FormEvent) => { + e.preventDefault(); + const r = feedPatch(settings, form); + if ("error" in r) { + setMsg({ ok: false, text: r.error }); + return; + } + if (!Object.keys(r.patch).length) { + setMsg({ ok: true, text: "Nothing changed." }); + return; + } + setBusy(true); + setMsg(null); + try { + await feedAdminApi.saveSettings(r.patch); + setMsg({ ok: true, text: "Saved. The worker picks up the new settings at once." }); + reload(); + reloadJobs(); + } catch (err) { + setMsg({ ok: false, text: errorText(err) }); + } finally { + setBusy(false); + } + }; + + const runNow = async (job: string) => { + setBusy(true); + setMsg(null); + try { + await jobsApi.run(job); + setMsg({ ok: true, text: job === "visual-feed" ? "Feed run requested." : "Drawing lookup requested." }); + reloadJobs(); + } catch (err) { + setMsg({ ok: false, text: errorText(err) }); + } finally { + setBusy(false); + } + }; + + const hb = jobs.state === "ok" ? jobs.data.heartbeat : null; + const feedJob = jobs.state === "ok" ? jobs.data.jobs.find((j) => j.job === "visual-feed") : undefined; + const next = settings.next_run_at ?? feedJob?.next_run_at ?? null; + + return ( +
+
+ + {hb ? ( + hb.stale ? ( + Worker not running (last seen {fmtAgo(hb.at)}). + ) : ( + <>Worker seen {fmtAgo(hb.at)}{hb.build ? ` · build ${hb.build}` : ""}. + ) + ) : null} + {settings.paused ? Paused. : null} + Next feed run: {next ? `${fmtTime(next)} (${fmtIn(next)})` : "not scheduled"} + + + + +
+ + + + + +

Schedule

+
+ {DAYS.map(([d, l]) => ( + + ))} +
+
+ + + {NUMS.slice(0, 2).map(([k, l, lo, hi]) => ( + + ))} +
+ +

What is read

+
+ {NUMS.slice(2, 4).map(([k, l, lo, hi]) => ( + + ))} + +
+
Work orders released before the go-live date stay on paper. Clerical can pull one in.
+
+ Departments Only work orders for these become packets. None picked means every department the resource map routes to. +
+
+ {depts.map((d) => ( + + ))} +
+ + +

Drawings and release

+
+ {NUMS.slice(4).map(([k, l, lo, hi]) => ( + + ))} +
+ +
+ +
+ + +

Last ten feed runs

+ {runs.state === "ok" ? : runs.state === "error" ?
{runs.message}
: null} +
+ ); +} + +/* ---------- VISUAL resources ---------- */ + +export function ResourcesTab() { + const depts = useShopDepartments(); + const [load, reload] = useLoad((s) => feedAdminApi.resourceMap(s)); + const [edits, setEdits] = useState>({}); + const [busy, setBusy] = useState(false); + const [msg, setMsg] = useState<{ ok: boolean; text: string } | null>(null); + + const rows = useMemo( + () => (load.state === "ok" ? [...load.data].sort((a, b) => (b.rows_last_run ?? 0) - (a.rows_last_run ?? 0) || a.resource.localeCompare(b.resource)) : []), + [load], + ); + const byName = useMemo(() => new Map(rows.map((r) => [r.resource, r])), [rows]); + const pending = Object.keys(edits).filter((k) => (byName.get(k)?.target ?? "") !== edits[k]); + + const saveOne = async (r: ResourceMapRow, target: string) => { + if (target === "") { + if (r.target) await feedAdminApi.unmapResource(r.resource); + } else { + await feedAdminApi.mapResource(r.resource, target); + } + }; + + const save = async (names: string[]) => { + setBusy(true); + setMsg(null); + let done = 0; + try { + for (const k of names) { + const r = byName.get(k); + const t = edits[k]; + if (!r || t === undefined) continue; + await saveOne(r, t); + done++; + setEdits((e) => { + const n = { ...e }; + delete n[k]; + return n; + }); + } + setMsg({ ok: true, text: done === 1 ? "Saved. The feed uses it from its next run." : `${done} saved. The feed uses them from its next run.` }); + } catch (err) { + setMsg({ ok: false, text: errorText(err) }); + } finally { + setBusy(false); + reload(); + } + }; + + if (load.state === "loading") return
Loading
; + if (load.state === "error") return
{load.message}
; + const unassigned = rows.filter((r) => !r.target).length; + + return ( +
+
+ + Which department gets the packet for each VISUAL resource (RESOURCE_ID). Starting assignments are a best guess from the resource names. Check each one. + Lines are counted from the last feed run. + + + {pending.length ? ( + + ) : null} +
+ + {unassigned ? ( +
+ {unassigned === 1 ? "1 resource is" : `${unassigned} resources are`} not assigned. Their work orders are left out of the feed. +
+ ) : null} +
+ + + + + + + + + + + {rows.map((r) => { + const cur = edits[r.resource] ?? r.target ?? ""; + const dirty = cur !== (r.target ?? ""); + return ( + + + + + + + + ); + })} + {rows.length === 0 ? ( + + + + ) : null} + +
ResourceLines in last runGoes toConfirmed +
{r.resource}{r.rows_last_run ?? "-"} + + {dirty ? unsaved : null} + {r.confirmed ? "✓ confirmed" : r.target ? guess : not assigned} + {dirty ? ( + + ) : null} +
+ No resources yet. They appear after the first feed run. +
+
+ {rows.some((r) => r.updated_at) ? ( +
Last change {fmtTime(rows.map((r) => r.updated_at ?? "").sort().pop() ?? null)}.
+ ) : null} +
+ ); +} diff --git a/frontend/src/screens/clerical/ClericalHome.test.tsx b/frontend/src/screens/clerical/ClericalHome.test.tsx index 314b329..b493e4e 100644 --- a/frontend/src/screens/clerical/ClericalHome.test.tsx +++ b/frontend/src/screens/clerical/ClericalHome.test.tsx @@ -12,14 +12,62 @@ const CARDS = [ makeCard({ id: 5, status: "complete", routed_to: "databook", unit: { id: 13, number: "L999003" } }), makeCard({ id: 6, status: "complete", routed_to: "databook", vclosed: { at: "2026-10-15T10:00:00Z", by: null }, unit: { id: 14, number: "L999004" } }), makeCard({ id: 7, status: "filed", unit: { id: 15, number: "L999005" } }), + makeCard({ id: 8, status: "in_progress", co_pending: true, source_kind: "visual", unit: { id: 16, number: "L999006" } }), ]; +const OVERVIEW = { + status: { paused: false, next_run_at: "2026-10-16T08:03:00Z", worker: { at: "2026-10-16T07:59:50Z", build: "1.0", running: [], stale: false } }, + last_run: { + id: 42, + job: "visual-feed", + trigger: "schedule", + started_at: "2026-10-16T07:58:00Z", + finished_at: "2026-10-16T07:58:20Z", + status: "ok", + counts: { new_drafts: 4, drafts_updated: 2, cos_opened: 1 }, + errors: [], + }, + runs: [ + { + id: 42, + job: "visual-feed", + trigger: "schedule", + started_at: "2026-10-16T07:58:00Z", + finished_at: "2026-10-16T07:58:20Z", + status: "ok", + counts: { new_drafts: 4, drafts_updated: 2, cos_opened: 1 }, + errors: [], + }, + { + id: 41, + job: "visual-feed", + trigger: "request", + started_at: "2026-10-16T07:50:00Z", + finished_at: "2026-10-16T07:50:30Z", + status: "degraded", + counts: {}, + errors: ["orders read failed"], + }, + ], + drawing_runs: [ + { id: 90, job: "drawing-lookup", trigger: "chain", started_at: "2026-10-16T07:58:21Z", finished_at: null, status: "ok", counts: { attached: 3 }, errors: [] }, + ], + ignored: [ + { id: 5, wo_base: "L900099", lot: "1", split: "0", sub: "0", rel: "2026-09-01", project: "L900", unit_number: "L900001", res: "WELD-PART", dwg: "D-1", pulled_in_at: null }, + ], + unmapped: [{ resource: "NEWRES", rows: 3 }], + projects_without_names: [{ id: 70, number: "L900", units: 2, packets: 4 }], + golive_date: "2026-10-01", + departments: [], +}; + function base() { return { "/api/auth/me": { body: makeMe() }, "/api/departments": { body: DEPARTMENTS }, "/api/packets": { body: { packets: CARDS } }, "/api/projects": { body: [makeProject()] }, + "/api/feed/overview": { body: OVERVIEW }, }; } @@ -36,11 +84,13 @@ describe("ClericalHome", () => { expect(screen.getByLabelText("Flagged: 1")).toHaveClass("kpi", "hot"); expect(screen.getByLabelText("With warehouse: 1")).toBeInTheDocument(); - expect(screen.getByLabelText("In the shop: 1")).toBeInTheDocument(); + expect(screen.getByLabelText("In the shop: 2")).toBeInTheDocument(); expect(screen.getByLabelText("Ready to close: 1")).toHaveClass("hot"); - expect(screen.getByLabelText("Changes to approve: 0")).toHaveClass("zero"); + expect(screen.getByLabelText("Changes to approve: 1")).toHaveClass("hot"); - expect(within(column("Changes to approve")).getByText("Arrives in phase 3 with the VISUAL feed.")).toBeInTheDocument(); + const cos = column("Changes to approve"); + expect(within(cos).getByText("L999006 · WELD-PART")).toBeInTheDocument(); + expect(within(cos).getByText("Change from VISUAL to approve")).toBeInTheDocument(); const flagged = column("Flagged"); expect(within(flagged).getByText("L999001 · WELD-PART")).toBeInTheDocument(); expect(within(flagged).getByText(/Made by hand/)).toBeInTheDocument(); @@ -53,8 +103,8 @@ describe("ClericalHome", () => { // The list view holds the same packets. fireEvent.click(screen.getByRole("button", { name: "List" })); - const table = await screen.findByRole("table"); - expect(within(table).getAllByRole("row")).toHaveLength(1 + 5); + const table = await screen.findByRole("table", { name: "Jobs" }); + expect(within(table).getAllByRole("row")).toHaveLength(1 + 6); }); it("the picker loads the units after a project is chosen", async () => { @@ -105,4 +155,74 @@ describe("ClericalHome", () => { expect(await screen.findByRole("heading", { name: "L998 · New one" })).toBeInTheDocument(); expect(callsTo(fetch, "POST", "/api/projects/9/units")).toEqual([{ lots: ["001", "002", "003"], name: "", due: null, notes: "" }]); }); + + it("the From VISUAL panel shows the status and the runs, and Run now posts a request", async () => { + const fetch = mockFetch({ ...base(), "POST /api/feed/run": { body: { requested: true, pending: 1 } } }); + renderApp("/clerical"); + const status = await screen.findByTestId("feed-status"); + expect(status).toHaveTextContent(/^On · next run .* · last run .*: 4 new, 2 updated, 1 change order$/); + const runs = screen.getByRole("table", { name: "Last feed runs" }); + const rows = within(runs).getAllByRole("row"); + expect(rows).toHaveLength(3); + expect(within(rows[1] as HTMLElement).getByText("OK")).toHaveClass("pill", "st-ok"); + // The chained drawing run after run 42 attached 3 drawings. + expect(within(rows[1] as HTMLElement).getAllByRole("cell")[6]).toHaveTextContent("3"); + expect(within(rows[2] as HTMLElement).getByText("orders read failed")).toBeInTheDocument(); + expect(screen.getByText("NEWRES")).toBeInTheDocument(); + expect(screen.getByText(/IT assigns them on the Admin page/)).toBeInTheDocument(); + expect(screen.getByRole("link", { name: "L900" })).toHaveAttribute("href", "/clerical/projects/70"); + + fireEvent.click(screen.getByRole("button", { name: "Run now" })); + expect(await screen.findByText("Run requested. The boards update when it finishes.")).toBeInTheDocument(); + expect(callsTo(fetch, "POST", "/api/feed/run")).toEqual([{ job: "visual-feed" }]); + }); + + it("shows Worker not running and Paused from the overview", async () => { + mockFetch({ ...base(), "/api/feed/overview": { body: { ...OVERVIEW, status: { ...OVERVIEW.status, worker: { ...OVERVIEW.status.worker, stale: true } } } } }); + renderApp("/clerical"); + expect(await screen.findByTestId("feed-status")).toHaveTextContent(/^Worker not running/); + }); + + it("Pull in asks a reason and posts it", async () => { + const fetch = mockFetch({ ...base(), "POST /api/feed/ignored/5/pull-in": { body: { ...OVERVIEW.ignored[0], pulled_in_at: "2026-10-16T08:00:00Z" } } }); + renderApp("/clerical"); + fireEvent.click(await screen.findByRole("button", { name: "Pull in L900099/1/0/0" })); + const sheet = await screen.findByRole("dialog", { name: "Pull in work order" }); + fireEvent.change(within(sheet).getByLabelText("Reason"), { target: { value: "Customer wants it here" } }); + fireEvent.click(within(sheet).getByRole("button", { name: "Pull in" })); + expect(await screen.findByText("L900099/1/0/0 pulled in. It comes through on the next run.")).toBeInTheDocument(); + expect(callsTo(fetch, "POST", "/api/feed/ignored/5/pull-in")).toEqual([{ reason: "Customer wants it here" }]); + }); + + it("the flag kind select filters the list and the Flagged column shows the flag text", async () => { + const flagged = makeCard({ id: 9, source_kind: "visual", flag: "Rev mismatch: VISUAL calls rev 2, M-Files has rev 3", flag_kinds: ["rev_mismatch"] }); + const fetch = mockFetch({ + ...base(), + "/api/packets?limit=500&flag=rev_mismatch": { body: { packets: [flagged] } }, + }); + renderApp("/clerical"); + const bar = await screen.findByRole("region", { name: "Find jobs" }); + const select = within(bar).getByLabelText("Flags"); + expect(within(select).getAllByRole("option").map((o) => o.textContent)).toEqual([ + "All packets", + "All flags", + "Drawing needed", + "Rev mismatch", + "Not released", + "Not found", + "Lookup failed", + "Work order gone", + "Cancelled", + "Hand upload", + ]); + fireEvent.change(select, { target: { value: "rev_mismatch" } }); + await waitFor(() => expect(fetch.mock.calls.map(([u]) => String(u))).toContain("/api/packets?limit=500&flag=rev_mismatch")); + const col = await waitFor(() => { + const c = column("Flagged"); + expect(within(c).getByText(/Rev mismatch: VISUAL calls rev 2/)).toHaveClass("due", "over"); + return c; + }); + expect(within(col).queryByText("L999001 · WELD-PART", { exact: false })).toBeInTheDocument(); + expect(screen.getByLabelText("Flagged: 1")).toBeInTheDocument(); + }); }); diff --git a/frontend/src/screens/clerical/ClericalHome.tsx b/frontend/src/screens/clerical/ClericalHome.tsx index 958ff1b..7a73a7b 100644 --- a/frontend/src/screens/clerical/ClericalHome.tsx +++ b/frontend/src/screens/clerical/ClericalHome.tsx @@ -14,6 +14,7 @@ import { STATUSES } from "../../api/types"; import { dueInfo, flagClass, jobName, pillClass, statusLabel } from "../../lib/packets"; import { useLoad } from "../../lib/useLoad"; import { byTime, clericalBoards } from "./boards"; +import { FromVisualPanel } from "./FromVisualPanel"; import { ProjectsSection } from "./ProjectsSection"; import { NewPacketSheet } from "./sheets"; @@ -71,7 +72,7 @@ function JobList({ rows, sort, onSort }: { rows: PacketCard[]; sort: SortKey; on
- +
@@ -139,6 +140,7 @@ export function ClericalHome() { const [view, setView] = useState("board"); const [sort, setSort] = useState("updated"); const [drafting, setDrafting] = useState(false); + const [flag, setFlag] = useState(""); const projectId = pick.project?.id; const unitId = pick.unit?.id; @@ -148,8 +150,12 @@ export function ClericalHome() { }, [q]); const [load, reload] = useLoad( - (s) => packetsApi.list({ ...(query ? { q: query } : {}), ...(unitId ? { unit_id: unitId } : projectId ? { project_id: projectId } : {}) }, s), - `${query}|${projectId ?? ""}|${unitId ?? ""}`, + (s) => + packetsApi.list( + { ...(query ? { q: query } : {}), ...(flag ? { flag } : {}), ...(unitId ? { unit_id: unitId } : projectId ? { project_id: projectId } : {}) }, + s, + ), + `${query}|${flag}|${projectId ?? ""}|${unitId ?? ""}`, ); useChanged(reload, { delayMs: 500 }); @@ -187,7 +193,7 @@ export function ClericalHome() { - {b.flagged || b.toClose ? ( + {editable ? : null} + + {b.cos || b.flagged || b.toClose ? (
+ {b.cos ? ( +
+ {b.cos === 1 ? "1 change from VISUAL waits" : `${b.cos} changes from VISUAL wait`} for your approval. Nothing goes to the shop + packet until you approve it. +
+ ) : null} {b.flagged ? (
{b.flagged === 1 ? "1 draft is flagged." : `${b.flagged} drafts are flagged.`} Check {b.flagged === 1 ? "it" : "them"} and send {b.flagged === 1 ? "it" : "them"} on. @@ -210,7 +224,7 @@ export function ClericalHome() {
) : null} - + {load.state === "loading" ?
Loading
: null} {load.state === "error" ?
{load.message}
: null} diff --git a/frontend/src/screens/clerical/FromVisualPanel.tsx b/frontend/src/screens/clerical/FromVisualPanel.tsx new file mode 100644 index 0000000..fc7f784 --- /dev/null +++ b/frontend/src/screens/clerical/FromVisualPanel.tsx @@ -0,0 +1,207 @@ +import { useEffect, useState } from "react"; +import { Link } from "react-router"; +import { feedApi } from "../../api/feed"; +import type { IgnoredWo } from "../../api/types"; +import { useGuard } from "../../components/useGuard"; +import { RunsTable } from "../../components/RunsTable"; +import { useChanged } from "../../lib/changes"; +import { drawingsAfter, feedRunning, statusLine } from "../../lib/feed"; +import { errorText, fmtDay } from "../../lib/format"; +import { useLoad } from "../../lib/useLoad"; + +const IGNORED_SHOWN = 25; + +const woText = (w: Pick) => `${w.wo_base}/${w.lot}/${w.split}/${w.sub}`; + +/** From VISUAL on the Clerical home: the worker's status, Run now, the last runs and what needs a hand. */ +export function FromVisualPanel() { + const [load, reload] = useLoad((s) => feedApi.overview(s)); + const { askReason, sheet } = useGuard(); + const [busy, setBusy] = useState(false); + const [msg, setMsg] = useState<{ ok: boolean; text: string } | null>(null); + const [allIgnored, setAllIgnored] = useState(false); + const [, setTick] = useState(0); + + // A feed run or a packet change refreshes the panel. Bursts give one reload. + useChanged(reload, { delayMs: 1000 }); + // The "in 3 min" and "2 min ago" words age without a reload. + useEffect(() => { + const t = window.setInterval(() => setTick((n) => n + 1), 30_000); + return () => window.clearInterval(t); + }, []); + + const runNow = async () => { + setBusy(true); + setMsg(null); + try { + const r = await feedApi.run("visual-feed"); + setMsg({ ok: true, text: r.pending > 1 ? "Run requested. One is already waiting, so they run together." : "Run requested. The boards update when it finishes." }); + reload(); + } catch (err) { + setMsg({ ok: false, text: errorText(err) }); + } finally { + setBusy(false); + } + }; + + const pullIn = async (w: IgnoredWo) => { + const reason = await askReason({ + title: "Pull in work order", + sub: woText(w), + note: "It was released before the go-live date, so the feed left it out. Pulled in, it becomes a packet on the next run.", + min: 3, + max: 300, + submitLabel: "Pull in", + }); + if (reason === null) return; + setBusy(true); + setMsg(null); + try { + await feedApi.pullIn(w.id, reason); + setMsg({ ok: true, text: `${woText(w)} pulled in. It comes through on the next run.` }); + reload(); + } catch (err) { + setMsg({ ok: false, text: errorText(err) }); + } finally { + setBusy(false); + } + }; + + const head = (running: boolean) => ( +
+

From VISUAL

+ read by the server, nothing to upload + + +
+ ); + + if (load.state !== "ok") { + return ( +
+
+ {head(false)} + {load.state === "loading" ?
Loading
:
The feed status could not be read: {load.message}
} + {msg ? ( +
+ {msg.text} +
+ ) : null} +
+
+ ); + } + + const ov = load.data; + const running = feedRunning(ov); + const line = statusLine(ov); + const attached = drawingsAfter(ov.runs, ov.drawing_runs ?? []); + const ignored = ov.ignored ?? []; + const shownIgnored = allIgnored ? ignored : ignored.slice(0, IGNORED_SHOWN); + const unmapped = ov.unmapped ?? []; + const noName = ov.projects_without_names ?? []; + + return ( +
+
+ {head(running)} +
+ {line.text} +
+
+ Released work orders become packets ready to release under their unit, one per drawing, lot and operation. Drawings found in M-Files are attached + automatically. Anything that needs a hand goes to the Flagged column. + {ov.departments?.length ? ` Departments read: ${ov.departments.join(", ")}.` : ""} +
+ {msg ? ( +
+ {msg.text} +
+ ) : null} + + {noName.length ? ( +
+ {noName.length === 1 ? "1 project" : `${noName.length} projects`} came in from VISUAL without a name, customer or PO:{" "} + {noName.slice(0, 12).map((p, i) => ( + + {i ? " · " : ""} + + {p.number} + + + ))} + {noName.length > 12 ? ` and ${noName.length - 12} more` : ""}. +
+ ) : null} + + {unmapped.length ? ( +
+ + {unmapped.length === 1 ? "1 VISUAL resource isn't" : `${unmapped.length} VISUAL resources aren't`} assigned to a department yet + + , so their work orders were left out:{" "} + {unmapped.slice(0, 10).map((u, i) => ( + + {i ? ", " : ""} + {u.resource} ({u.rows}) + + ))} + {unmapped.length > 10 ? " and more" : ""}. IT assigns them on the Admin page under VISUAL resources. +
+ ) : null} + +
+ Last {ov.runs.length === 1 ? "run" : `${ov.runs.length} runs`} + +
+ + {ignored.length ? ( +
+ + {ignored.length === 1 ? "1 work order" : `${ignored.length} work orders`} released before go-live{ov.golive_date ? ` (${fmtDay(ov.golive_date)})` : ""}, left out + +
These stay on paper. Pull one in when it should get a packet here.
+
+
Job
+ + + + + + + + + + + {shownIgnored.map((w) => ( + + + + + + + + + ))} + +
Work orderUnitResourceDrawingReleased +
{woText(w)}{w.unit_number ?? w.project ?? "-"}{w.res ?? "-"}{w.dwg || "-"}{fmtDay(w.rel)} + +
+
+ {ignored.length > IGNORED_SHOWN && !allIgnored ? ( + + ) : null} + + ) : null} + + {sheet} +
+ ); +} diff --git a/frontend/src/screens/clerical/OrdersSection.tsx b/frontend/src/screens/clerical/OrdersSection.tsx new file mode 100644 index 0000000..3f55f50 --- /dev/null +++ b/frontend/src/screens/clerical/OrdersSection.tsx @@ -0,0 +1,130 @@ +import { useState } from "react"; +import { projectsApi } from "../../api/projects"; +import type { Project, VisualOrder } from "../../api/types"; +import { useGuard } from "../../components/useGuard"; +import { errorText, fmtDay } from "../../lib/format"; + +/** How the feed chose the order (orders.choose `via`). */ +const VIA_LABEL: Record = { + pin: "pinned by Clerical", + base: "linked to the work order", + unit_top_job: "names this unit", + project_open: "first open order", + project_newest: "newest order", +}; + +const ORDER_STATUS: Record = { F: "Firmed", R: "Released", H: "On hold", C: "Closed", X: "Cancelled", U: "Unreleased" }; + +/** The project's orders as VISUAL has them, which one the feed uses, and the pin. */ +export function OrdersSection({ project, editable, onSaved }: { project: Project; editable: boolean; onSaved: (text: string) => void }) { + const guard = useGuard(); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const orders: VisualOrder[] = project.orders ?? []; + const ref = project.order_ref ?? null; + const pin = project.order_pin ?? null; + if (!orders.length && !ref) return null; + + const setPin = async (next: string | null) => { + const reason = await guard.askReason({ + title: next ? `Pin order ${next}` : "Unpin the order", + sub: project.number, + note: next + ? "The feed uses this order for the customer, PO and dates from its next run. A value typed by hand is never overwritten." + : "The feed goes back to choosing the order itself on its next run.", + min: 3, + max: 300, + submitLabel: next ? "Pin" : "Unpin", + }); + if (reason === null) return; + setBusy(true); + setError(null); + try { + const r = await guard.run( + (g) => projectsApi.patch(project.id, { set: { order_pin: next, order_pin_reason: reason }, base: { order_pin: pin }, ...g }), + { sub: project.number }, + ); + if (r !== undefined) onSaved(next ? `Order ${next} pinned` : "Order unpinned"); + } catch (err) { + setError(errorText(err)); + } finally { + setBusy(false); + } + }; + + return ( +
+
+

Orders in VISUAL

+ + {ref ? ( + <> + the feed uses {ref.cust_order_id} ({VIA_LABEL[ref.via] ?? ref.via}) + + ) : ( + "no order chosen yet" + )} + + + {editable && pin ? ( + + ) : null} +
+ {project.order_pin_reason && pin ?
Pinned: {project.order_pin_reason}
: null} + {error ? ( +
+ {error} +
+ ) : null} + {orders.length ? ( +
+ + + + + + + + + + + + + {orders.map((o) => { + const used = ref?.cust_order_id === o.cust_order_id; + const pinned = pin === o.cust_order_id; + return ( + + + + + + + + + + ); + })} + +
OrderStatusCustomerCustomer POOrderedShip +
+ {o.cust_order_id} + {used ?
used by the feed
: null} + 		{o.status ? (ORDER_STATUS[o.status] ?? o.status) : "-"}{o.customer_name ?? "-"}{o.customer_po_ref ?? "-"}{fmtDay(o.order_date)}{fmtDay(o.promise_date ?? o.desired_ship_date)} + {pinned ? Pinned : null} + {editable && !pinned ? ( + + ) : null} +
+
+ ) : ( +
No orders cached for this project.
+ )} + {guard.sheet} +
+ ); +} diff --git a/frontend/src/screens/clerical/ProjectPage.test.tsx b/frontend/src/screens/clerical/ProjectPage.test.tsx new file mode 100644 index 0000000..f5caef7 --- /dev/null +++ b/frontend/src/screens/clerical/ProjectPage.test.tsx @@ -0,0 +1,64 @@ +import { fireEvent, screen, within } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import type { OrderRef, VisualOrder } from "../../api/types"; +import { DEPARTMENTS, makeMe, renderApp } from "../../test/auth"; +import { callsTo, mockFetch } from "../../test/fetchMock"; +import { makeProject, makeUnit } from "../../test/packets"; + +const ORDERS: VisualOrder[] = [ + { + cust_order_id: "CO-100", + status: "F", + customer_name: "Test Co", + customer_po_ref: "PO-1", + order_date: "2026-09-01", + desired_ship_date: "2026-12-01", + promise_date: null, + }, + { + cust_order_id: "CO-101", + status: "F", + customer_name: "Test Co", + customer_po_ref: "PO-2", + order_date: "2026-09-15", + desired_ship_date: null, + promise_date: "2026-12-20", + }, +]; + +const REF: OrderRef = { ...ORDERS[0]!, via: "project_open", chosen_at: "2026-10-16T08:00:00Z" }; + +describe("ProjectPage orders", () => { + it("lists the VISUAL orders, marks the one in use, and pins another with a reason", async () => { + const project = makeProject({ units: [makeUnit()], orders: ORDERS, order_ref: REF, order_pin: null }); + const fetch = mockFetch({ + "/api/auth/me": { body: makeMe() }, + "/api/departments": { body: DEPARTMENTS }, + "/api/projects/7": { body: project }, + "PATCH /api/projects/7": { body: { ...project, order_pin: "CO-101", version: 2 } }, + }); + renderApp("/clerical/projects/7"); + const table = await screen.findByRole("table", { name: "Orders in VISUAL" }); + expect(screen.getByText("first open order", { exact: false })).toBeInTheDocument(); + expect(within(table).getByText("used by the feed")).toBeInTheDocument(); + fireEvent.click(within(table).getByRole("button", { name: "Pin order CO-101" })); + const ask = await screen.findByRole("dialog", { name: "Pin order CO-101" }); + fireEvent.change(within(ask).getByLabelText("Reason"), { target: { value: "Second order covers this unit" } }); + fireEvent.click(within(ask).getByRole("button", { name: "Pin" })); + expect(await screen.findByText("Order CO-101 pinned")).toBeInTheDocument(); + expect(callsTo(fetch, "PATCH", "/api/projects/7")).toEqual([ + { set: { order_pin: "CO-101", order_pin_reason: "Second order covers this unit" }, base: { order_pin: null } }, + ]); + }); + + it("shows no orders section for a project without VISUAL orders", async () => { + mockFetch({ + "/api/auth/me": { body: makeMe() }, + "/api/departments": { body: DEPARTMENTS }, + "/api/projects/7": { body: makeProject({ units: [makeUnit()] }) }, + }); + renderApp("/clerical/projects/7"); + await screen.findByRole("heading", { name: "Units" }); + expect(screen.queryByRole("heading", { name: "Orders in VISUAL" })).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/src/screens/clerical/ProjectPage.tsx b/frontend/src/screens/clerical/ProjectPage.tsx index 8dd5cf6..894eb16 100644 --- a/frontend/src/screens/clerical/ProjectPage.tsx +++ b/frontend/src/screens/clerical/ProjectPage.tsx @@ -10,6 +10,7 @@ import { useChanged } from "../../lib/changes"; import { dueInfo, PROJECT_STATUS_LABEL, stageCounts } from "../../lib/packets"; import { useLoad } from "../../lib/useLoad"; import { ProgressBar, ProjectPill } from "./common"; +import { OrdersSection } from "./OrdersSection"; import { AddUnitsSheet, ProjectSheet, RenumberSheet } from "./sheets"; type UnitShow = "active" | "hold" | "closed" | "all"; @@ -241,6 +242,15 @@ export function ProjectPage() { {units.length ? :
No units yet.{editable ? " Add one to start adding packets." : ""}
} + { + setToast(text); + reload(); + }} + /> + {sheet === "edit" ? ( (a ?? "").localeCompare(b ?? ""); /** The boards of the Clerical home, from one list answer. */ -export function clericalBoards(all: PacketCard[]): { columns: ColumnDef[]; flagged: number; warehouse: number; shop: number; toClose: number } { +export function clericalBoards(all: PacketCard[]): { + columns: ColumnDef[]; + cos: number; + flagged: number; + warehouse: number; + shop: number; + toClose: number; +} { + const cos = all.filter((p) => p.co_pending).sort((a, b) => byTime(a.updated_at, b.updated_at)); const drafts = all.filter((p) => p.status === "draft").sort((a, b) => byTime(a.updated_at, b.updated_at)); const back = all.filter((p) => p.status === "complete" && p.routed_to === "databook"); const flagged = drafts.filter((p) => p.flag); @@ -18,7 +26,12 @@ export function clericalBoards(all: PacketCard[]): { columns: ColumnDef[]; flagg .sort((a, b) => byTime(b.status_at, a.status_at)) .slice(0, 25); const columns: ColumnDef[] = [ - { key: "cos", title: "Changes to approve", cards: [], empty: "Arrives in phase 3 with the VISUAL feed." }, + { + key: "cos", + title: "Changes to approve", + cards: cos, + empty: "Changes from VISUAL to packets already past draft wait here for your approval.", + }, { key: "flagged", title: "Flagged", cards: flagged, empty: "Nothing flagged." }, { key: "going", title: "Going to the warehouse", cards: going, empty: "Drafts with nothing flagged wait here." }, { @@ -32,6 +45,7 @@ export function clericalBoards(all: PacketCard[]): { columns: ColumnDef[]; flagg ]; return { columns, + cos: cos.length, flagged: flagged.length, warehouse: all.filter((p) => p.status === "warehouse").length, shop: all.filter((p) => IN_SHOP_OR_QC.includes(p.status)).length, diff --git a/pyproject.toml b/pyproject.toml index 1a7c3cd..8e763fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,8 @@ dependencies = [ "tzdata>=2026.5", "pypdf==6.21.0", "python-multipart==0.0.32", + "pypdfium2==5.14.0", + "openpyxl==3.1.5", "visualkit==0.3.0", "mfileskit==0.1.2", ] diff --git a/server/packetstation/api/feed.py b/server/packetstation/api/feed.py new file mode 100644 index 0000000..b9cbf6a --- /dev/null +++ b/server/packetstation/api/feed.py @@ -0,0 +1,308 @@ +"""The VISUAL feed for Clerical: the From VISUAL panel (status, runs, ignored work orders, unmapped resources, +projects without names), Run now, pulling in a work order released before go-live, and what VISUAL changed. + +Clerical (clerical or clerical_supervisor) or an admin. Pulling in a work order changes what the feed tracks, so an +admin outside Clerical gives an IT reason for it. Screens read PostgreSQL only: nothing here calls VISUAL. Run now +inserts a job request and NOTIFYs the worker, which does the reading. +""" + +from __future__ import annotations + +import json +from typing import Any, Literal + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel, Field + +from .. import audit +from ..domain import packets +from ..domain.pipeline import jsonable +from ..text import LIMITS, clean_opt, reason_ok +from .deps import Db, Principal, current_principal, get_db, require_password_session +from .errors import ApiError +from .packets import WRITE + +router = APIRouter(prefix="/api/feed", tags=["feed"]) + +CLERICAL = frozenset({"clerical", "clerical_supervisor"}) +JOBS = ("visual-feed", "drawing-lookup") +IGNORED_MAX = 200 +UNMAPPED_MAX = 30 +STALE_DEFAULT = 90 +PULL_IN_REASON_MIN = 3 +PULL_IN_REASON_MAX = 300 + + +class RunBody(BaseModel): + job: Literal["visual-feed", "drawing-lookup"] = "visual-feed" + note: str | None = Field(default=None, max_length=1000) + + +class PullInBody(BaseModel): + reason: str | None = Field(default=None, max_length=2000) + it_reason: str | None = Field(default=None, max_length=1000) + + +# ---------------------------------------------------------------- access + + +def require_clerical(p: Principal = Depends(current_principal)) -> Principal: # noqa: B008 + if CLERICAL & set(p.roles) or p.is_admin: + return p + raise ApiError(403, "forbidden", "Only Clerical can see the VISUAL feed.") + + +def _it_reason(p: Principal, it_reason: str | None) -> str | None: + """None for Clerical. An admin outside Clerical needs a password session and an IT reason.""" + if CLERICAL & set(p.roles): + return None + require_password_session(p) + why = reason_ok(it_reason) + if why is None: + raise ApiError(400, "it_reason_required", "Give an IT reason of at least 8 characters.") + return why + + +# ---------------------------------------------------------------- shapes + + +def _run_out(r: dict[str, Any]) -> dict[str, Any]: + return { + "id": r["id"], + "job": r["job"], + "trigger": r["trigger"], + "requested_by": [str(u) for u in r.get("requested_by") or []], + "started_at": r["started_at"], + "finished_at": r["finished_at"], + "status": r["status"], + "counts": r["counts"] or {}, + "errors": r["errors"] or [], + "build_version": r.get("build_version"), + "visual_target": r.get("visual_target"), + "host": r.get("host"), + } + + +def _runs(conn: Any, job: str, limit: int) -> list[dict[str, Any]]: + return [ + _run_out(r) + for r in packets.rows(conn, "SELECT * FROM job_runs WHERE job = %s ORDER BY id DESC LIMIT %s", (job, limit)) + ] + + +def _ignored_out(r: dict[str, Any]) -> dict[str, Any]: + keys = ("id", "wo_base", "lot", "split", "sub", "rel", "project", "unit_number", "res", "dwg", "last_seen_at") + out = {k: r[k] for k in keys} + out["pulled_in_at"] = r["pulled_in_at"] + out["pulled_in_by"] = str(r["pulled_in_by"]) if r["pulled_in_by"] else None + out["pulled_in_reason"] = r["pulled_in_reason"] + return out + + +def _stale_seconds() -> int: + try: + from ..worker import STALE_SECONDS + except ImportError: # the worker package is agent A's; the panel still answers without it + return STALE_DEFAULT + return int(STALE_SECONDS) + + +def _feed_settings(conn: Any) -> Any: + from ..feed import config + + return config.load(conn) + + +def _request_run(conn: Any, p: Principal, job: str, note: str | None) -> int: + """Insert a job request and NOTIFY the worker in the same transaction. Returns the pending request count.""" + conn.execute( + "INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)", + (job, p.user_id, clean_opt(note, LIMITS["job_note"])), + ) + conn.execute("SELECT ps_notify('ps_job_request', %s)", (json.dumps({"job": job}),)) + return int( + conn.execute("SELECT count(*) FROM job_requests WHERE job = %s AND claimed_at IS NULL", (job,)).fetchone()[0] + ) + + +# ---------------------------------------------------------------- endpoints + + +@router.get("/overview") +def overview( + p: Principal = Depends(require_clerical), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + """Everything the From VISUAL panel shows, from PostgreSQL only.""" + conn = d.conn + fs = _feed_settings(conn) + sched = packets.one(conn, "SELECT next_run_at, paused FROM job_schedule WHERE job = 'visual-feed'") + beat = packets.one( + conn, + "SELECT at, build, running, now() - at > make_interval(secs => %s) AS stale FROM worker_heartbeat WHERE id = 1", + (_stale_seconds(),), + ) + runs = _runs(conn, "visual-feed", 10) + last_run = runs[0] if runs else None + finished = next((r for r in runs if r["status"] in ("ok", "degraded")), None) + unmapped_raw = (finished or {}).get("counts", {}).get("unmapped") or {} + unmapped: list[dict[str, Any]] = [] + if isinstance(unmapped_raw, dict): + for k, v in unmapped_raw.items(): + if isinstance(v, int) or (isinstance(v, str) and v.isdigit()): + unmapped.append({"resource": str(k), "rows": int(v)}) + unmapped.sort(key=lambda x: (-int(x["rows"]), str(x["resource"]))) + unmapped = unmapped[:UNMAPPED_MAX] + ignored = packets.rows( + conn, + "SELECT * FROM visual_ignored_wos WHERE pulled_in_at IS NULL ORDER BY rel DESC NULLS LAST, wo_base, lot, id" + " LIMIT %s", + (IGNORED_MAX,), + ) + nameless = packets.rows( + conn, + "SELECT p.id, p.number," + " (SELECT count(*) FROM units u WHERE u.project_id = p.id AND u.removed_at IS NULL) AS units," + " (SELECT count(*) FROM packets k JOIN units u ON u.id = k.unit_id" + " WHERE u.project_id = p.id AND k.removed_at IS NULL) AS packets" + " FROM projects p WHERE p.source = 'visual' AND p.needs_details AND p.name IS NULL AND p.removed_at IS NULL" + " ORDER BY p.number LIMIT 200", + ) + return { + "status": { + "paused": bool(fs.paused), + "next_run_at": (sched or {}).get("next_run_at"), + "worker": None + if beat is None + else { + "at": beat["at"], + "build": beat["build"], + "running": list(beat["running"] or []), + "stale": bool(beat["stale"]), + }, + }, + "last_run": last_run, + "runs": runs, + "drawing_runs": _runs(conn, "drawing-lookup", 10), + "pending_requests": int( + conn.execute( + "SELECT count(*) FROM job_requests WHERE job = 'visual-feed' AND claimed_at IS NULL" + ).fetchone()[0] + ), + "ignored": [_ignored_out(r) for r in ignored], + "unmapped": unmapped, + "projects_without_names": [ + {"id": r["id"], "number": r["number"], "units": int(r["units"]), "packets": int(r["packets"])} + for r in nameless + ], + "golive_date": fs.golive_date, + "departments": list(fs.departments), + } + + +@router.post("/run", dependencies=WRITE) +def run_now( + body: RunBody | None = None, + p: Principal = Depends(require_clerical), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + """Run now: ask the worker for a feed (or drawing) run. Requests made while one is pending share that run.""" + body = body or RunBody() + conn = d.conn + with conn.transaction(): + pending = _request_run(conn, p, body.job, body.note) + audit.append( + conn, + action="feed.run_requested", + user_id=p.user_id, + session_id=p.session_id, + role_snapshot=p.role_snapshot, + subject_type="job", + subject_id=body.job, + detail={"job": body.job, "pending": pending}, + ip=p.ip, + ) + return {"requested": True, "job": body.job, "pending": pending} + + +@router.post("/ignored/{row_id}/pull-in", dependencies=WRITE) +def pull_in( + row_id: int, + body: PullInBody, + p: Principal = Depends(require_clerical), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + """Track a work order released before the go-live date after all. The next feed run makes its packet.""" + it_reason = _it_reason(p, body.it_reason) + reason = clean_opt(body.reason, PULL_IN_REASON_MAX) + if reason is None or len(reason) < PULL_IN_REASON_MIN: + raise ApiError(400, "reason_required", "Say why this work order is pulled in (at least 3 characters).") + conn = d.conn + with conn.transaction(): + row = packets.one(conn, "SELECT * FROM visual_ignored_wos WHERE id = %s FOR UPDATE", (row_id,)) + if row is None: + raise ApiError(404, "not_found", "No such work order on the ignored list.") + if row["pulled_in_at"] is not None: + raise ApiError( + 409, + "already_pulled_in", + "This work order was already pulled in.", + extra={"current": jsonable(_ignored_out(row))}, + ) + row = packets.one( + conn, + "UPDATE visual_ignored_wos SET pulled_in_at = now(), pulled_in_by = %s, pulled_in_reason = %s" + " WHERE id = %s RETURNING *", + (p.user_id, reason, row_id), + ) + assert row is not None + key = "/".join(str(row[k]) for k in ("wo_base", "lot", "split", "sub")) + audit.append( + conn, + action="feed.pulled_in", + user_id=p.user_id, + session_id=p.session_id, + role_snapshot=p.role_snapshot, + subject_type="work_order", + subject_id=key, + detail={"id": row_id, "key": key, "rel": str(row["rel"]) if row["rel"] else None, "reason": reason}, + it_reason=it_reason, + ip=p.ip, + ) + _request_run(conn, p, "visual-feed", f"pull in {key}") + return _ignored_out(row) + + +@router.get("/batches") +def batches( + limit: int = Query(20, ge=1, le=200), + p: Principal = Depends(require_clerical), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> list[dict[str, Any]]: + return packets.rows(d.conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT %s", (limit,)) + + +@router.get("/changes") +def changes( + batch_id: int | None = Query(None), + limit: int = Query(200, ge=1, le=2000), + p: Principal = Depends(require_clerical), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + """What VISUAL changed in one batch (the latest when none is given): work order rows and material lines.""" + conn = d.conn + if batch_id is None: + batch = packets.one(conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT 1") + else: + batch = packets.one(conn, "SELECT * FROM visual_batches WHERE id = %s", (batch_id,)) + if batch is None: + raise ApiError(404, "not_found", "No such feed batch.") + if batch is None: + return {"batch": None, "rows": [], "materials": []} + rows = packets.rows( + conn, "SELECT * FROM visual_row_changes WHERE batch_id = %s ORDER BY id LIMIT %s", (batch["id"], limit) + ) + mats = packets.rows( + conn, "SELECT * FROM visual_material_changes WHERE batch_id = %s ORDER BY id LIMIT %s", (batch["id"], limit) + ) + return {"batch": batch, "rows": rows, "materials": mats} diff --git a/server/packetstation/api/feed_settings.py b/server/packetstation/api/feed_settings.py new file mode 100644 index 0000000..496f041 --- /dev/null +++ b/server/packetstation/api/feed_settings.py @@ -0,0 +1,204 @@ +"""Admin: the feed settings and the VISUAL resource map. Every route needs the admin role and a password session. + +A settings save NOTIFYs `ps_settings` so the worker reschedules at once. Resource map changes apply on the next +feed run. Every change is audited with before and after. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Body, Depends, Response +from psycopg.rows import dict_row +from pydantic import BaseModel, Field + +from .. import audit +from ..domain.projects import Who +from ..feed import config +from .deps import Db, Principal, admin_principal, get_db +from .errors import ApiError +from .jobs import recent_runs +from .packets import WRITE + +router = APIRouter(prefix="/api/admin/feed", tags=["admin"]) + +FEED_JOB = "visual-feed" +SPECIAL_TARGETS = ("fold", "ignore") +RESOURCE_MAX = 100 + + +def _who(p: Principal) -> Who: + return Who(user_id=p.user_id, session_id=p.session_id, role_snapshot=p.role_snapshot, ip=p.ip) + + +def _shop_codes(conn: Any) -> set[str]: + rows = conn.execute("SELECT code FROM departments WHERE kind = 'shop' AND active").fetchall() + return {r[0] for r in rows} + + +def _settings_out(conn: Any) -> dict[str, Any]: + fs, version, updated_at, updated_by = config.load_versioned(conn) + nxt = conn.execute("SELECT next_run_at FROM job_schedule WHERE job = %s", (FEED_JOB,)).fetchone() + return { + **fs.model_dump(mode="json"), + "version": version, + "updated_at": updated_at, + "updated_by": str(updated_by) if updated_by else None, + "next_run_at": nxt[0] if nxt else None, + "runs": recent_runs(conn, FEED_JOB, 10), + } + + +@router.get("/settings") +def get_settings(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> dict[str, Any]: # noqa: B008 + return _settings_out(d.conn) + + +@router.patch("/settings", dependencies=WRITE) +def patch_settings( + patch: dict[str, Any] = Body(...), # noqa: B008 + p: Principal = Depends(admin_principal), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + conn = d.conn + if "departments" in patch and isinstance(patch["departments"], list): + shop = _shop_codes(conn) + bad = sorted({str(x).strip() for x in patch["departments"]} - shop) + if bad: + raise ApiError( + 422, + "bad_settings", + f"Not an active shop department: {', '.join(bad)}.", + extra={"errors": [f"departments: {b} is not an active shop department" for b in bad]}, + ) + try: + config.save(conn, who=_who(p), patch=patch) + except config.SettingsError as e: + raise ApiError(422, "bad_settings", str(e), extra={"errors": e.errors}) from None + return _settings_out(conn) + + +# ---------------------------------------------------------------- resource map + + +class MapBody(BaseModel): + target: str = Field(max_length=20) + confirmed: bool = True + + +def _resource(resource: str) -> str: + r = resource.strip() + if not r or len(r) > RESOURCE_MAX: + raise ApiError(400, "bad_resource", "Give the VISUAL resource name.") + return r + + +def _map_row(conn: Any, resource: str) -> dict[str, Any] | None: + with conn.cursor(row_factory=dict_row) as cur: + r = cur.execute( + "SELECT resource, target, confirmed, updated_at, updated_by FROM visual_resource_map WHERE resource = %s", + (resource,), + ).fetchone() + return dict(r) if r else None + + +def _row_out(r: dict[str, Any], rows_last_run: int | None) -> dict[str, Any]: + return { + "resource": r["resource"], + "target": r.get("target"), + "confirmed": bool(r.get("confirmed")), + "rows_last_run": rows_last_run, + "updated_at": r.get("updated_at"), + } + + +def _last_counts(conn: Any) -> dict[str, int]: + rows = conn.execute( + "SELECT res, count(*) FROM visual_rows" + " WHERE last_batch = (SELECT max(id) FROM visual_batches) AND res <> '' GROUP BY res" + ).fetchall() + return {r[0]: int(r[1]) for r in rows} + + +@router.get("/resource-map") +def resource_map(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> list[dict[str, Any]]: # noqa: B008 + conn = d.conn + counts = _last_counts(conn) + with conn.cursor(row_factory=dict_row) as cur: + mapped = cur.execute("SELECT resource, target, confirmed, updated_at FROM visual_resource_map").fetchall() + out = {r["resource"]: _row_out(r, counts.get(r["resource"])) for r in mapped} + for res, n in counts.items(): + if res not in out: + out[res] = {"resource": res, "target": None, "confirmed": False, "rows_last_run": n, "updated_at": None} + return sorted(out.values(), key=lambda r: (r["target"] is not None, r["resource"].lower())) + + +def _public(r: dict[str, Any] | None) -> dict[str, Any] | None: + if r is None: + return None + return {"target": r["target"], "confirmed": r["confirmed"]} + + +@router.put("/resource-map/{resource}", dependencies=WRITE) +def map_resource( + resource: str, + body: MapBody, + p: Principal = Depends(admin_principal), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + conn = d.conn + res = _resource(resource) + target = body.target.strip() + if target.lower() in SPECIAL_TARGETS: + target = target.lower() + elif target not in _shop_codes(conn): + raise ApiError(422, "bad_target", "The target is Fold, Ignore or an active shop department.") + with conn.transaction(): + before = _map_row(conn, res) + conn.execute( + "INSERT INTO visual_resource_map (resource, target, confirmed, updated_at, updated_by)" + " VALUES (%s, %s, %s, now(), %s) ON CONFLICT (resource) DO UPDATE SET target = EXCLUDED.target," + " confirmed = EXCLUDED.confirmed, updated_at = now(), updated_by = EXCLUDED.updated_by", + (res, target, body.confirmed, p.user_id), + ) + after = _map_row(conn, res) + audit.append( + conn, + action="feed.resource_mapped", + user_id=p.user_id, + session_id=p.session_id, + role_snapshot=p.role_snapshot, + subject_type="visual_resource", + subject_id=res, + detail={"resource": res, "before": _public(before), "after": _public(after)}, + ip=p.ip, + ) + assert after is not None + return _row_out(after, _last_counts(conn).get(res)) + + +@router.delete("/resource-map/{resource}", status_code=204, dependencies=WRITE) +def unmap_resource( + resource: str, + p: Principal = Depends(admin_principal), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> Response: + conn = d.conn + res = _resource(resource) + with conn.transaction(): + before = _map_row(conn, res) + if before is None: + raise ApiError(404, "not_found", "That resource is not mapped.") + conn.execute("DELETE FROM visual_resource_map WHERE resource = %s", (res,)) + audit.append( + conn, + action="feed.resource_unmapped", + user_id=p.user_id, + session_id=p.session_id, + role_snapshot=p.role_snapshot, + subject_type="visual_resource", + subject_id=res, + detail={"resource": res, "before": _public(before), "after": None}, + ip=p.ip, + ) + return Response(status_code=204) diff --git a/server/packetstation/api/jobs.py b/server/packetstation/api/jobs.py new file mode 100644 index 0000000..3a47e19 --- /dev/null +++ b/server/packetstation/api/jobs.py @@ -0,0 +1,170 @@ +"""The worker's jobs: schedule, heartbeat, recent runs, and "run now". + +Admin (password session) or Clerical (clerical or clerical_supervisor). "Run now" inserts a job_requests row and +NOTIFYs `ps_job_request` in the same transaction; the worker claims every pending request in one run. +`run_out`, `worker_status` and `request_run` are shared with api/feed.py. +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +from fastapi import APIRouter, Depends, Query +from psycopg.rows import dict_row +from pydantic import BaseModel, Field + +from .. import audit +from ..worker.heartbeat import STALE_SECONDS +from .deps import Db, Principal, current_principal, get_db, require_password_session +from .errors import ApiError +from .packets import WRITE + +router = APIRouter(prefix="/api/jobs", tags=["jobs"]) + +CLERICAL = frozenset({"clerical", "clerical_supervisor"}) +JOB_CHANNEL = "ps_job_request" +_JOB_NAME = re.compile(r"^[a-z0-9][a-z0-9-]{0,39}$") + + +def jobs_principal(p: Principal = Depends(current_principal)) -> Principal: # noqa: B008 + """Clerical in any session, or an admin with a password session.""" + if CLERICAL & set(p.roles): + return p + if p.is_admin: + return require_password_session(p) + raise ApiError(403, "forbidden", "Only Clerical or an admin can see and run the worker's jobs.") + + +class RunBody(BaseModel): + note: str | None = Field(None, max_length=500) + + +def run_out(r: dict[str, Any] | None) -> dict[str, Any] | None: + """One job_runs row as JSON.""" + if r is None: + return None + return { + "id": r["id"], + "job": r["job"], + "trigger": r["trigger"], + "requested_by": [str(u) for u in (r.get("requested_by") or [])], + "started_at": r["started_at"], + "finished_at": r["finished_at"], + "status": r["status"], + "counts": r.get("counts") or {}, + "errors": r.get("errors") or [], + "build_version": r.get("build_version"), + "visual_target": r.get("visual_target"), + "host": r.get("host"), + } + + +def recent_runs(conn: Any, job: str, limit: int = 10) -> list[dict[str, Any]]: + with conn.cursor(row_factory=dict_row) as cur: + rows = cur.execute("SELECT * FROM job_runs WHERE job = %s ORDER BY id DESC LIMIT %s", (job, limit)).fetchall() + return [o for o in (run_out(r) for r in rows) if o is not None] + + +def worker_status(conn: Any) -> dict[str, Any]: + """{at, build, host, running, stale}: stale when the worker has not beaten for STALE_SECONDS or never did.""" + with conn.cursor(row_factory=dict_row) as cur: + r = cur.execute( + "SELECT at, build, host, running, now() - at > make_interval(secs => %s) AS stale" + " FROM worker_heartbeat WHERE id = 1", + (STALE_SECONDS,), + ).fetchone() + if r is None: + return {"at": None, "build": None, "host": None, "running": [], "stale": True} + return { + "at": r["at"], + "build": r["build"], + "host": r["host"], + "running": list(r["running"] or []), + "stale": bool(r["stale"]), + } + + +def known_job(conn: Any, job: str) -> bool: + if not _JOB_NAME.match(job): + return False + return conn.execute("SELECT 1 FROM job_schedule WHERE job = %s", (job,)).fetchone() is not None + + +def request_run( + conn: Any, p: Principal, job: str, *, note: str | None = None, action: str = "job.run_requested" +) -> dict[str, Any]: + """Insert a request and NOTIFY the worker in one transaction. Returns {requested, pending}.""" + with conn.transaction(): + conn.execute( + "INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)", + (job, p.user_id, (note or "").strip() or None), + ) + conn.execute("SELECT ps_notify(%s, %s)", (JOB_CHANNEL, json.dumps({"job": job}, separators=(",", ":")))) + pending = conn.execute( + "SELECT count(*) FROM job_requests WHERE job = %s AND claimed_at IS NULL", (job,) + ).fetchone()[0] + audit.append( + conn, + action=action, + user_id=p.user_id, + session_id=p.session_id, + role_snapshot=p.role_snapshot, + subject_type="job", + subject_id=job, + detail={"job": job, "pending": int(pending)}, + ip=p.ip, + ) + return {"requested": True, "pending": int(pending)} + + +@router.get("") +def list_jobs(p: Principal = Depends(jobs_principal), d: Db = Depends(get_db)) -> dict[str, Any]: # noqa: B008 + conn = d.conn + with conn.cursor(row_factory=dict_row) as cur: + rows = cur.execute( + "SELECT s.job, s.next_run_at, s.interval_seconds, s.paused, s.updated_at," + " (SELECT count(*) FROM job_requests q WHERE q.job = s.job AND q.claimed_at IS NULL) AS pending," + " CASE WHEN r.id IS NULL THEN NULL ELSE to_jsonb(r) END AS last_run" + " FROM job_schedule s" + " LEFT JOIN LATERAL (SELECT * FROM job_runs j WHERE j.job = s.job ORDER BY j.id DESC LIMIT 1) r ON true" + " ORDER BY s.job" + ).fetchall() + jobs = [ + { + "job": r["job"], + "next_run_at": r["next_run_at"], + "interval_seconds": r["interval_seconds"], + "paused": r["paused"], + "updated_at": r["updated_at"], + "pending": int(r["pending"]), + "last_run": run_out(r["last_run"]), + } + for r in rows + ] + return {"jobs": jobs, "heartbeat": worker_status(conn)} + + +@router.get("/{job}/runs") +def job_runs( + job: str, + p: Principal = Depends(jobs_principal), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 + limit: int = Query(10, ge=1, le=100), +) -> list[dict[str, Any]]: + if not known_job(d.conn, job): + raise ApiError(404, "not_found", "No such job.") + return recent_runs(d.conn, job, limit) + + +@router.post("/{job}/run", dependencies=WRITE) +def run_now( + job: str, + body: RunBody | None = None, + p: Principal = Depends(jobs_principal), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + if not known_job(d.conn, job): + raise ApiError(404, "not_found", "No such job.") + return request_run(d.conn, p, job, note=body.note if body else None) diff --git a/server/packetstation/api/packet_children.py b/server/packetstation/api/packet_children.py index 5834eca..b1c04ff 100644 --- a/server/packetstation/api/packet_children.py +++ b/server/packetstation/api/packet_children.py @@ -1392,6 +1392,191 @@ def issue_co( return _read(d, pp, packet_id) +FEED_ORIGINS = ("visual", "drawing") +MSG_CO_NOT_STAGED = "The new drawing has not been fetched from M-Files yet." + + +def _vis_text(leg: Any, *names: str) -> str: + for n in names: + v = leg.get(n) if isinstance(leg, dict) else None + if v is not None and v != "": + return str(v) + return "" + + +def _leg_row(leg: Any) -> dict[str, Any]: + """One work order row from a change order's `vis.legs` entry. Accepts a flat leg ({wo|wo_base|base, lot, split, + sub, seq, ...}) or one with a nested key ({key: {base, lot, split, sub}, seq, ...}).""" + if not isinstance(leg, dict): + raise ApiError(422, "bad_change_order", "The change order's work order list cannot be read.") + key = leg.get("key") if isinstance(leg.get("key"), dict) else leg + base = _vis_text(key, "base", "wo_base", "wo") + if not base: + raise ApiError(422, "bad_change_order", "The change order's work order list cannot be read.") + seq_raw = leg.get("seq") + return { + "wo_base": base, + "lot": _vis_text(key, "lot"), + "split": _vis_text(key, "split"), + "sub": _vis_text(key, "sub"), + "seq": int(seq_raw) if seq_raw not in (None, "") else None, + "qty": _num(leg.get("qty")), + "target_hrs": _num(leg.get("tgt", leg.get("target_hrs"))), + "op": clean_opt(leg.get("op"), LIMITS["op"]), + "dwg": clean_opt(leg.get("dwg"), LIMITS["drawing"]), + "rev": clean_opt(leg.get("rev"), LIMITS["rev"]), + "partspecs": clean_opt(leg.get("partspecs"), 4000), + "visual_status": clean_opt(leg.get("st", leg.get("visual_status")), 10), + } + + +def _apply_legs(conn: Any, packet_id: int, legs: list[Any], title: str | None) -> dict[str, int]: + """Replace the packet's work orders with the change order's legs. Rows whose key and seq match are updated in + place (their ids and gone marks stay), new ones are added, the rest are deleted.""" + want = [_leg_row(lg) for lg in legs] + have = packets.rows(conn, "SELECT * FROM packet_work_orders WHERE packet_id = %s", (packet_id,)) + + def k(r: dict[str, Any]) -> tuple[Any, ...]: + return (r["wo_base"], r["lot"], r["split"], r["sub"], r["seq"]) + + by_key = {k(r): r for r in have} + seen: set[tuple[Any, ...]] = set() + counts = {"added": 0, "updated": 0, "removed": 0} + for sort, r in enumerate(want): + key = k(r) + if key in seen: + continue + seen.add(key) + cur = by_key.get(key) + if cur is not None: + conn.execute( + "UPDATE packet_work_orders SET qty = %s, target_hrs = %s, op = %s, dwg = %s, rev = %s, partspecs = %s," + " visual_status = %s, sort = %s, row_version = row_version + 1 WHERE id = %s", + ( + r["qty"], + r["target_hrs"], + r["op"], + r["dwg"], + r["rev"], + r["partspecs"], + r["visual_status"], + sort, + cur["id"], + ), + ) + counts["updated"] += 1 + else: + conn.execute( + "INSERT INTO packet_work_orders (packet_id, wo_base, lot, split, sub, seq, qty, target_hrs, op," + " description, visual_status, dwg, rev, partspecs, sort)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ( + packet_id, + r["wo_base"], + r["lot"], + r["split"], + r["sub"], + r["seq"], + r["qty"], + r["target_hrs"], + r["op"], + title, + r["visual_status"], + r["dwg"], + r["rev"], + r["partspecs"], + sort, + ), + ) + counts["added"] += 1 + gone = [r["id"] for r in have if k(r) not in seen] + if gone: + conn.execute("DELETE FROM packet_work_orders WHERE id = ANY(%s)", (gone,)) + counts["removed"] = len(gone) + return counts + + +def _plural(n: int, word: str) -> str: + return f"{n} {word}{'' if n == 1 else 's'}" + + +def _apply_feed_co(w: Write, co: dict[str, Any]) -> str | None: + """Apply a VISUAL or M-Files change order on approval: work orders, BOM, header drawing and rev, the fingerprint, + and the staged drawing (old pages voided, new pages carry the change order). Returns a note for the log.""" + from . import packet_visual as pv + + conn = w.conn + pid = w.id + vis = co["vis"] if isinstance(co["vis"], dict) else {} + staged = co["staged"] if isinstance(co["staged"], dict) else None + row = w.packet + visual = dict(row["visual"] or {}) + notes: list[str] = [] + sets: dict[str, Any] = {} + if isinstance(vis.get("legs"), list) and vis["legs"]: + c = _apply_legs(conn, pid, vis["legs"], row["title"]) + notes.append(f"work orders: {c['added']} added, {c['updated']} kept, {c['removed']} removed") + if isinstance(vis.get("bom"), list) and not visual.get("manual_bom"): + c = pv.merge_visual_bom(conn, pid, vis["bom"]) + notes.append(f"BOM: {c['added']} added, {c['updated']} kept, {c['removed'] + c['na']} dropped") + if co["origin"] == "visual": + if vis.get("to_dwg") not in (None, ""): + sets["drawing"] = clean_opt(vis["to_dwg"], LIMITS["drawing"]) + if vis.get("to_rev") not in (None, ""): + sets["rev"] = clean_opt(vis["to_rev"], LIMITS["rev"]) + if co["target_fp"]: + sets["visual_fp_applied"] = co["target_fp"] + for col, key in (("visual_want", "want"), ("visual_rel", "rel")): + if key in vis: + sets[col] = vis[key] or None + for key in ("fp_parts", "folded", "partspecs"): + if key in vis: + visual[key] = vis[key] + flags = list(row["flags"] or []) + if staged and staged.get("asset_id"): + asset = pv.pdf_asset(conn, staged["asset_id"]) + dwg = sets.get("drawing") or staged.get("dwg") or row["drawing"] + rev = str(staged["rev"]) if staged.get("rev") not in (None, "") else (sets.get("rev") or row["rev"]) + first, count, old = pv.replace_drawing_pages( + conn, pid, asset, dwg=dwg, rev=rev, co_id=int(co["id"]), void_by=w.user_id + ) + for i in range(max(len(old), count)): + o = old[i] if i < len(old) else None + new = i < count + conn.execute( + "INSERT INTO co_items (co_id, old_page, old_dwg, old_rev, new_page, new_dwg, new_rev, added)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s)", + ( + co["id"], + o["n"] if o else None, + o["dwg"] if o else None, + o["rev"] if o else None, + first + i if new else None, + dwg if new else None, + rev if new else None, + o is None, + ), + ) + sets["drawing_state"] = "attached" + flags = pv.flags_without(flags, (*packets.DRAWING_FLAGS, "hand_drawing")) + visual["rev_confirmed"] = True + visual["vault"] = { + "asset_id": str(asset["id"]), + "name": staged.get("name"), + "rev": rev, + "at": pv.now_iso(), + "co_no": co["co_no"], + } + visual.pop("manual_dwg", None) + notes.append(f"drawing {dwg} rev {rev}, {_plural(count, 'page')}; {_plural(len(old), 'old page')} voided") + sets["flags"] = flags + sets["visual"] = Jsonb(visual) + cols = ", ".join(f"{c} = %s" + ("::drawing_state" if c == "drawing_state" else "") for c in sets) + conn.execute(f"UPDATE packets SET {cols} WHERE id = %s", (*sets.values(), pid)) # noqa: S608 (fixed names) + w.changed(*(c for c in sets if c not in ("flags", "visual")), "change_orders") + return "; ".join(notes) or None + + @router.post("/{packet_id}/change-orders/{no}/approve", dependencies=WRITE) def approve_co( packet_id: int, @@ -1401,9 +1586,19 @@ def approve_co( p: Principal = Depends(current_principal), # noqa: B008 d: Db = Depends(get_db), # noqa: B008 ) -> dict[str, Any]: + """Approve a pending change order. A VISUAL or M-Files one is applied to the packet here; a minor one (target + hours or want date only) needs no department verification.""" pp = _pp(d, p) conn = d.conn - content = {"action": str(A.CO_APPROVE), "packet_id": packet_id, "co_no": no} + pre = packets.one( + conn, + "SELECT origin::text AS origin, staged, target_fp FROM change_orders WHERE packet_id = %s AND co_no = %s", + (packet_id, no), + ) + content: dict[str, Any] = {"action": str(A.CO_APPROVE), "packet_id": packet_id, "co_no": no} + if pre is not None and pre["origin"] in FEED_ORIGINS: + content["target_fp"] = pre["target_fp"] + content["staged_sha256"] = pre["staged"].get("sha256") if isinstance(pre["staged"], dict) else None with packet_write( d, p, @@ -1417,6 +1612,12 @@ def approve_co( ) as w: co = _co(conn, packet_id, no) _co_state(co, "pending", "approved") + note: str | None = None + if co["origin"] in FEED_ORIGINS: + vis = co["vis"] if isinstance(co["vis"], dict) else {} + if vis.get("dwg_change") is True and not co["staged"]: + raise ApiError(409, "co_not_staged", MSG_CO_NOT_STAGED, extra={"waiting": co["waiting"]}) + note = _apply_feed_co(w, co) update_row( conn, "change_orders", @@ -1433,7 +1634,7 @@ def approve_co( for code in co_touched(conn, w.packet): conn.execute("INSERT INTO co_required_depts (co_id, dept_code) VALUES (%s, %s)", (co["id"], code)) _co_log_detail(w, no) - w.log(f"Change order {no} approved") + w.log(f"Change order {no} approved", note) return _read(d, pp, packet_id) diff --git a/server/packetstation/api/packet_visual.py b/server/packetstation/api/packet_visual.py new file mode 100644 index 0000000..754b696 --- /dev/null +++ b/server/packetstation/api/packet_visual.py @@ -0,0 +1,580 @@ +"""VISUAL packets by hand: approve the M-Files rev, upload the drawing or the BOM on a flagged draft, and go back to +VISUAL's BOM. + +Every write goes through domain.pipeline.packet_write with its policy action and a signature. The hand uploads are +for a VISUAL draft that is flagged (or already has a hand upload): Clerical checks it and sends it on; auto-release +never takes it because its flags are not empty. Every answer is the full read model. +""" + +from __future__ import annotations + +import hashlib +import uuid +from collections.abc import Mapping +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Depends, Request +from psycopg.types.json import Jsonb +from pydantic import BaseModel, Field +from starlette.concurrency import run_in_threadpool +from starlette.datastructures import UploadFile + +from .. import policy +from ..domain import assets as asset_store +from ..domain import handbom, packets +from ..domain.pipeline import SignBody, Write, next_counter, packet_write +from ..text import LIMITS, clean_opt, reason_ok +from .deps import Db, Principal, current_principal, get_db +from .errors import ApiError +from .packets import WRITE + +router = APIRouter(prefix="/api/packets", tags=["packet-visual"]) + +A = policy.Action + +# A hand BOM file is a small sheet. The multipart overhead on top is allowed before the body is refused unread. +BOM_MAX_BYTES = 5 * 1024 * 1024 +FORM_SLACK = 64 * 1024 + +MSG_NOT_FLAGGED = ( + "A drawing or BOM is uploaded by hand only on a flagged VISUAL draft. This one is not flagged: send it on as it is." +) +MSG_NOT_VISUAL = "This is not a packet made from VISUAL." +MSG_DRAFT_ONLY = "Only a draft takes a hand upload. Once it leaves Clerical, changes come in as change orders." + + +# ---------------------------------------------------------------- bodies + + +class RevApproveBody(BaseModel): + reason: str | None = Field(default=None, max_length=2000) + sign: SignBody | None = None + it_reason: str | None = Field(default=None, max_length=1000) + + +class HandDrawingBody(BaseModel): + asset_id: uuid.UUID + reason: str | None = Field(default=None, max_length=2000) + sign: SignBody | None = None + it_reason: str | None = Field(default=None, max_length=1000) + + +class HandBomOffBody(BaseModel): + sign: SignBody | None = None + it_reason: str | None = Field(default=None, max_length=1000) + + +# ---------------------------------------------------------------- shared helpers + + +def _pp(d: Db, p: Principal) -> policy.Principal: + return packets.policy_principal_for(d.conn, p) + + +def _read(d: Db, pp: policy.Principal, packet_id: int) -> dict[str, Any]: + return packets.read_packet(d.conn, packet_id, pp) + + +def now_iso() -> str: + return datetime.now(UTC).isoformat() + + +def who_of(w: Write) -> dict[str, Any]: + return {"by": str(w.user_id), "by_name": w.principal.user.get("display_name")} + + +def hand_flagged(row: Mapping[str, Any]) -> bool: + """The hand-upload guard: a VISUAL draft that is flagged, still needs its drawing, or already has a hand upload.""" + if row["source_kind"] != "visual" or row["status"] != "draft": + return False + visual = row["visual"] or {} + return bool( + list(row["flags"] or []) + or row["drawing_state"] in packets.DRAWING_STATES_FLAGGED + or visual.get("manual_dwg") + or visual.get("manual_bom") + ) + + +def _hand_guard(w: Write) -> None: + row = w.packet + if row["source_kind"] != "visual": + raise ApiError(409, "not_visual", MSG_NOT_VISUAL) + if row["status"] != "draft": + raise ApiError(403, "wrong_status", MSG_DRAFT_ONLY) + if not hand_flagged(row): + raise ApiError(409, "not_flagged", MSG_NOT_FLAGGED) + + +def pdf_asset(conn: Any, asset_id: Any) -> dict[str, Any]: + asset = asset_store.get(conn, asset_id) + if asset is None: + raise ApiError(404, "not_found", "No such file.") + if asset["mime"] != asset_store.PDF or not asset["pages"]: + raise ApiError(422, "not_pdf", "Attach a PDF drawing.") + return asset + + +def flags_without(flags: Any, drop: tuple[str, ...], add: tuple[str, ...] = ()) -> list[str]: + out = [f for f in (flags or []) if f not in drop] + for f in add: + if f not in out: + out.append(f) + return out + + +def replace_drawing_pages( + conn: Any, + packet_id: int, + asset: Mapping[str, Any], + *, + dwg: str | None, + rev: str | None, + co_id: int | None = None, + void_by: Any = None, +) -> tuple[int, int, list[dict[str, Any]]]: + """Put an asset's pages on the packet as its drawing. + + Without `co_id` (a draft) the current drawing pages are deleted (the assets stay). With `co_id` (a change order + approval) they are voided and point at the first new page. Returns (first new page, page count, old pages). + """ + old = packets.rows( + conn, + "SELECT * FROM packet_pages WHERE packet_id = %s AND type = 'drawing' AND void_at IS NULL ORDER BY n", + (packet_id,), + ) + count = int(asset["pages"] or 0) + if count < 1: + raise ApiError(422, "not_pdf", "That drawing has no pages.") + first = next_counter(conn, packet_id, "page", count) + if co_id is None: + conn.execute( + "DELETE FROM packet_pages WHERE packet_id = %s AND type = 'drawing' AND void_at IS NULL", (packet_id,) + ) + elif old: + conn.execute( + "UPDATE packet_pages SET void_co_id = %s, void_at = now(), void_by = %s, void_new_n = %s," + " row_version = row_version + 1 WHERE id = ANY(%s)", + (co_id, void_by, first, [pg["id"] for pg in old]), + ) + for i in range(count): + conn.execute( + "INSERT INTO packet_pages (packet_id, n, type, asset_id, pn, dwg, rev, co_id)" + " VALUES (%s, %s, 'drawing', %s, %s, %s, %s, %s)", + (packet_id, first + i, asset["id"], i + 1, dwg, rev, co_id), + ) + return first, count, old + + +def set_visual(conn: Any, packet_id: int, visual: Mapping[str, Any]) -> None: + conn.execute("UPDATE packets SET visual = %s WHERE id = %s", (Jsonb(dict(visual)), packet_id)) + + +def _sha(asset: Mapping[str, Any]) -> str: + return bytes(asset["sha256"]).hex() + + +# ---------------------------------------------------------------- approve the M-Files rev + + +@router.post("/{packet_id}/rev-approve", dependencies=WRITE) +def rev_approve( + packet_id: int, + body: RevApproveBody, + request: Request, + p: Principal = Depends(current_principal), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + """A Clerical Supervisor approves the rev M-Files has in place of the rev VISUAL calls, for this draft only.""" + conn = d.conn + reason = reason_ok(body.reason) + pp = _pp(d, p) + row = packets.load_row(conn, packet_id) + staged = packets.staged_lookup(conn, row["drawing"]) if row else None + content = { + "action": str(A.REV_APPROVE), + "packet_id": packet_id, + "drawing": row["drawing"] if row else None, + "visual_rev": row["rev"] if row else None, + "rev": staged["rev"] if staged else None, + "sha256": bytes(staged["sha256"]).hex() if staged and staged["sha256"] else None, + "reason": reason, + } + with packet_write( + d, + p, + packet_id, + A.REV_APPROVE, + request=request, + extra={"reason": body.reason}, + it_reason=body.it_reason, + sign=body.sign, + sign_content=content, + policy_principal=pp, + ) as w: + if w.packet["source_kind"] != "visual": + raise ApiError(409, "not_visual", MSG_NOT_VISUAL) + lk = packets.staged_lookup(conn, w.packet["drawing"]) + if lk is None: + raise ApiError( + 409, "nothing_staged", "No M-Files copy of this drawing is ready yet. Try after the next run." + ) + asset = asset_store.get(conn, lk["asset_id"]) + if asset is None or not asset["pages"]: + raise ApiError( + 409, "nothing_staged", "The M-Files copy of this drawing is missing. Try after the next run." + ) + rev = str(lk["rev"]) + vis_rev = w.packet["rev"] or "" + replace_drawing_pages(conn, packet_id, asset, dwg=w.packet["drawing"], rev=rev) + visual = dict(w.packet["visual"] or {}) + visual["rev_ok"] = {"rev": rev, "vis_rev": vis_rev, **who_of(w), "at": now_iso(), "reason": reason} + visual["rev_confirmed"] = True + visual["vault"] = { + "asset_id": str(asset["id"]), + "name": lk["name"], + "rev": rev, + "note": lk["note"], + "at": now_iso(), + } + visual.pop("manual_dwg", None) + set_visual(conn, packet_id, visual) + conn.execute( + "UPDATE packets SET drawing_state = 'attached', flags = %s WHERE id = %s", + (flags_without(w.packet["flags"], (*packets.DRAWING_FLAGS, "hand_drawing")), packet_id), + ) + w.changed("pages", "drawing_state", "flags", "visual") + w.detail.update(asset_id=str(asset["id"]), rev=rev, vis_rev=vis_rev) + w.log("Different rev approved by hand", f"rev {rev} in place of VISUAL rev {vis_rev or '-'} · {reason}") + return _read(d, pp, packet_id) + + +# ---------------------------------------------------------------- hand drawing + + +@router.post("/{packet_id}/hand/drawing", dependencies=WRITE) +def hand_drawing( + packet_id: int, + body: HandDrawingBody, + request: Request, + p: Principal = Depends(current_principal), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + """Clerical uploads the drawing by hand on a flagged VISUAL draft. It replaces the drawing pages.""" + conn = d.conn + reason = clean_opt(body.reason, LIMITS["reason"]) + if reason is None: + raise ApiError(400, "reason_required", "Say why the drawing is uploaded by hand.") + asset = pdf_asset(conn, body.asset_id) + pp = _pp(d, p) + content = { + "action": "hand_drawing", + "packet_id": packet_id, + "asset_id": str(asset["id"]), + "sha256": _sha(asset), + "pages": asset["pages"], + "reason": reason, + } + with packet_write( + d, + p, + packet_id, + A.EDIT_DOCUMENT_SET, + request=request, + it_reason=body.it_reason, + sign=body.sign, + sign_content=content, + require_sign=True, + policy_principal=pp, + ) as w: + _hand_guard(w) + vis_rev = w.packet["rev"] or "" + first, count, _ = replace_drawing_pages(conn, packet_id, asset, dwg=w.packet["drawing"], rev=w.packet["rev"]) + name = asset["original_name"] or "drawing.pdf" + visual = dict(w.packet["visual"] or {}) + visual["manual_dwg"] = { + "asset_id": str(asset["id"]), + "name": name, + "reason": reason, + **who_of(w), + "at": now_iso(), + "vis_rev": vis_rev, + } + visual["rev_confirmed"] = False + set_visual(conn, packet_id, visual) + conn.execute( + "UPDATE packets SET drawing_state = 'hand', flags = %s WHERE id = %s", + (flags_without(w.packet["flags"], packets.DRAWING_FLAGS, ("hand_drawing",)), packet_id), + ) + w.changed("pages", "drawing_state", "flags", "visual") + w.detail.update(asset_id=str(asset["id"]), pages=count, first_page=first) + w.log("Drawing uploaded by hand", f"{name}, {count} page{'s' if count != 1 else ''} · {reason}") + return _read(d, pp, packet_id) + + +# ---------------------------------------------------------------- hand BOM + + +def _insert_hand_bom(conn: Any, packet_id: int, lines: list[dict[str, Any]]) -> None: + conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,)) + for i, b in enumerate(lines): + conn.execute( + "INSERT INTO bom_lines (packet_id, item, qty, part_id, description, dwg_ref, cl_item, length, material," + " src, sort) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'hand', %s)", + ( + packet_id, + b["item"], + b["qty"], + b["part_id"], + b["description"], + b["dwg_ref"], + b["cl_item"], + b["length"], + b["material"], + i, + ), + ) + + +def _hand_bom_write( + d: Db, + p: Principal, + request: Request, + packet_id: int, + *, + data: bytes, + name: str | None, + reason: str, + sign: SignBody, + it_reason: str | None, +) -> dict[str, Any]: + conn = d.conn + try: + lines = handbom.parse(data, name) + except handbom.BadFile as e: + raise ApiError(422, "bad_file", str(e)) from None + pp = _pp(d, p) + sha = hashlib.sha256(data).hexdigest() + fname = clean_opt(name, LIMITS["file_name"]) or "bom" + content = { + "action": "hand_bom", + "packet_id": packet_id, + "file": fname, + "sha256": sha, + "lines": len(lines), + "reason": reason, + } + with packet_write( + d, + p, + packet_id, + A.EDIT_BOM, + request=request, + it_reason=it_reason, + sign=sign, + sign_content=content, + require_sign=True, + policy_principal=pp, + ) as w: + _hand_guard(w) + _insert_hand_bom(conn, packet_id, lines) + visual = dict(w.packet["visual"] or {}) + visual["manual_bom"] = { + "name": fname, + "reason": reason, + **who_of(w), + "at": now_iso(), + "lines": len(lines), + "sha256": sha, + } + set_visual(conn, packet_id, visual) + conn.execute( + "UPDATE packets SET flags = %s WHERE id = %s", + (flags_without(w.packet["flags"], (), ("hand_bom",)), packet_id), + ) + w.changed("bom", "flags", "visual") + w.detail.update(file=fname, sha256=sha, lines=len(lines)) + n = len(lines) + w.log("BOM uploaded by hand", f"{fname} · {n} line{'s' if n != 1 else ''} · {reason}") + return _read(d, pp, packet_id) + + +@router.post("/{packet_id}/hand/bom") +async def hand_bom( + packet_id: int, + request: Request, + p: Principal = Depends(current_principal), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + """Clerical uploads the BOM by hand (multipart: file, reason, sign_factor, sign_secret, it_reason).""" + length = request.headers.get("content-length") + if length is not None and length.isdigit() and int(length) > BOM_MAX_BYTES + FORM_SLACK: + raise ApiError(413, "too_large", f"The BOM file is over the {BOM_MAX_BYTES // (1024 * 1024)} MB limit.") + form = await request.form(max_files=1, max_fields=8) + try: + file = form.get("file") + if not isinstance(file, UploadFile): + raise ApiError(422, "invalid_request", "Send the BOM file in the 'file' field.") + + def field(name: str, cap: int) -> str | None: + v = form.get(name) + return v[:cap] if isinstance(v, str) else None + + reason = clean_opt(field("reason", 2000), LIMITS["reason"]) + if reason is None: + raise ApiError(400, "reason_required", "Say why the BOM is uploaded by hand.") + factor, secret = field("sign_factor", 20), field("sign_secret", 1024) + if not factor or secret is None: + raise ApiError( + 400, "signature_required", "This step needs your signature.", extra={"meaning": "Edit the BOM"} + ) + if factor not in ("pin", "password"): + raise ApiError(422, "invalid_request", "sign_factor is pin or password.") + sign = SignBody(factor=factor, secret=secret) # type: ignore[arg-type] + data = await file.read(BOM_MAX_BYTES + 1) + if len(data) > BOM_MAX_BYTES: + raise ApiError(413, "too_large", f"The BOM file is over the {BOM_MAX_BYTES // (1024 * 1024)} MB limit.") + name = file.filename + it_reason = field("it_reason", 1000) + finally: + await form.close() + return await run_in_threadpool( + lambda: _hand_bom_write( + d, p, request, packet_id, data=data, name=name, reason=reason, sign=sign, it_reason=it_reason + ) + ) + + +@router.delete("/{packet_id}/hand/bom", dependencies=WRITE) +def hand_bom_off( + packet_id: int, + request: Request, + body: HandBomOffBody | None = None, + p: Principal = Depends(current_principal), # noqa: B008 + d: Db = Depends(get_db), # noqa: B008 +) -> dict[str, Any]: + """Drop the hand BOM and use VISUAL's BOM again, rebuilt from the cached materials.""" + body = body or HandBomOffBody() + conn = d.conn + pp = _pp(d, p) + with packet_write( + d, + p, + packet_id, + A.EDIT_BOM, + request=request, + it_reason=body.it_reason, + sign=body.sign, + sign_content={"action": "hand_bom_off", "packet_id": packet_id}, + require_sign=body.sign is not None, + policy_principal=pp, + ) as w: + if w.packet["source_kind"] != "visual": + raise ApiError(409, "not_visual", MSG_NOT_VISUAL) + if w.packet["status"] != "draft": + raise ApiError(403, "wrong_status", MSG_DRAFT_ONLY) + visual = dict(w.packet["visual"] or {}) + old = visual.pop("manual_bom", None) + if not old: + raise ApiError(409, "no_hand_bom", "This packet has no BOM uploaded by hand.") + from ..feed import materials as feed_materials # agent B's module, imported where it is used + + lines = feed_materials.bom_for_packet(conn, packet_id) + conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,)) + feed_materials.insert_bom(conn, packet_id, lines) + set_visual(conn, packet_id, visual) + conn.execute( + "UPDATE packets SET flags = %s WHERE id = %s", + (flags_without(w.packet["flags"], ("hand_bom",)), packet_id), + ) + w.changed("bom", "flags", "visual") + n = len(lines) + name = old.get("name") if isinstance(old, dict) else None + note = f"{n} line{'s' if n != 1 else ''} from VISUAL" if n else "VISUAL has no materials for it yet" + w.log("Hand-uploaded BOM taken off", " · ".join(x for x in (name, note) if x)) + return _read(d, pp, packet_id) + + +# ---------------------------------------------------------------- VISUAL BOM lines (change order approval) + + +def merge_visual_bom(conn: Any, packet_id: int, raw_lines: list[Any]) -> dict[str, int]: + """Bring a packet's BOM to VISUAL's lines (change order `vis.bom`, dicts or BomLine objects) without losing work. + + When nothing has been recorded on the lines (no pulled mark, heat, MTR or short) the BOM is replaced through + feed.materials.insert_bom, exactly as the feed writes it. Otherwise lines are matched on vkey and updated in + place, so pulled marks, heats and shorts stay. New lines are added with the same encoding (item numbers from + feed.materials.db_items, the label in cl_item, cons Y/N, sub Y or NULL). Lines VISUAL dropped are deleted, or + marked n/a when they were pulled or have a short. + """ + from ..feed.materials import BomLine, db_items, insert_bom + + lines = [ln if isinstance(ln, BomLine) else BomLine.from_dict(ln) for ln in raw_lines] + counts = {"added": 0, "updated": 0, "removed": 0, "na": 0} + existing = packets.rows(conn, "SELECT * FROM bom_lines WHERE packet_id = %s ORDER BY item", (packet_id,)) + worked = ( + any(e["pulled"] or e["heat"] or e["mtr_asset_id"] for e in existing) + or conn.execute("SELECT 1 FROM bom_shorts WHERE packet_id = %s LIMIT 1", (packet_id,)).fetchone() + ) + if not worked: + # Nothing recorded on the lines yet: write them exactly as the feed does. + conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,)) + insert_bom(conn, packet_id, lines) + counts["added"] = len(lines) + counts["removed"] = len(existing) + return counts + by_vkey = {e["vkey"]: e for e in existing if e["vkey"]} + used = {int(e["item"]) for e in existing} + wanted = db_items(lines) + kept: set[int] = set() + for sort, (b, want_item) in enumerate(zip(lines, wanted, strict=True)): + values = ( + b.qty, + b.part_id[:60] or None, + b.desc[:240] or None, + b.loc[:120] or None, + b.whs or None, + b.comm or None, + "Y" if b.cons else "N", + "Y" if b.sub else None, + b.na, + b.grp, + b.issued, + sort, + ) + match = by_vkey.get(b.vkey) if b.vkey else None + if match is not None and int(match["id"]) not in kept: + kept.add(int(match["id"])) + conn.execute( + "UPDATE bom_lines SET qty = %s, part_id = %s, description = %s, loc = %s, whs = %s, comm = %s," + " cons = %s, sub = %s, na = %s, grp = %s, issued = coalesce(%s, issued), sort = %s, cl_item = %s," + " row_version = row_version + 1 WHERE id = %s", + (*values, b.item, match["id"]), + ) + counts["updated"] += 1 + continue + item = want_item + while item in used: + item += 1 + used.add(item) + new_id = conn.execute( + "INSERT INTO bom_lines (packet_id, item, cl_item, qty, part_id, description, loc, whs, comm, cons, sub, na," + " grp, issued, sort, vkey, src) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s," + " 'visual') RETURNING id", + (packet_id, item, b.item, *values, b.vkey), + ).fetchone()[0] + kept.add(int(new_id)) + counts["added"] += 1 + for e in existing: + if int(e["id"]) in kept: + continue + busy = e["pulled"] or conn.execute("SELECT 1 FROM bom_shorts WHERE bom_line_id = %s", (e["id"],)).fetchone() + if busy: + if not e["na"]: + conn.execute("UPDATE bom_lines SET na = true, row_version = row_version + 1 WHERE id = %s", (e["id"],)) + counts["na"] += 1 + else: + conn.execute("DELETE FROM bom_lines WHERE id = %s", (e["id"],)) + counts["removed"] += 1 + return counts diff --git a/server/packetstation/api/packets.py b/server/packetstation/api/packets.py index 81cf7a9..d106afe 100644 --- a/server/packetstation/api/packets.py +++ b/server/packetstation/api/packets.py @@ -299,6 +299,7 @@ def list_packets( project_id: int | None = Query(None), q: str = Query("", max_length=200), flag: str = Query("", max_length=20), + co: str = Query("", max_length=20), limit: int = Query(500, ge=1, le=2000), removed: int = Query(0, ge=0, le=1), ) -> dict[str, Any]: @@ -309,6 +310,7 @@ def list_packets( "project_id": project_id, "q": clean_text(q, 200), "flag": flag, + "co": co.strip().lower() or None, "limit": limit, "removed": bool(removed) and p.is_admin, } diff --git a/server/packetstation/cli.py b/server/packetstation/cli.py index 436465b..ec251f1 100644 --- a/server/packetstation/cli.py +++ b/server/packetstation/cli.py @@ -1,4 +1,4 @@ -"""`packetstation` command line: app, worker, migrate, doctor, version, healthcheck.""" +"""`packetstation` command line: app, worker, job, fixtures, migrate, doctor, version, healthcheck.""" from __future__ import annotations @@ -55,7 +55,52 @@ def cmd_worker(_: argparse.Namespace) -> int: _setup_logging(s.log_level) guard.startup_checks(s) beat = Path(os.environ.get("WORKER_BEAT_FILE") or s.worker_beat_file) - return worker.run(beat, s.ps_build) + return worker.run(beat, s.ps_build, settings=s) + + +def cmd_job_run(a: argparse.Namespace) -> int: + """Run one job once in the foreground (trigger `cli`) and print its job_runs row as JSON.""" + from . import guard + from .worker.jobs import load_jobs + from .worker.runs import run_job + from .worker.scheduler import Scheduler + + s = _settings() + _setup_logging(s.log_level) + guard.startup_checks(s) + jobs = load_jobs() + if a.name not in jobs: + print(f"packetstation job run: no job {a.name!r}; jobs: {', '.join(sorted(jobs))}", file=sys.stderr) + return 2 + sched = Scheduler(s, jobs=jobs) + try: + row = run_job(sched, a.name, "cli") + except Exception as e: + from .worker.redaction import redact + + print(f"packetstation job run: {redact(f'{type(e).__name__}: {e}')}", file=sys.stderr) + return 1 + print(json.dumps(row, indent=2, default=str)) + return 1 if row.get("status") == "failed" else 0 + + +def cmd_fixtures_use(a: argparse.Namespace) -> int: + """Point the fixture sources at state N: writes DIR/state. Never on STACK=prod.""" + if os.environ.get("STACK", "").strip().lower() == "prod": + print("packetstation fixtures: refused on STACK=prod", file=sys.stderr) + return 2 + if a.state < 1: + print("packetstation fixtures use: the state is a number from 1", file=sys.stderr) + return 2 + d = Path(a.dir or os.environ.get("FIXTURE_DIR") or "/fixtures") + if not (d / f"feed.{a.state}.json").is_file(): + print(f"packetstation fixtures use: {d / f'feed.{a.state}.json'} not found", file=sys.stderr) + return 1 + tmp = d / "state.tmp" + tmp.write_text(str(a.state) + "\n", encoding="utf-8") + tmp.replace(d / "state") + print(f"fixture state {a.state} in {d}") + return 0 def cmd_migrate(a: argparse.Namespace) -> int: @@ -148,6 +193,17 @@ def build_parser() -> argparse.ArgumentParser: sub = ap.add_subparsers(dest="cmd", required=True) sub.add_parser("app", help="Run the web app (uvicorn on 0.0.0.0:8000)").set_defaults(func=cmd_app) sub.add_parser("worker", help="Run the background worker").set_defaults(func=cmd_worker) + j = sub.add_parser("job", help="Worker jobs") + jsub = j.add_subparsers(dest="job_cmd", required=True) + jr = jsub.add_parser("run", help="Run one job once now and print its run as JSON (exit 1 when it failed)") + jr.add_argument("name", help="visual-feed, drawing-lookup or housekeeping") + jr.set_defaults(func=cmd_job_run) + f = sub.add_parser("fixtures", help="Synthetic feed fixtures (CI and tests)") + fsub = f.add_subparsers(dest="fixtures_cmd", required=True) + fu = fsub.add_parser("use", help="Point the fixture sources at state N (writes DIR/state)") + fu.add_argument("state", type=int) + fu.add_argument("--dir", help="The fixture directory (default FIXTURE_DIR or /fixtures)") + fu.set_defaults(func=cmd_fixtures_use) m = sub.add_parser("migrate", help="Apply pending SQL migrations as the migrate role") m.add_argument( "--bootstrap-roles", diff --git a/server/packetstation/domain/assets.py b/server/packetstation/domain/assets.py index 88f596f..828763d 100644 --- a/server/packetstation/domain/assets.py +++ b/server/packetstation/domain/assets.py @@ -18,6 +18,7 @@ from pathlib import Path from typing import Any, BinaryIO from psycopg.rows import dict_row +from psycopg.types.json import Jsonb from .. import audit from ..api.errors import ApiError @@ -69,7 +70,35 @@ def path_of(settings: Settings, key: str) -> Path: def pdf_pages(path: Path) -> int: - """Page count through pypdf; 422 bad_pdf when it does not parse or has no pages.""" + """Page count through pypdfium2, or pypdf when pypdfium2 is missing or cannot read the file. + + 422 bad_pdf when neither parses it or it has no pages. + """ + n = _pages_pdfium(path) + if n is None: + n = _pages_pypdf(path) + if n < 1: + raise ApiError(422, "bad_pdf", "That PDF has no pages.") + return n + + +def _pages_pdfium(path: Path) -> int | None: + try: + import pypdfium2 + except ImportError: + return None + try: + doc = pypdfium2.PdfDocument(str(path)) + except Exception as e: # any pdfium failure falls back to pypdf + log.info("pypdfium2 could not read the PDF (%s); trying pypdf", type(e).__name__) + return None + try: + return len(doc) + finally: + doc.close() + + +def _pages_pypdf(path: Path) -> int: from pypdf import PdfReader from pypdf.errors import PyPdfError @@ -78,8 +107,6 @@ def pdf_pages(path: Path) -> int: except (PyPdfError, ValueError, KeyError, TypeError, OSError, IndexError, AttributeError) as e: log.info("upload is not a readable PDF: %s", type(e).__name__) raise ApiError(422, "bad_pdf", "That PDF could not be read.") from None - if n < 1: - raise ApiError(422, "bad_pdf", "That PDF has no pages.") return n @@ -115,6 +142,40 @@ def _fsync_dir(path: Path) -> None: os.close(fd) +def _existing(conn: Any, sha: bytes, size: int) -> dict[str, Any] | None: + """The stored asset with the same bytes (same sha256 and size), oldest first.""" + with conn.cursor(row_factory=dict_row) as cur: + r = cur.execute( + "SELECT * FROM assets WHERE sha256 = %s AND size = %s ORDER BY uploaded_at LIMIT 1", (sha, size) + ).fetchone() + return dict(r) if r else None + + +def _place(settings: Settings, tmp: Path) -> tuple[uuid.UUID, str, Path]: + """Rename a fsynced file to ASSETS_DIR//, then fsync the directory. Returns (id, key, path).""" + aid = uuid.uuid4() + key = storage_key(aid) + final = path_of(settings, key) + final.parent.mkdir(parents=True, exist_ok=True) + os.replace(tmp, final) + _fsync_dir(final.parent) + return aid, key, final + + +def _insert(conn: Any, values: tuple[Any, ...]) -> dict[str, Any]: + """Insert one assets row from (id, sha256, size, mime, name, key, source, source_ref, pages, by, packet_id).""" + v = list(values) + if v[7] is not None and not isinstance(v[7], Jsonb): + v[7] = Jsonb(v[7]) + with conn.cursor(row_factory=dict_row) as cur: + row = cur.execute( + "INSERT INTO assets (id, sha256, size, mime, original_name, storage_key, source, source_ref, pages," + " uploaded_by, packet_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING *", + v, + ).fetchone() + return dict(row) + + def store_upload( conn: Any, settings: Settings, @@ -166,29 +227,15 @@ def store_upload( if mime is None or size == 0: raise ApiError(415, "unsupported_type", "Only PDF, PNG and JPEG files can be stored.") sha = digest.digest() - with conn.cursor(row_factory=dict_row) as cur: - existing = cur.execute( - "SELECT * FROM assets WHERE sha256 = %s AND size = %s ORDER BY uploaded_at LIMIT 1", (sha, size) - ).fetchone() + existing = _existing(conn, sha, size) if existing is not None: - return public(dict(existing)) + return public(existing) pages = pdf_pages(tmp) if mime == PDF else None - aid = uuid.uuid4() - key = storage_key(aid) - final = path_of(settings, key) - final.parent.mkdir(parents=True, exist_ok=True) - os.replace(tmp, final) + aid, key, placed = _place(settings, tmp) keep = True - placed = final - _fsync_dir(final.parent) name = safe_name(original_name) with conn.transaction(): - with conn.cursor(row_factory=dict_row) as cur: - row = cur.execute( - "INSERT INTO assets (id, sha256, size, mime, original_name, storage_key, source, source_ref, pages," - " uploaded_by, packet_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING *", - (aid, sha, size, mime, name, key, source, source_ref, pages, user, packet_id), - ).fetchone() + row = _insert(conn, (aid, sha, size, mime, name, key, source, source_ref, pages, user, packet_id)) audit.append( conn, action="asset.uploaded", @@ -213,6 +260,69 @@ def store_upload( tmp.unlink() +def store_file( + conn: Any, + settings: Settings, + path: Path, + *, + mime: str = PDF, + original_name: Any, + source: str, + source_ref: Any = None, + uploaded_by: Any = None, + packet_id: int | None = None, +) -> dict[str, Any]: + """Store a file the server wrote itself (the worker's M-Files download). Returns the full assets row. + + The file is consumed: renamed into ASSETS_DIR when its bytes are new, deleted when the same bytes are already + stored (same sha256 and size; that row comes back). Write it under ASSETS_DIR (ASSETS_DIR/tmp) so the rename + stays on one file system. It is hashed in chunks, its first bytes must match `mime`, and it is fsynced before + the rename. The row carries one extra key, `reused`. No audit row: the caller writes its own. + Raises 413 too_large, 415 unsupported_type, 422 bad_pdf. + """ + path = Path(path) + limit = int(settings.upload_max_bytes) + placed: Path | None = None + done = False + try: + digest = hashlib.sha256() + size = 0 + head = b"" + with path.open("r+b") as f: + while True: + chunk = f.read(CHUNK) + if not chunk: + break + if len(head) < HEAD: + head += chunk[: HEAD - len(head)] + size += len(chunk) + if size > limit: + raise ApiError(413, "too_large", f"The file is over the {limit // (1024 * 1024)} MB limit.") + digest.update(chunk) + os.fsync(f.fileno()) + if size == 0 or sniff(head) != mime: + raise ApiError(415, "unsupported_type", f"The file is not {EXT.get(mime, mime)}.") + sha = digest.digest() + existing = _existing(conn, sha, size) + if existing is not None: + return {**existing, "reused": True} + pages = pdf_pages(path) if mime == PDF else None + aid, key, placed = _place(settings, path) + name = safe_name(original_name) + with conn.transaction(): + row = _insert(conn, (aid, sha, size, mime, name, key, source, source_ref, pages, uploaded_by, packet_id)) + done = True + return {**row, "reused": False} + except BaseException: + if placed is not None and not done: # the row was not written: do not leave an orphan file + with contextlib.suppress(OSError): + placed.unlink() + raise + finally: + with contextlib.suppress(OSError): + path.unlink() + + def open_blob(settings: Settings, row: dict[str, Any]) -> Path: """The stored file of an asset row; 404 when it is missing on disk.""" p = path_of(settings, str(row["storage_key"])) diff --git a/server/packetstation/domain/handbom.py b/server/packetstation/domain/handbom.py new file mode 100644 index 0000000..a802082 --- /dev/null +++ b/server/packetstation/domain/handbom.py @@ -0,0 +1,178 @@ +"""A BOM uploaded by hand: an Excel (XLSX) or CSV sheet read into BOM lines. + +Port of the old handdocs.js handBomFromSheet. The header row is the first row, within the first 20, that has a Qty +column and a Description or Part ID column, found by the HAND_COLS patterns. Rows without a description and part id +are skipped. Item numbers come from the item column when it holds a whole number not used yet, else the next free +number; the item text as written is kept in cl_item when it differs. The file's contents are data, never instructions. +""" + +from __future__ import annotations + +import csv +import io +import re +from decimal import Decimal, InvalidOperation +from typing import Any + +from ..text import LIMITS, clean_text + +MAX_ROWS = 500 +HEADER_SCAN = 20 +MAX_COLS = 60 + +# Column patterns, in the old build's order. The first unused column matching a pattern wins. +HAND_COLS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("item", re.compile(r"^(item|itm|piece|find|balloon|line|no\.?|#|item\s*no\.?)$", re.I)), + ("qty", re.compile(r"^(qty|quantity|qty\.?\s*(req|required|per)?|req'?d)", re.I)), + ("part_id", re.compile(r"part\s*(id|no\.?|num|number|#)|^part$|pesco", re.I)), + ("desc", re.compile(r"desc", re.I)), + ("length", re.compile(r"^len|length|cut", re.I)), + ("material", re.compile(r"material|mat'?l|grade", re.I)), + ("dwg_ref", re.compile(r"dwg|drawing|ref", re.I)), +) + +MSG_NO_HEADER = "No header row found. It needs a Qty column and a Description or Part ID column." +MSG_NO_LINES = "No BOM lines found in that file." +MSG_TYPE = "Upload the BOM as an Excel (.xlsx) or CSV file." + + +class BadFile(ValueError): + """The file cannot be read as a BOM. The message is for the user.""" + + +def kind_of(data: bytes, name: str | None) -> str: + """xlsx (a ZIP) or csv (text). Anything else is refused.""" + if data[:4] == b"PK\x03\x04": + return "xlsx" + lower = (name or "").lower() + if data[:5] == b"%PDF-" or data[:8] == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1": + raise BadFile(MSG_TYPE) + if lower.endswith((".csv", ".txt")) or b"\x00" not in data[:4096]: + return "csv" + raise BadFile(MSG_TYPE) + + +def _cell(value: Any) -> str: + if value is None: + return "" + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + return str(value).strip() + + +def _rows_xlsx(data: bytes) -> list[list[str]]: + try: + from openpyxl import load_workbook + except ImportError as e: # pragma: no cover - openpyxl is a dependency + raise BadFile("The server cannot read Excel files right now. Use a CSV file.") from e + try: + wb = load_workbook(io.BytesIO(data), read_only=True, data_only=True) + except Exception as e: # openpyxl raises many types on a bad file + raise BadFile("That Excel file could not be read.") from e + try: + ws = wb.worksheets[0] if wb.worksheets else None + if ws is None: + raise BadFile(MSG_NO_LINES) + out: list[list[str]] = [] + for row in ws.iter_rows(values_only=True): + out.append([_cell(v) for v in row[:MAX_COLS]]) + if len(out) > MAX_ROWS + HEADER_SCAN: + break + return out + finally: + wb.close() + + +def _rows_csv(data: bytes) -> list[list[str]]: + try: + text = data.decode("utf-8-sig") + except UnicodeDecodeError: + text = data.decode("cp1252", errors="replace") + sample = text[:8192] + try: + dialect: Any = csv.Sniffer().sniff(sample, delimiters=",;\t|") + except csv.Error: + dialect = csv.excel + out: list[list[str]] = [] + for row in csv.reader(io.StringIO(text), dialect): + out.append([_cell(v) for v in row[:MAX_COLS]]) + if len(out) > MAX_ROWS + HEADER_SCAN: + break + return out + + +def find_header(rows: list[list[str]]) -> tuple[int, dict[str, int]]: + """The header row index and the column map, or BadFile.""" + for i, row in enumerate(rows[:HEADER_SCAN]): + m: dict[str, int] = {} + for key, rx in HAND_COLS: + for j, h in enumerate(row): + if h and rx.search(h) and j not in m.values(): + m[key] = j + break + if "qty" in m and ("desc" in m or "part_id" in m): + return i, m + raise BadFile(MSG_NO_HEADER) + + +def _qty(text: str) -> Decimal: + digits = re.sub(r"[^\d.\-]", "", text) + try: + q = Decimal(digits) if digits else Decimal(0) + except InvalidOperation: + return Decimal(0) + return q if q.is_finite() else Decimal(0) + + +def lines_from_rows(rows: list[list[str]]) -> list[dict[str, Any]]: + hi, m = find_header(rows) + + def cell(r: list[str], key: str) -> str: + j = m.get(key) + return r[j].strip() if j is not None and j < len(r) else "" + + out: list[dict[str, Any]] = [] + used: set[int] = set() + for r in rows[hi + 1 :]: + desc, pid, qty = cell(r, "desc"), cell(r, "part_id"), cell(r, "qty") + if not desc and not pid: + continue # blank rows, totals, notes + raw_item = cell(r, "item") + item: int | None = None + if re.fullmatch(r"\d{1,5}", raw_item) and 1 <= int(raw_item) <= 100_000 and int(raw_item) not in used: + item = int(raw_item) + if item is None: + item = len(out) + 1 + while item in used: + item += 1 + used.add(item) + cl_item = clean_text(raw_item, LIMITS["part_id"]) or None if raw_item != str(item) else None + out.append( + { + "item": item, + "cl_item": cl_item, + "qty": _qty(qty), + "part_id": clean_text(pid, LIMITS["part_id"]) or None, + "description": clean_text(desc, LIMITS["description"]) or None, + "length": clean_text(cell(r, "length"), LIMITS["length"]) or None, + "material": clean_text(cell(r, "material"), LIMITS["material"]) or None, + "dwg_ref": clean_text(cell(r, "dwg_ref"), LIMITS["drawing"]) or None, + } + ) + if len(out) >= MAX_ROWS: + break + if not out: + raise BadFile(MSG_NO_LINES) + return out + + +def parse(data: bytes, name: str | None) -> list[dict[str, Any]]: + """BOM lines from an XLSX or CSV file: [{item, cl_item, qty, part_id, description, length, material, dwg_ref}]. + + Raises BadFile with the reason for the user. + """ + if not data: + raise BadFile("The file is empty.") + kind = kind_of(data, name) + rows = _rows_xlsx(data) if kind == "xlsx" else _rows_csv(data) + return lines_from_rows(rows) diff --git a/server/packetstation/domain/packets.py b/server/packetstation/domain/packets.py index 31e40bd..8ca2c3b 100644 --- a/server/packetstation/domain/packets.py +++ b/server/packetstation/domain/packets.py @@ -7,6 +7,7 @@ boards from header columns, v_open_cos and the open short count only. from __future__ import annotations +import re import uuid from collections.abc import Iterable, Mapping from dataclasses import dataclass, field @@ -20,7 +21,7 @@ from ..api.errors import ApiError from . import checklists HAND_FLAG = "Made by hand - check it and send it on" -DRAWING_NEEDED_FLAG = "Drawing needed - looking it up in the vault" +DRAWING_NEEDED_FLAG = "Made from VISUAL - drawing not attached yet" DRAWING_STATES_ATTACHED = ("attached", "hand") # Keys left out of the sealed content: they change without the packet changing (who can do what, the counter, @@ -37,6 +38,7 @@ SEAL_EXCLUDE = ( "checklist_items", "qc_items", "flag", + "visual_state", ) @@ -67,19 +69,102 @@ def load_row(conn: Any, packet_id: int, *, lock: bool = False) -> dict[str, Any] return one(conn, sql, (packet_id,)) -def flag_of(status: str, source_kind: str, flags: Iterable[str] | None, drawing_state: str | None = None) -> str | None: - """The draft flag shown on cards and the packet page. The old packetFlag(), phase 2 part.""" +# Flags the drawing job owns; a hand drawing or an approved rev clears them. +DRAWING_FLAGS = ("drawing_needed", "rev_mismatch", "not_released", "not_found", "lookup_failed", "bad_rev") +# drawing_state values that mean the drawing still needs attention. +DRAWING_STATES_FLAGGED = ("needed", "rev_mismatch", "not_released", "not_found", "lookup_failed") +HAND_UPLOAD_FLAGS = ("hand_drawing", "hand_bom") + +# Flag kinds in display priority (the first one present gives the text). +FLAG_PRIORITY = ( + "hand_drawing", + "hand_bom", + "rev_mismatch", + "not_released", + "not_found", + "lookup_failed", + "bad_rev", + "drawing_needed", + "no_drawing", + "wo_gone", + "cancelled", +) +FLAG_TEXTS = { + "not_found": "Drawing not in M-Files - check the drawing number in VISUAL", + "bad_rev": "VISUAL rev is not a number - drawing not attached", + "lookup_failed": "M-Files lookup failed", + "drawing_needed": DRAWING_NEEDED_FLAG, + "no_drawing": "No drawing number in VISUAL - add it there or upload the drawing by hand", + "wo_gone": "Work order no longer in VISUAL", + "cancelled": "Work order cancelled in VISUAL", +} +# drawing_state -> flag kind, for a visual draft whose flags list does not name the problem. +STATE_KINDS = { + "needed": "drawing_needed", + "rev_mismatch": "rev_mismatch", + "not_released": "not_released", + "not_found": "not_found", + "lookup_failed": "lookup_failed", +} +RX_FLAG_KIND = re.compile(r"^[a-z_]{1,30}$") + + +def _appr_note(visual: Mapping[str, Any] | None) -> str: + appr = (visual or {}).get("appr") if isinstance(visual, Mapping) else None + if isinstance(appr, Mapping): + note = appr.get("note") + if isinstance(note, str) and note.strip(): + return note.strip() + return "" + + +def flag_of( + status: str, + source_kind: str, + flags: Iterable[str] | None, + drawing_state: str | None = None, + visual: Mapping[str, Any] | None = None, +) -> str | None: + """The draft flag shown on cards and the packet page: the old packetFlag(), handFlag() and revMismatch() texts. + + Only drafts carry a flag. A hand draft keeps the phase 2 text. A VISUAL draft shows its most important flag kind. + """ if status != "draft": return None if source_kind == "hand": return HAND_FLAG - if "drawing_needed" in (flags or ()): - return DRAWING_NEEDED_FLAG - return None + kinds = list(flags or ()) + if not kinds and drawing_state in STATE_KINDS: + kinds = [STATE_KINDS[drawing_state]] + if not kinds: + return None + dwg, bom = "hand_drawing" in kinds, "hand_bom" in kinds + if dwg or bom: + what = "Drawing and BOM" if dwg and bom else "Drawing" if dwg else "BOM" + return f"{what} uploaded by hand - check it and send it on" + note = _appr_note(visual) + for kind in FLAG_PRIORITY: + if kind not in kinds: + continue + if kind == "rev_mismatch": + return note or "Rev mismatch: VISUAL and M-Files show different revs" + if kind == "not_released": + return note or "Not released in M-Files yet" + if kind in ("lookup_failed", "bad_rev"): + # The drawing job writes the whole text ("M-Files lookup failed: ...", "VISUAL rev A is not a number; + # M-Files has rev 3 - not attached"). + return note or FLAG_TEXTS[kind] + return FLAG_TEXTS[kind] + return "Flagged: " + ", ".join(k.replace("_", " ") for k in kinds) # SQL for the same rule, for the list filter. -FLAG_SQL = "(p.status = 'draft' AND (p.source_kind = 'hand' OR 'drawing_needed' = ANY(p.flags)))" +FLAG_SQL = "(p.status = 'draft' AND (p.source_kind = 'hand' OR cardinality(p.flags) > 0))" +# A pending change order from the feed (VISUAL or M-Files): the "Changes to approve" board. +CO_PENDING_SQL = ( + "EXISTS (SELECT 1 FROM change_orders c WHERE c.packet_id = p.id AND c.review = 'pending'" + " AND c.origin IN ('visual', 'drawing'))" +) def policy_principal_for(conn: Any, p: deps.Principal) -> policy.Principal: @@ -378,6 +463,57 @@ def _unit(conn: Any, unit_id: int, *, seal: bool) -> dict[str, Any]: } +def _visual(conn: Any, row: Mapping[str, Any], *, seal: bool) -> dict[str, Any]: + """The packet's visual jsonb. On a VISUAL draft (outside seal mode) it gains `staged_rev`: the M-Files copy the + drawing job staged for this drawing ({rev, name}), or None. It changes without the packet changing, so the seal + leaves it out.""" + out = dict(row["visual"] or {}) + if seal or row["source_kind"] != "visual" or row["status"] != "draft": + return out + out["staged_rev"] = staged_rev(conn, row["drawing"]) + return out + + +def staged_rev(conn: Any, drawing: Any) -> dict[str, Any] | None: + lk = staged_lookup(conn, drawing) + if lk is None: + return None + return {"rev": lk["rev"], "name": lk["name"]} + + +def staged_lookup(conn: Any, drawing: Any) -> dict[str, Any] | None: + """The drawing_lookups row for a drawing number when it holds a fetched copy (asset and rev set).""" + dwg = str(drawing or "").strip() + if not dwg: + return None + return one( + conn, + "SELECT * FROM drawing_lookups WHERE dwg = %s AND asset_id IS NOT NULL AND rev IS NOT NULL", + (dwg,), + ) + + +def visual_state(conn: Any, row: Mapping[str, Any]) -> dict[str, Any] | None: + """What the feed last observed for a VISUAL packet: leg statuses, ticket hours, the chosen order, and the closed, + cancelled and gone times. None for a hand packet.""" + if row["source_kind"] != "visual": + return None + vs = one( + conn, + "SELECT legs, ticket_hours, order_ref, observed_at FROM packet_visual_state WHERE packet_id = %s", + (row["id"],), + ) + return { + "legs": (vs or {}).get("legs") or [], + "ticket_hours": (vs or {}).get("ticket_hours"), + "order_ref": (vs or {}).get("order_ref"), + "observed_at": (vs or {}).get("observed_at"), + "closed_at": row.get("visual_closed_at"), + "cancelled_at": row.get("visual_cancelled_at"), + "wo_gone_at": row.get("wo_gone_at"), + } + + def build( conn: Any, row: Mapping[str, Any], @@ -424,7 +560,7 @@ def build( "source": row["source"] or {}, "drawing_state": row["drawing_state"], "flags": list(row["flags"] or []), - "visual": row["visual"] or {}, + "visual": _visual(conn, row, seal=seal), "removed": None if row["removed_at"] is None else {"at": row["removed_at"], "by": people.ref(row["removed_by"]), "reason": row["removed_reason"]}, @@ -451,6 +587,7 @@ def build( "row_version", ) } + | {k: w.get(k) for k in ("dwg", "rev", "partspecs")} for w in ch.work_orders ] out["pages"] = [ @@ -513,7 +650,9 @@ def build( ) out["checklist_items"] = checklists.checklist_for(weld, row["drawing"], row["rev"], row["wps"]) out["qc_items"] = checklists.qc_items_for(row["drawing"], row["rev"], row["wps"], row["welder_stamp"]) - out["flag"] = flag_of(row["status"], row["source_kind"], row["flags"], row["drawing_state"]) + out["flag"] = flag_of(row["status"], row["source_kind"], row["flags"], row["drawing_state"], row["visual"]) + if not seal: + out["visual_state"] = visual_state(conn, row) if pp is not None: allowed, blocked = policy.can(pp, state) out["can"] = [str(a) for a in allowed] @@ -599,8 +738,12 @@ def _co(c: Mapping[str, Any], ch: Children, people: People) -> dict[str, Any]: "rejected_at", "rejected_reason", "row_version", + "vis", + "waiting", + "staged", + "target_fp", ) - out = {k: c[k] for k in keys} + out = {k: c.get(k) for k in keys} out["issued_by"] = people.ref(c["issued_by"]) out["approved_by"] = people.ref(c["approved_by"]) out["rejected_by"] = people.ref(c["rejected_by"]) @@ -641,7 +784,10 @@ SELECT p.id, p.version, p.unit_id, u.lot, pr.number || u.lot AS unit_number, pr. st.name AS station_name, p.welder_id, wu.display_name AS welder_name, p.welder_stamp, p.routed_to::text AS routed_to, p.vclosed_at, p.vclosed_by, vu.display_name AS vclosed_name, p.source_kind::text AS source_kind, p.flags, p.drawing_state::text AS drawing_state, p.wh_skipped, - p.heat_map, p.removed_at, coalesce(u.due, pr.due) AS due_effective, p.updated_at, + p.heat_map, p.removed_at, coalesce(u.due, pr.due) AS due_effective, p.updated_at, p.visual, + p.visual_closed_at, p.visual_cancelled_at, p.wo_gone_at, + EXISTS (SELECT 1 FROM change_orders c WHERE c.packet_id = p.id AND c.review = 'pending' + AND c.origin IN ('visual', 'drawing')) AS co_pending, (SELECT w.stage::text FROM packet_wh w WHERE w.packet_id = p.id AND w.closed_at IS NULL) AS wh_stage, (SELECT count(*) FROM bom_shorts s WHERE s.packet_id = p.id AND s.closed_at IS NULL) AS open_shorts, (SELECT count(*) FROM packet_work_orders wo WHERE wo.packet_id = p.id) AS wo_count, @@ -697,7 +843,8 @@ def card_state(r: Mapping[str, Any], names: Mapping[str, str]) -> policy.PacketS def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[str, Any]) -> list[dict[str, Any]]: - """The board cards. Filters: status (comma list), dept, unit_id, project_id, q, flag, limit, removed.""" + """The board cards. Filters: status (comma list), dept, unit_id, project_id, q, flag (1|0|hand|hand_upload| + ), co (pending), limit, removed.""" where: list[str] = [] params: list[Any] = [] if not filters.get("removed"): @@ -721,7 +868,24 @@ def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[s elif flag in FLAG_OFF: where.append("NOT " + FLAG_SQL) elif flag == "hand": - where.append("(p.status = 'draft' AND p.source_kind = 'hand')") + # Made by hand, or a VISUAL draft with a drawing or BOM uploaded by hand. + where.append("(p.status = 'draft' AND (p.source_kind = 'hand' OR p.flags && %s::text[]))") + params.append(list(HAND_UPLOAD_FLAGS)) + elif flag in ("hand_upload", "hand_uploads"): + where.append("(p.status = 'draft' AND p.flags && %s::text[])") + params.append(list(HAND_UPLOAD_FLAGS)) + elif flag and RX_FLAG_KIND.match(flag): + # One flag kind on the Flagged board (drafts). A visual draft whose flags are empty but whose drawing state + # still names the problem counts too, the same way flag_of reads it. + states = [st for st, k in STATE_KINDS.items() if k == flag] + where.append( + "(p.status = 'draft' AND (%s = ANY(p.flags) OR (cardinality(p.flags) = 0 AND p.source_kind = 'visual'" + " AND p.drawing_state::text = ANY(%s))))" + ) + params += [flag, states] + co = str(filters.get("co") or "").strip().lower() + if co == "pending": + where.append(CO_PENDING_SQL) for word in str(filters.get("q") or "").split()[:8]: pat = _like(word) where.append( @@ -771,7 +935,11 @@ def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[s else {"id": str(r["vclosed_by"]), "display_name": r["vclosed_name"]}, }, "source_kind": r["source_kind"], - "flag": flag_of(r["status"], r["source_kind"], r["flags"], r["drawing_state"]), + "flag": flag_of(r["status"], r["source_kind"], r["flags"], r["drawing_state"], r["visual"]), + "flag_kinds": list(r["flags"] or []), + "co_pending": bool(r["co_pending"]), + "visual_closed": r["visual_closed_at"] is not None, + "drawing_state": r["drawing_state"], "open_cos": [c["co_no"] for c in r["open_co_rows"] or []], "open_shorts": int(r["open_shorts"] or 0), "wh_stage": r["wh_stage"], diff --git a/server/packetstation/domain/projects.py b/server/packetstation/domain/projects.py index 1963b08..ff18c62 100644 --- a/server/packetstation/domain/projects.py +++ b/server/packetstation/domain/projects.py @@ -10,7 +10,7 @@ from __future__ import annotations import re from collections.abc import Iterable, Mapping from dataclasses import dataclass -from datetime import date +from datetime import UTC, date, datetime from typing import Any from psycopg import errors as pgerr @@ -28,8 +28,11 @@ MAX_UNITS_AT_ONCE = 200 MAX_WORDS = 8 LIST_LIMIT = 500 -PROJECT_FIELDS = ("name", "customer", "po", "due", "status", "notes") -UNIT_FIELDS = ("name", "due", "status", "notes") +PROJECT_FIELDS = ("name", "customer", "po", "due", "status", "notes", "order_pin", "order_pin_reason") +UNIT_FIELDS = ("name", "customer", "po", "due", "status", "notes", "order_pin", "order_pin_reason") +# Order-derived fields: the value in use, where it came from (visual | hand) and what VISUAL last said. +ORDER_FIELDS = ("customer", "po", "order_date", "ship_date") +RX_ORDER_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ./_-]{0,39}$") _DIGITS = re.compile(r"^[0-9]+$") _ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") @@ -159,6 +162,28 @@ def _removed(r: Mapping[str, Any]) -> dict[str, Any] | None: # ---------------------------------------------------------------- shapes +def _order_out(r: Mapping[str, Any]) -> dict[str, Any]: + """The order-derived columns with their provenance, and the order pin. Shared by projects and units.""" + pin_by = r.get("order_pin_by") + return { + "customer_src": r.get("customer_src"), + "customer_visual": r.get("customer_visual"), + "po_src": r.get("po_src"), + "po_visual": r.get("po_visual"), + "order_date": r.get("order_date"), + "order_date_src": r.get("order_date_src"), + "order_date_visual": r.get("order_date_visual"), + "ship_date": r.get("ship_date"), + "ship_date_src": r.get("ship_date_src"), + "ship_date_visual": r.get("ship_date_visual"), + "order_ref": r.get("order_ref"), + "order_pin": r.get("order_pin"), + "order_pin_by": str(pin_by) if pin_by else None, + "order_pin_at": r.get("order_pin_at"), + "order_pin_reason": r.get("order_pin_reason"), + } + + def project_out(r: Mapping[str, Any]) -> dict[str, Any]: return { "id": r["id"], @@ -171,6 +196,7 @@ def project_out(r: Mapping[str, Any]) -> dict[str, Any]: "notes": r["notes"], "source": r["source"], "needs_details": r["needs_details"], + **_order_out(r), "units": int(r.get("unit_count") or 0), "packets": _counts(r), "version": r["version"], @@ -193,6 +219,9 @@ def unit_out(r: Mapping[str, Any]) -> dict[str, Any]: "status": status, "status_effective": "closed" if r["project_status"] == "closed" else status, "notes": r["notes"], + "customer": r.get("customer"), + "po": r.get("po"), + **_order_out(r), "packets": _counts(r), "version": r["version"], "created_at": r["created_at"], @@ -322,6 +351,7 @@ def project_detail(conn: Any, project_id: int) -> dict[str, Any]: out["unit_count"] = len(rows) out["aliases"] = aliases_of(conn, project_id=project_id) out["next_lot"] = next_lot(conn, project_id, out["number"]) + out["orders"] = cached_orders(conn, out["number"], _project_bases(conn, project_id)) return out @@ -370,6 +400,7 @@ def unit_detail(conn: Any, unit_id: int) -> dict[str, Any]: raise ApiError(404, "not_found", "No such unit.") out["project"] = {k: p[k] for k in ("id", "number", "name", "customer", "po", "due", "status", "version")} out["aliases"] = aliases_of(conn, unit_id=unit_id) + out["orders"] = cached_orders(conn, p["number"], _unit_bases(conn, unit_id)) return out @@ -479,6 +510,13 @@ def _norm_field(key: str, value: Any, *, unit: bool) -> Any: return clean_opt(value, LIMITS["po"]) if key == "notes": return clean_opt(value, LIMITS["notes"]) + if key == "order_pin": + pin = clean_opt(value, LIMITS["order_pin"] + 20) + if pin is not None and not RX_ORDER_ID.match(pin): + raise ApiError(422, "invalid_order", "Give a VISUAL customer order ID, or nothing to clear the pin.") + return pin + if key == "order_pin_reason": + return clean_opt(value, LIMITS["reason"]) raise ApiError(422, "invalid_field", f"{key} cannot be changed here.") @@ -515,6 +553,100 @@ def _change_detail(row: Mapping[str, Any], changes: Mapping[str, Any]) -> dict[s return out +def _columns(row: Mapping[str, Any], changes: Mapping[str, Any], who: Who) -> dict[str, Any]: + """The columns a field edit writes: a hand customer or po marks its source, a pin records who and when.""" + cols = dict(changes) + for k in ("customer", "po"): + if k in cols: + cols[f"{k}_src"] = "hand" if cols[k] else None + if "order_pin" in cols: + if cols["order_pin"] is None: + cols.update(order_pin_by=None, order_pin_at=None, order_pin_reason=None) + else: + cols["order_pin_by"] = who.user_id + cols["order_pin_at"] = datetime.now(UTC) + return cols + + +def _check_pin( + conn: Any, row: Mapping[str, Any], changes: Mapping[str, Any], *, project_number: str, bases: list[str] +) -> None: + """A new pin needs a reason and, when VISUAL's orders are cached for this project, must name one of them.""" + if "order_pin" not in changes and "order_pin_reason" in changes and row.get("order_pin") is None: + raise ApiError(422, "invalid_field", "There is no order pin to give a reason for.") + pin = changes.get("order_pin") + if pin is None: + return + if not changes.get("order_pin_reason"): + raise ApiError(400, "reason_required", "Say why this order is pinned.") + known = [o["cust_order_id"] for o in cached_orders(conn, project_number, bases)] + if known and pin.upper() not in {k.upper() for k in known}: + raise ApiError( + 422, + "unknown_order", + f"Order {pin} is not one of the VISUAL orders for {project_number}.", + extra={"orders": known}, + ) + + +def _project_bases(conn: Any, project_id: int) -> list[str]: + return [ + r[0] + for r in conn.execute( + "SELECT DISTINCT wo.wo_base FROM packet_work_orders wo JOIN packets k ON k.id = wo.packet_id" + " JOIN units u ON u.id = k.unit_id WHERE u.project_id = %s ORDER BY 1", + (project_id,), + ) + ] + + +def _unit_bases(conn: Any, unit_id: int) -> list[str]: + return [ + r[0] + for r in conn.execute( + "SELECT DISTINCT wo.wo_base FROM packet_work_orders wo JOIN packets k ON k.id = wo.packet_id" + " WHERE k.unit_id = %s ORDER BY 1", + (unit_id,), + ) + ] + + +def cached_orders(conn: Any, project_number: str, bases: Iterable[str]) -> list[dict[str, Any]]: + """Every cached VISUAL order of a project: the ones linked to the project number, then the ones linked to its + work order bases. Each once, with `via` (project | base) and the link's lines and top jobs.""" + with conn.cursor(row_factory=dict_row) as cur: + rows = cur.execute( + "SELECT o.*, l.kind AS via, l.key, l.rank, l.lines, l.top_jobs FROM visual_order_links l" + " JOIN visual_orders o ON o.cust_order_id = l.cust_order_id" + " WHERE (l.kind = 'project' AND upper(l.key) = upper(%s)) OR (l.kind = 'base' AND l.key = ANY(%s))" + " ORDER BY CASE l.kind WHEN 'project' THEN 0 ELSE 1 END, l.rank, l.key, o.cust_order_id", + (project_number, list(bases)), + ).fetchall() + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for r in rows: + if r["cust_order_id"] in seen: + continue + seen.add(r["cust_order_id"]) + out.append( + { + "cust_order_id": r["cust_order_id"], + "status": r["status"], + "customer_id": r["customer_id"], + "customer_name": r["customer_name"], + "customer_po_ref": r["customer_po_ref"], + "order_date": r["order_date"], + "desired_ship_date": r["desired_ship_date"], + "promise_date": r["promise_date"], + "via": r["via"], + "key": r["key"], + "lines": list(r["lines"] or []), + "top_jobs": list(r["top_jobs"] or []), + } + ) + return out + + # ---------------------------------------------------------------- writes: projects @@ -567,11 +699,8 @@ def update_project( row = _lock_project(conn, project_id) changes = field_cas(row, values, base, PROJECT_FIELDS, unit=False) if changes: - cols = dict(changes) - if "customer" in cols: - cols["customer_src"] = "hand" if cols["customer"] else None - if "po" in cols: - cols["po_src"] = "hand" if cols["po"] else None + _check_pin(conn, row, changes, project_number=row["number"], bases=_project_bases(conn, project_id)) + cols = _columns(row, changes, who) sets = ", ".join(f"{k} = %s" for k in cols) conn.execute( f"UPDATE projects SET {sets}, version = version + 1, updated_at = now() WHERE id = %s", # noqa: S608 @@ -800,10 +929,12 @@ def update_unit( row, project = _lock_unit(conn, unit_id) changes = field_cas(row, values, base, UNIT_FIELDS, unit=True) if changes: - sets = ", ".join(f"{k} = %s" for k in changes) + _check_pin(conn, row, changes, project_number=project["number"], bases=_unit_bases(conn, unit_id)) + cols = _columns(row, changes, who) + sets = ", ".join(f"{k} = %s" for k in cols) conn.execute( f"UPDATE units SET {sets}, version = version + 1, updated_at = now() WHERE id = %s", # noqa: S608 - [*changes.values(), unit_id], + [*cols.values(), unit_id], ) _audit( conn, diff --git a/server/packetstation/drawings/__init__.py b/server/packetstation/drawings/__init__.py new file mode 100644 index 0000000..aa3f50d --- /dev/null +++ b/server/packetstation/drawings/__init__.py @@ -0,0 +1,17 @@ +"""Drawings from M-Files, read-only: the lookup, the decision table, the queue and the fetch into assets. + +Packet Station never writes to M-Files. MFilesKit enforces that in its own code; this package only calls its +read methods (`DrawingLookup.status`, `DrawingLookup.download`). +""" + +from __future__ import annotations + +# The packet flags this package owns. The drawing job replaces only these and keeps every other flag. +DRAWING_FLAGS: tuple[str, ...] = ( + "drawing_needed", + "rev_mismatch", + "not_released", + "not_found", + "lookup_failed", + "bad_rev", +) diff --git a/server/packetstation/drawings/decide.py b/server/packetstation/drawings/decide.py new file mode 100644 index 0000000..51ab327 --- /dev/null +++ b/server/packetstation/drawings/decide.py @@ -0,0 +1,190 @@ +"""The drawing decision table. Pure: no database, no M-Files. + +`decide(target, found, same_file=...)` says what the drawing job does to one packet for one lookup. Note texts are +the app's own (from the old revMismatch and draftLine in visual.js, "-" in place of the em-dash), never M-Files' +note passed through, except inside the lookup_failed text. + +Rows, in the order they are tried: + 1. lookup failed -> flag lookup_failed, keep what is attached (state lookup_failed only when it was needed). + 2. past draft, a pending feed change order waits for this drawing at the released rev -> stage_for_co. + Past draft, the pending change order already stages this drawing at the released rev -> unchanged, or + stage_for_co again when M-Files has a different file for that rev. + 3. no released rev, a draft rev or a drafting assignment -> flag not_released with the drafting status. + 4. no released rev otherwise -> flag not_found. + 5. VISUAL rev not a number (and no approved rev equal to M-Files') -> flag bad_rev, not attached. + 6. revs differ (and no approved rev equal to M-Files') -> flag rev_mismatch. The job stages the M-Files copy + for Clerical Supervisor approval on drafts only. + 7. revs match, draft with a hand drawing -> unchanged (the feed already took the hand drawing off on a rev move). + 8. revs match, draft with this very file attached at this rev -> unchanged. + 9. revs match, draft -> attach ("sheet of drawing " for an H sheet). +10. revs match, past draft, nothing attached -> unchanged. +11. revs match, past draft, same file as attached -> unchanged. +12. revs match, past draft, a pending change order already stages this file, or Clerical rejected a change order + for this file -> unchanged. +13. revs match, past draft, attached rev lower -> open_co "M-Files released rev N of ; this packet has rev M". +14. revs match, past draft, attached rev equal, new file -> open_co "M-Files re-released rev N of (new file)". +15. revs match, past draft, attached rev higher or not a number -> unchanged. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .lookup import Found + +LOOKUP_FAILED = "lookup_failed" +NOT_RELEASED = "not_released" +NOT_FOUND = "not_found" +BAD_REV = "bad_rev" +REV_MISMATCH = "rev_mismatch" +ATTACH = "attach" +UNCHANGED = "unchanged" +OPEN_CO = "open_co" +STAGE_FOR_CO = "stage_for_co" + +KINDS = (ATTACH, UNCHANGED, REV_MISMATCH, NOT_RELEASED, NOT_FOUND, LOOKUP_FAILED, BAD_REV, OPEN_CO, STAGE_FOR_CO) +FLAG_KINDS = (LOOKUP_FAILED, NOT_RELEASED, NOT_FOUND, BAD_REV, REV_MISMATCH) + +# drawing_state enum values (0003). bad_rev has no state of its own: it is a rev mismatch Clerical can approve. +STATE_OF = {NOT_RELEASED: "not_released", NOT_FOUND: "not_found", BAD_REV: "rev_mismatch", REV_MISMATCH: "rev_mismatch"} +HELD_STATES = ("attached", "hand") + +NOT_FOUND_NOTE = "Drawing not in M-Files - check the drawing number in VISUAL" +_FAILED_PREFIX = "M-Files lookup failed: " + + +@dataclass(frozen=True) +class Target: + dwg: str + visual_rev: str + attached_rev: str | None # rev of the live drawing pages; None when no drawing page is attached + attached_sha: bytes | None + status: str # packet status + drawing_state: str + has_hand_drawing: bool + rev_ok: str | None # the rev Clerical approved for this VISUAL rev, if any + # Additions to the contract, all defaulted: the pending feed change order (origin visual or drawing), if any. + co_waiting: tuple[str, str] | None = None # (dwg, rev) the pending CO waits for + co_staged: tuple[str, str] | None = None # (dwg, rev) the pending CO already stages + co_pending: bool = False + co_staged_sha: bytes | None = None # sha256 of the file the pending CO already stages + rejected_sha: bytes | None = None # sha256 staged on the last drawing-origin CO Clerical rejected + + +@dataclass(frozen=True) +class Decision: + kind: str + flag: str | None + drawing_state: str + note: str + + +def _int(v: object) -> int | None: + s = "" if v is None else str(v).strip() + return int(s) if s.isdigit() else None + + +def draft_line(f: Found) -> str: + """The old draftLine: "rev N ", or "in drafting, no released PDF yet", or "".""" + if f.draft_rev is not None and f.draft_where: + return f"rev {f.draft_rev} {f.draft_where}" + if f.in_drafting: + return "in drafting, no released PDF yet" + if f.draft_rev is not None: + return f"rev {f.draft_rev}" + return "" + + +def failed_note(f: Found) -> str: + detail = (f.note or "").strip() + if detail.startswith(_FAILED_PREFIX): # MFilesKit's own note already carries the prefix + detail = detail[len(_FAILED_PREFIX) :] + return f"{_FAILED_PREFIX}{detail or 'no detail'}" + + +def rev_confirmed(t: Target, f: Found) -> bool: + """The old revConfirmed: M-Files' released rev is the rev VISUAL calls, or Clerical approved that rev.""" + if not f.ok or f.rev is None: + return False + v = _int(t.visual_rev) + if v is not None and v == f.rev: + return True + return t.rev_ok is not None and str(t.rev_ok).strip() == str(f.rev) + + +def decide(t: Target, f: Found, *, same_file: bool, fetched_sha: bytes | None = None) -> Decision: + """What to do to one packet. + + `same_file`: the M-Files file has the sha256 of the attached drawing asset. `fetched_sha`: the sha256 of the + M-Files file, when it was fetched (row 12 compares it with the pending and the rejected change orders). + """ + held = t.drawing_state in HELD_STATES + draft = t.status == "draft" + + def flag(kind: str, note: str) -> Decision: + return Decision(kind, kind, t.drawing_state if held else STATE_OF[kind], note) + + def keep(kind: str, note: str = "") -> Decision: + return Decision(kind, None, t.drawing_state, note) + + # 1 + if not f.ok: + return Decision( + LOOKUP_FAILED, + LOOKUP_FAILED, + LOOKUP_FAILED if t.drawing_state == "needed" else t.drawing_state, + failed_note(f), + ) + # 2 + if not draft and f.rev is not None and t.co_waiting is not None: + w_dwg, w_rev = t.co_waiting + if w_dwg.strip() == t.dwg.strip() and str(w_rev).strip() == str(f.rev): + return keep(STAGE_FOR_CO, f"Drawing for the change order is in M-Files (rev {f.rev})") + if not draft and f.rev is not None and t.co_staged is not None: + s_dwg, s_rev = t.co_staged + if s_dwg.strip() == t.dwg.strip() and str(s_rev).strip() == str(f.rev): + if fetched_sha is not None and t.co_staged_sha is not None and fetched_sha != t.co_staged_sha: + return keep(STAGE_FOR_CO, f"New file for rev {f.rev} in M-Files; staged again for the change order") + return keep(UNCHANGED, f"Rev {f.rev} is staged on the pending change order") + # 3, 4 + if f.rev is None: + d = draft_line(f) + if d: + on = f" (sheet of {f.base})" if f.base else "" + return flag(NOT_RELEASED, f"Not released in M-Files yet{on} - drafting status: {d}") + return flag(NOT_FOUND, NOT_FOUND_NOTE) + approved = t.rev_ok is not None and str(t.rev_ok).strip() == str(f.rev) + vrev = (t.visual_rev or "").strip() + v = _int(vrev) + if not approved: + # 5 + if v is None: + return flag(BAD_REV, f"VISUAL rev {vrev or '-'} is not a number; M-Files has rev {f.rev} - not attached") + # 6 + if v != f.rev: + d = f"; drafting status: {draft_line(f)}" if f.draft_rev is not None and f.draft_rev > f.rev else "" + return flag(REV_MISMATCH, f"Rev mismatch: VISUAL calls rev {vrev}, M-Files has rev {f.rev}{d}") + sheet = f"sheet of drawing {f.base}" if f.base else "" + if draft: + # 7 + if t.has_hand_drawing: + return keep(UNCHANGED, "A drawing uploaded by hand stays at the same rev") + # 8 + if t.drawing_state == "attached" and same_file and t.attached_rev == str(f.rev): + return keep(UNCHANGED, sheet) + # 9 + return Decision(ATTACH, None, "attached", sheet) + # 10, 11 + if t.attached_rev is None or same_file: + return keep(UNCHANGED, sheet) + # 12 + if fetched_sha is not None and fetched_sha in (t.co_staged_sha, t.rejected_sha): + return keep(UNCHANGED, sheet) + att = _int(t.attached_rev) + if att is None or att > f.rev: + return keep(UNCHANGED, sheet) # 15 + if att < f.rev: + reason = f"M-Files released rev {f.rev} of {t.dwg}; this packet has rev {att}" + else: + reason = f"M-Files re-released rev {f.rev} of {t.dwg} (new file)" + return keep(OPEN_CO, reason) diff --git a/server/packetstation/drawings/fake.py b/server/packetstation/drawings/fake.py new file mode 100644 index 0000000..f99e589 --- /dev/null +++ b/server/packetstation/drawings/fake.py @@ -0,0 +1,118 @@ +"""The fixture lookup for CI and tests: `dir/drawings..json` and the PDFs it names. + +`dir/state` holds the state number (default "1"). The JSON maps a drawing number to +`{"rev": 3, "file": "L073762X r3.pdf", "draft_rev": null, "draft_where": "", "in_drafting": false, "ok": true, +"note": "", "obj_id": 1, "obj_version": 1, "file_id": 1, "file_version": 1}`; every key is optional. A drawing +not in the file is not found, after the same H-sheet fallback MFilesKit uses (K978-01-6H13 -> K978-01-6). +`download` copies `dir/`. +""" + +from __future__ import annotations + +import json +import re +import shutil +from pathlib import Path +from typing import Any + +from .lookup import DrawingFetchError, Found, failed + +NOT_FOUND_NOTE = "not in M-Files - check the drawing number in VISUAL" +_H_SHEET = re.compile(r"^(.*\d)H\d{1,3}$") + + +def base_drawing(dwg: str) -> str | None: + m = _H_SHEET.match(dwg) + return m.group(1) if m else None + + +def _int(v: Any) -> int | None: + if v is None or isinstance(v, bool): + return None + try: + return int(str(v).strip()) + except ValueError: + return None + + +class FixtureLookup: + def __init__(self, directory: Path, state: str | None = None) -> None: + self.dir = Path(directory) + if state is None: + try: + state = (self.dir / "state").read_text(encoding="utf-8").strip() or "1" + except OSError: + state = "1" + self.state = state + self._data: dict[str, dict[str, Any]] | None = None + self._error: str | None = None + + def _load(self) -> dict[str, dict[str, Any]]: + if self._data is None: + path = self.dir / f"drawings.{self.state}.json" + try: + raw = json.loads(path.read_text(encoding="utf-8")) + self._data = {str(k).strip(): v for k, v in raw.items() if isinstance(v, dict)} + except (OSError, ValueError, AttributeError) as e: + self._error = f"fixture {path.name} could not be read: {type(e).__name__}" + self._data = {} + return self._data + + def status(self, dwg: str) -> Found: + dwg = (dwg or "").strip() + data = self._load() + if self._error: + return failed(dwg, self._error) + if not dwg: + return Found(dwg="", note="no drawing number") + entry, base = data.get(dwg), None + if entry is None: + b = base_drawing(dwg) + if b and b in data: + entry, base = data[b], b + if entry is None: + return Found(dwg=dwg, note=NOT_FOUND_NOTE) + if entry.get("ok") is False: + return failed(dwg, str(entry.get("note") or "M-Files lookup failed: fixture says so")) + rev = _int(entry.get("rev")) + file = entry.get("file") or None + size = None + if file and rev is not None: + p = self._file(str(file)) + size = p.stat().st_size if p is not None and p.is_file() else None + has_file = rev is not None and bool(file) + return Found( + dwg=dwg, + ok=True, + rev=rev, + name=str(entry.get("name") or file or "") or None if rev is not None else None, + obj_id=_int(entry.get("obj_id", 1)) if has_file else None, + obj_version=_int(entry.get("obj_version", 1)) if has_file else None, + file_id=_int(entry.get("file_id", 1)) if has_file else None, + file_version=_int(entry.get("file_version", 1)) if has_file else None, + file_size=size, + base=base, + draft_rev=_int(entry.get("draft_rev")), + draft_where=str(entry.get("draft_where") or ""), + in_drafting=bool(entry.get("in_drafting")), + note=str(entry.get("note") or ""), + ) + + def _file(self, name: str) -> Path | None: + root = self.dir.resolve() + p = (root / name).resolve() + return p if p.is_relative_to(root) else None + + def download(self, found: Found, dest: Path) -> int: + entry = self._load().get(found.base or found.dwg) or self._load().get(found.dwg) or {} + name = entry.get("file") + src = self._file(str(name)) if name else None + if src is None or not src.is_file(): + raise DrawingFetchError(f"{found.dwg}: fixture file {name!r} is missing") + dest = Path(dest) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dest) + return dest.stat().st_size + + def describe(self) -> str: + return f"fixture {self.dir} state {self.state}" diff --git a/server/packetstation/drawings/fetch.py b/server/packetstation/drawings/fetch.py new file mode 100644 index 0000000..bf797ec --- /dev/null +++ b/server/packetstation/drawings/fetch.py @@ -0,0 +1,125 @@ +"""Copy a released drawing PDF from M-Files into assets. + +Downloads only when the M-Files identity (object id and version, file id and version) differs from what the +`drawing_lookups` row last stored, or that row has no asset (or the asset's file is gone). The download goes to +ASSETS_DIR/tmp/; the PDF magic and the size limit are checked, then `assets.store_file` hashes it in chunks, +fsyncs and renames it into place, and reuses an existing asset row with the same sha256 and size. + +Read-only to M-Files: only `Lookup.download` is called. +""" + +from __future__ import annotations + +import contextlib +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from psycopg.rows import dict_row + +from ..api.errors import ApiError +from ..domain import assets +from .lookup import Found, Lookup + +PDF_MAGIC = b"%PDF-" + + +class FetchError(Exception): + """The drawing could not be copied. The message is safe to log.""" + + +@dataclass(frozen=True, slots=True) +class FetchResult: + asset_id: uuid.UUID + sha256: bytes + pages: int + downloaded: bool # False when the stored asset was still current + name: str | None = None + reused: bool = False # downloaded, but the same bytes were already stored + + +def _current(conn: Any, settings: Any, found: Found, dwg: str) -> FetchResult | None: + with conn.cursor(row_factory=dict_row) as cur: + row = cur.execute( + "SELECT l.obj_id, l.obj_version, l.file_id, l.file_version, l.asset_id, a.sha256, a.pages," + " a.storage_key, a.original_name FROM drawing_lookups l LEFT JOIN assets a ON a.id = l.asset_id" + " WHERE l.dwg = %s", + (dwg,), + ).fetchone() + if row is None or row["asset_id"] is None or row["sha256"] is None: + return None + if (row["obj_id"], row["obj_version"], row["file_id"], row["file_version"]) != found.identity: + return None + if None in found.identity: + return None + try: + if not assets.path_of(settings, str(row["storage_key"])).is_file(): + return None + except ApiError: + return None + return FetchResult( + asset_id=row["asset_id"], + sha256=bytes(row["sha256"]), + pages=int(row["pages"] or 0), + downloaded=False, + name=row["original_name"], + ) + + +def fetch(conn: Any, settings: Any, lookup: Lookup, found: Found, *, dwg: str) -> FetchResult: + """The asset for M-Files' released file of `dwg`. Raises FetchError.""" + if not found.ok or found.rev is None: + raise FetchError(f"{dwg}: no released file in M-Files") + cur = _current(conn, settings, found, dwg) + if cur is not None: + return cur + limit = int(settings.upload_max_bytes) + if found.file_size is not None and found.file_size > limit: + raise FetchError(f"{dwg}: the M-Files file is {found.file_size} bytes, over the {limit} byte limit") + tmp_dir = Path(settings.assets_dir) / "tmp" + tmp_dir.mkdir(parents=True, exist_ok=True) + tmp = tmp_dir / uuid.uuid4().hex + try: + try: + lookup.download(found, tmp) + except LookupError as e: + raise FetchError(str(e)) from None + except OSError as e: + raise FetchError(f"{dwg}: could not write the download: {type(e).__name__}") from None + try: + size = tmp.stat().st_size + with tmp.open("rb") as f: + head = f.read(len(PDF_MAGIC)) + except OSError: + raise FetchError(f"{dwg}: the download left no file") from None + if size > limit: + raise FetchError(f"{dwg}: the M-Files file is {size} bytes, over the {limit} byte limit") + if head != PDF_MAGIC: + raise FetchError(f"{dwg}: the M-Files file is not a PDF") + name = found.name or f"{dwg} r{found.rev}.pdf" + source_ref = { + "dwg": dwg, + "obj_id": found.obj_id, + "obj_version": found.obj_version, + "file_id": found.file_id, + "file_version": found.file_version, + "rev": found.rev, + } + try: + row = assets.store_file( + conn, settings, tmp, mime=assets.PDF, original_name=name, source="mfiles", source_ref=source_ref + ) + except ApiError as e: + raise FetchError(f"{dwg}: {e.detail}") from None + return FetchResult( + asset_id=row["id"], + sha256=bytes(row["sha256"]), + pages=int(row["pages"] or 0), + downloaded=True, + name=row.get("original_name") or name, + reused=bool(row.get("reused")), + ) + finally: + with contextlib.suppress(OSError): + tmp.unlink() diff --git a/server/packetstation/drawings/lookup.py b/server/packetstation/drawings/lookup.py new file mode 100644 index 0000000..dde8f4f --- /dev/null +++ b/server/packetstation/drawings/lookup.py @@ -0,0 +1,183 @@ +"""The drawing lookup the job talks to: M-Files through MFilesKit, or the fixture files (fake.py). + +One M-Files `Vault` per worker process (each sign-in can use a licence), kept in the dict the scheduler passes +(`JobContext.shared`) or, when there is none, in this module's own cache keyed by MFILES_URL. A new +`DrawingLookup` per run, because its result cache lives on the instance. + +Read-only. Nothing here writes to M-Files. +""" + +from __future__ import annotations + +import logging +import os +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from mfileskit.drawings import DrawingStatus + from mfileskit.vault import Vault + + from ..settings import Settings + +log = logging.getLogger(__name__) + +VAULT_KEY = "mfiles_vault" +NOTE_MAX = 500 + + +class DrawingFetchError(LookupError): + """A download did not work. The message is safe to log (no credentials).""" + + +@dataclass(slots=True) +class Found: + """What M-Files has for one drawing number: the subset the decision needs, plus identity for change detection.""" + + dwg: str + ok: bool = True + rev: int | None = None + name: str | None = None + obj_id: int | None = None + obj_version: int | None = None + file_id: int | None = None + file_version: int | None = None + file_size: int | None = None + base: str | None = None + draft_rev: int | None = None + draft_where: str = "" + in_drafting: bool = False + note: str = "" + + @property + def identity(self) -> tuple[int | None, int | None, int | None, int | None]: + return (self.obj_id, self.obj_version, self.file_id, self.file_version) + + +def failed(dwg: str, note: str) -> Found: + return Found(dwg=dwg, ok=False, note=note[:NOTE_MAX]) + + +class Lookup(Protocol): + def status(self, dwg: str) -> Found: ... + + def download(self, found: Found, dest: Path) -> int: + """Write the released PDF to `dest`; bytes written. Raises DrawingFetchError (a LookupError).""" + ... + + def describe(self) -> str: ... + + +def from_status(st: DrawingStatus) -> Found: + """MFilesKit's DrawingStatus -> Found.""" + ov, fi = st.obj_ver, st.file + return Found( + dwg=st.drawing, + ok=bool(st.ok), + rev=st.rev, + name=st.name, + obj_id=ov.id if ov is not None else None, + obj_version=ov.version if ov is not None else None, + file_id=fi.id if fi is not None else None, + file_version=fi.version if fi is not None else None, + file_size=fi.size if fi is not None else None, + base=st.base, + draft_rev=st.draft_rev, + draft_where=st.draft_where or "", + in_drafting=bool(st.in_drafting), + note=(st.note or "")[:NOTE_MAX], + ) + + +class MFilesLookup: + """M-Files through `mfileskit.drawings.DrawingLookup`. Read-only.""" + + def __init__(self, vault: Vault) -> None: + from mfileskit.drawings import DrawingLookup + + self.vault = vault + self._lookup = DrawingLookup(vault) # reads the vault schema; raises MFilesError when the vault is down + self._seen: dict[str, DrawingStatus] = {} + + def status(self, dwg: str) -> Found: + from mfileskit.errors import MFilesError + + dwg = (dwg or "").strip() + try: + st = self._lookup.status(dwg) + except MFilesError as e: # status() catches these itself; this is a second net + return failed(dwg, f"M-Files lookup failed: {e}") + self._seen[dwg] = st + return from_status(st) + + def download(self, found: Found, dest: Path) -> int: + from mfileskit.errors import MFilesError + + st = self._seen.get(found.dwg) + if st is None or st.obj_ver is None or st.file is None: + raise DrawingFetchError(f"{found.dwg}: no released file to download") + try: + return int(self._lookup.download(st, dest)) + except MFilesError as e: + raise DrawingFetchError(f"{found.dwg}: download failed: {e}"[:NOTE_MAX]) from None + + def describe(self) -> str: + url = getattr(getattr(self.vault, "settings", None), "url", "") or "" + return f"M-Files {url}".strip() + + +# ---------------------------------------------------------------- one vault per process + +_LOCK = threading.Lock() +_CACHE: dict[str, Any] = {} + + +def _cache_for(cache: dict[str, Any] | None) -> tuple[dict[str, Any], str]: + if cache is not None: + return cache, VAULT_KEY + return _CACHE, f"{VAULT_KEY}:{os.environ.get('MFILES_URL', '')}" + + +def vault_for(settings: Settings, cache: dict[str, Any] | None = None) -> Vault: + """The process's Vault, made once with `Vault.from_env()` (MFILES_* from the environment; the entrypoint + exports MFILES_PASSWORD from its secret file) and signed in.""" + from mfileskit.vault import Vault + + store, key = _cache_for(cache) + with _LOCK: + vault = store.get(key) + if vault is None: + vault = Vault.from_env().connect() + store[key] = vault + return vault + + +def forget_vault(cache: dict[str, Any] | None = None) -> None: + """Drop (and close) the cached Vault, so the next run signs in again.""" + store, key = _cache_for(cache) + with _LOCK: + vault = store.pop(key, None) + if vault is not None: + try: + vault.close() + except Exception as e: # closing a broken connection may fail; nothing to do + log.info("closing the M-Files vault failed: %s", type(e).__name__) + + +def make(settings: Settings, cache: dict[str, Any] | None = None) -> Lookup: + """The lookup for one run. `drawing_source = fixture` reads `fixture_dir`; otherwise M-Files. + + Raises when M-Files cannot be opened (the job then records a degraded run). A failed open drops the cached + Vault so the next run tries a fresh sign-in. + """ + if getattr(settings, "drawing_source", "mfiles") == "fixture": + from .fake import FixtureLookup + + return FixtureLookup(Path(settings.fixture_dir)) + try: + return MFilesLookup(vault_for(settings, cache)) + except Exception: + forget_vault(cache) + raise diff --git a/server/packetstation/drawings/queue.py b/server/packetstation/drawings/queue.py new file mode 100644 index 0000000..ced1fe8 --- /dev/null +++ b/server/packetstation/drawings/queue.py @@ -0,0 +1,113 @@ +"""Which drawings the drawing job looks up this run, in order. + +Over live packets made from VISUAL that are not filed, have a drawing number, and sit in a department in scope +(feed setting `departments`; empty = every department). One item per drawing number (trimmed), the best priority +winning: + +1. the VISUAL rev changed since the last lookup (`drawing_lookups.visual_rev_seen`), or a pending feed change + order waits for a drawing (the item is the CO's target drawing and rev; at most every 15 minutes); +2. drafts still needing a drawing (`drawing_state = 'needed'`, or never looked up), unless a failed fetch set a + later `next_check_at`; +3. failed lookups whose backoff ran out (`ok = false and next_check_at <= now`); +4. rechecks due (`checked_at` older than `drawing_recheck_hours`). + +A packet with a hand drawing joins only through (1) or (4) once the feed has taken the hand drawing off; while +`drawing_state = 'hand'` it is skipped. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any + +BACKOFF_FIRST = timedelta(minutes=15) +BACKOFF_MAX = timedelta(hours=24) +CO_RECHECK = timedelta(minutes=15) # a change order's target drawing is looked up at most this often + +QUEUE_SQL = """ +WITH own AS ( + SELECT p.id AS packet_id, btrim(p.drawing) AS dwg, coalesce(btrim(p.rev), '') AS visual_rev, NULL::bigint AS co_id, + CASE + WHEN d.dwg IS NOT NULL AND d.checked_at IS NOT NULL + AND d.visual_rev_seen IS DISTINCT FROM coalesce(btrim(p.rev), '') THEN 1 + WHEN p.status = 'draft' AND (d.dwg IS NULL OR p.drawing_state = 'needed') + AND (d.next_check_at IS NULL OR d.next_check_at <= %(now)s) THEN 2 + WHEN d.ok = false AND (d.next_check_at IS NULL OR d.next_check_at <= %(now)s) THEN 3 + WHEN d.checked_at IS NULL OR d.checked_at < %(recheck_before)s THEN 4 + END AS priority, + d.checked_at + FROM packets p + LEFT JOIN drawing_lookups d ON d.dwg = btrim(p.drawing) + WHERE p.removed_at IS NULL AND p.source_kind = 'visual' AND p.status <> 'filed' + AND coalesce(btrim(p.drawing), '') <> '' AND p.drawing_state <> 'hand' + AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[])) +), +co AS ( + SELECT p.id AS packet_id, btrim(c.waiting->>'dwg') AS dwg, coalesce(btrim(c.waiting->>'rev'), '') AS visual_rev, + c.id AS co_id, 1 AS priority, d.checked_at + FROM change_orders c + JOIN packets p ON p.id = c.packet_id + LEFT JOIN drawing_lookups d ON d.dwg = btrim(c.waiting->>'dwg') + WHERE c.review = 'pending' AND c.origin IN ('visual', 'drawing') AND c.waiting IS NOT NULL + AND jsonb_typeof(c.waiting) = 'object' AND coalesce(btrim(c.waiting->>'dwg'), '') <> '' + AND p.removed_at IS NULL AND p.status NOT IN ('draft', 'filed') + AND (d.checked_at IS NULL OR d.checked_at < %(co_before)s) + AND (d.ok IS NOT false OR d.next_check_at IS NULL OR d.next_check_at <= %(now)s) + AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[])) +), +items AS ( + SELECT * FROM own WHERE priority IS NOT NULL + UNION ALL + SELECT * FROM co +), +best AS ( + SELECT DISTINCT ON (dwg) packet_id, dwg, visual_rev, priority, co_id, checked_at + FROM items + ORDER BY dwg, priority, co_id NULLS LAST, packet_id +) +SELECT packet_id, dwg, visual_rev, priority, co_id +FROM best +ORDER BY priority, checked_at NULLS FIRST, dwg +""" + + +@dataclass(frozen=True, slots=True) +class QueueItem: + packet_id: int + dwg: str + visual_rev: str + priority: int + co_id: int | None + + +def backoff(fail_count: int) -> timedelta: + """15 minutes x 2^(fail_count - 1), at most 24 hours. fail_count counts this failure (1 = first).""" + n = max(0, int(fail_count) - 1) + if n >= 10: + return BACKOFF_MAX + return min(BACKOFF_FIRST * (2**n), BACKOFF_MAX) + + +def params(feed_settings: Any, now: datetime) -> dict[str, Any]: + hours = int(getattr(feed_settings, "drawing_recheck_hours", 4)) + depts = [str(d) for d in (getattr(feed_settings, "departments", None) or [])] + return { + "now": now, + "recheck_before": now - timedelta(hours=hours), + "co_before": now - CO_RECHECK, + "depts": depts, + } + + +def queue(conn: Any, feed_settings: Any, now: datetime, *, limit: int | None = None) -> list[QueueItem]: + """The ordered lookup list. `limit` cuts it (the job passes the budget).""" + sql = QUEUE_SQL + p = params(feed_settings, now) + if limit is not None: + sql += " LIMIT %(limit)s" + p["limit"] = max(0, int(limit)) + rows = conn.execute(sql, p).fetchall() + return [ + QueueItem(int(r[0]), str(r[1]), str(r[2] or ""), int(r[3]), None if r[4] is None else int(r[4])) for r in rows + ] diff --git a/server/packetstation/feed/__init__.py b/server/packetstation/feed/__init__.py new file mode 100644 index 0000000..6427c32 --- /dev/null +++ b/server/packetstation/feed/__init__.py @@ -0,0 +1 @@ +"""The VISUAL feed: settings, the read, the cache, candidates, fingerprints and the apply.""" diff --git a/server/packetstation/feed/apply.py b/server/packetstation/feed/apply.py new file mode 100644 index 0000000..bd31c71 --- /dev/null +++ b/server/packetstation/feed/apply.py @@ -0,0 +1,1173 @@ +"""The apply: VISUAL candidates onto projects, units and packets, in one transaction. + +Steps, in order (phase3-design "The apply"): projects and units with their order-derived fields; match candidates +to packets by visual_key; new drafts; drafts updated in place; change orders on packets past draft; drafts whose +work orders moved retired; work orders gone from VISUAL flagged after two misses; closed and cancelled observed; +auto-release; one `feed.run` audit row and a `feed` event. + +Every packet write goes through `writes.system_write` as the system principal, so the policy decides. A packet row +someone else holds is skipped this run (FOR UPDATE SKIP LOCKED) and counted in `skipped_busy`. A candidate that +fails is rolled back to its savepoint, counted in `errors` and the run is degraded; the others carry on. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable, Collection, Iterable, Mapping +from datetime import UTC, date, datetime +from decimal import Decimal +from typing import Any + +import psycopg.errors as pgerr +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb + +from .. import audit, events, policy +from ..domain import projects as projects_domain +from ..text import LIMITS, RX_PROJECT, clean_opt +from . import changes, materials +from . import orders as orders_mod +from .candidates import NO_DRAWING, Built, Candidate, project_of, unit_number +from .fingerprint import Fingerprint, bom_sig, classify, compute, drawing_key, merge +from .source import redact +from .types import Order, Row, WoKey +from .writes import Busy, Missing, Refused, SysWrite, packets_jsonable, system_write + +log = logging.getLogger(__name__) + +A = policy.Action + +COUNT_KEYS = ( + "wos_window", + "wos_tracked", + "rows", + "materials", + "orders", + "labor_legs", + "new_drafts", + "drafts_updated", + "cos_opened", + "cos_reaimed", + "cos_withdrawn", + "cos_unchanged", + "retired", + "wo_gone_flagged", + "wo_back", + "closed_observed", + "cancelled_flagged", + "ignored_golive", + "projects_created", + "units_created", + "auto_released", + "unmapped_resources", + "skipped_removed", + "skipped_closed", + "skipped_lots", + "skipped_busy", + "skipped_conflict", + "skipped_hand", + "refused", + "errors", + "suspect", +) + +# Every flag the drawing side sets. A drawing or rev change on a draft clears them and starts over. +DRAWING_FLAGS = ( + "drawing_needed", + "rev_mismatch", + "not_released", + "not_found", + "lookup_failed", + "bad_rev", + "no_drawing", + "hand_drawing", +) +FIELDS = orders_mod.FIELDS # customer, po, order_date, ship_date +TOP = 30 + +Counts = dict[str, Any] +LaborMap = Mapping[tuple[WoKey, int], Decimal] + + +def _now() -> datetime: + return datetime.now(UTC) + + +def _iso(v: Any) -> Any: + return v.isoformat() if isinstance(v, date | datetime) else v + + +def _top(d: Mapping[str, int]) -> dict[str, int]: + return dict(sorted(d.items(), key=lambda kv: (-kv[1], kv[0]))[:TOP]) + + +class _Run: + """The apply's working state: counts, errors and the context.""" + + def __init__(self, conn: Any, ctx: Any, batch: Any, feed_settings: Any, today: date) -> None: + self.conn = conn + self.ctx = ctx + self.batch = batch + self.fs = feed_settings + self.today = today + self.counts: Counts = dict.fromkeys(COUNT_KEYS, 0) + self.now = _now() + + @property + def batch_id(self) -> int | None: + return getattr(self.batch, "id", None) + + def error(self, what: str, e: BaseException | str) -> None: + msg = f"{what}: {redact(e)}" + log.warning("feed apply: %s", msg) + self.counts["errors"] += 1 + if self.ctx is None: + return + if callable(getattr(self.ctx, "error", None)): + self.ctx.error(msg) + return + self.ctx.degraded = True + errs = getattr(self.ctx, "errors", None) + if isinstance(errs, list) and len(errs) < 20: + errs.append(msg[:500]) + + def guarded(self, what: str, fn: Callable[..., Any], *args: Any) -> Any: + """Run fn(*args) in a savepoint. Busy and lock timeouts are skipped; refusals and errors counted.""" + try: + with self.conn.transaction(): + return fn(*args) + except (Busy, pgerr.LockNotAvailable): + self.counts["skipped_busy"] += 1 + except Missing: + pass + except Refused as e: + self.counts["refused"] += 1 + log.info("feed apply: %s refused: %s", what, e) + except Exception as e: + self.error(what, e) + return None + + +# ---------------------------------------------------------------- 1. projects and units + + +def _audit_system(conn: Any, action: str, subject_type: str, subject_id: Any, detail: Any) -> None: + audit.append( + conn, + action=action, + role_snapshot="system", + subject_type=subject_type, + subject_id=subject_id, + detail=packets_jsonable(detail), + ) + + +def _publish_project(conn: Any, project_id: int, unit_id: int | None, action: str) -> None: + payload: dict[str, Any] = {"event": "changed", "kind": "project", "project_id": project_id, "action": action} + if unit_id is not None: + payload["unit_id"] = unit_id + events.publish(conn, payload) + + +def _one(conn: Any, sql: str, params: Any) -> dict[str, Any] | None: + with conn.cursor(row_factory=dict_row) as cur: + r = cur.execute(sql, params).fetchone() + return dict(r) if r else None + + +def _ensure_project(run: _Run, number: str, due: date | None) -> dict[str, Any] | None: + conn = run.conn + found = projects_domain.lookup(conn, number) + if found is not None and found["kind"] != "project": + run.counts["skipped_conflict"] += 1 + return None + if found is None: + conn.execute("SELECT pg_advisory_xact_lock(7312)") + found = projects_domain.lookup(conn, number) + if found is None: + pid = int( + conn.execute( + "INSERT INTO projects (number, name, source, needs_details, created_by, due)" + " VALUES (%s, NULL, 'visual', true, NULL, %s) RETURNING id", + (number, due), + ).fetchone()[0] + ) + conn.execute( + "INSERT INTO project_aliases (alias, project_id, unit_id, kind) VALUES (%s, %s, NULL, 'import')" + " ON CONFLICT (alias) DO NOTHING", + (number.upper(), pid), + ) + _audit_system(conn, "project.created", "project", pid, {"number": number, "source": "visual", "due": due}) + _publish_project(conn, pid, None, "project.created") + run.counts["projects_created"] += 1 + else: + pid = int(found["project_id"]) + return _one(conn, "SELECT * FROM projects WHERE id = %s", (pid,)) + + +def _ensure_unit(run: _Run, project: Mapping[str, Any], unit_number: str, lot: str, due: date | None) -> Any: + conn = run.conn + found = projects_domain.lookup(conn, unit_number) + if found is not None and found["kind"] != "unit": + run.counts["skipped_conflict"] += 1 + return None + if found is None: + conn.execute("SELECT pg_advisory_xact_lock(7312)") + found = projects_domain.lookup(conn, unit_number) + if found is None: + uid = int( + conn.execute( + "INSERT INTO units (project_id, lot, due, created_by) VALUES (%s, %s, %s, NULL) RETURNING id", + (project["id"], lot, due), + ).fetchone()[0] + ) + conn.execute( + "INSERT INTO project_aliases (alias, project_id, unit_id, kind) VALUES (%s, %s, %s, 'import')" + " ON CONFLICT (alias) DO NOTHING", + (unit_number.upper(), project["id"], uid), + ) + _audit_system( + conn, + "unit.created", + "unit", + uid, + {"number": unit_number, "project_id": project["id"], "lot": lot, "due": due, "source": "visual"}, + ) + _publish_project(conn, int(project["id"]), uid, "unit.created") + run.counts["units_created"] += 1 + else: + uid = int(found["id"]) + return _one(conn, "SELECT * FROM units WHERE id = %s", (uid,)) + + +def _derive(run: _Run, table: str, row: Mapping[str, Any], chosen: orders_mod.Chosen | None) -> dict[str, Any] | None: + """Write the order-derived fields with provenance. Returns the order_ref now on the row.""" + current_ref = row.get("order_ref") + if chosen is None: + return current_ref + values = orders_mod.derive(chosen.order) + limits = {"customer": LIMITS["customer"], "po": LIMITS["po"]} + sets: list[str] = [] + params: list[Any] = [] + changed: list[str] = [] + for f in FIELDS: + v = values[f] + if f in limits: + v = clean_opt(v, limits[f]) + if row.get(f"{f}_visual") != v: + sets.append(f"{f}_visual = %s") + params.append(v) + changed.append(f"{f}_visual") + src = row.get(f"{f}_src") + if src in (None, "visual") and (row.get(f) != v or (v is not None and src is None)): + sets.append(f"{f} = %s") + params.append(v) + if v is not None: + sets.append(f"{f}_src = 'visual'") + changed.append(f) + ref = orders_mod.order_ref(chosen, run.now) + if not orders_mod.same_ref(current_ref, ref): + sets.append("order_ref = %s") + params.append(Jsonb(ref)) + changed.append("order_ref") + else: + ref = current_ref or ref + if not sets: + return ref + sets.append("version = version + 1") + sets.append("updated_at = now()") + run.conn.execute( + f"UPDATE {table} SET {', '.join(sets)} WHERE id = %s", # noqa: S608 (fixed table and column names) + (*params, row["id"]), + ) + kind = "project" if table == "projects" else "unit" + if any(c != "order_ref" for c in changed): + _audit_system(run.conn, f"{kind}.updated", kind, row["id"], {"fields": changed, "src": "visual"}) + return ref + + +# ---------------------------------------------------------------- shapes for the packet rows + + +def _visual_json(c: Candidate, fp: Fingerprint) -> dict[str, Any]: + return { + "folded": changes.folded_json(c), + "partspecs": list(c.partspecs), + "fp_parts": dict(fp.parts), + "mat_known": c.bom is not None, + } + + +def _legs_state(c: Candidate) -> list[dict[str, Any]]: + return [{"key": str(lg.key), "seq": lg.seq, "st": c.statuses.get(lg.key, lg.st)} for lg in c.legs] + + +def _hours(labor: LaborMap | None, legs: Iterable[tuple[WoKey, int]]) -> Decimal | None: + if labor is None: + return None + total = Decimal(0) + seen = False + for k in legs: + h = labor.get(k) + if h is not None: + total += h + seen = True + return total if seen else Decimal(0) + + +def _insert_legs(conn: Any, packet_id: int, c: Candidate) -> None: + for sort, lg in enumerate(c.legs): + conn.execute( + "INSERT INTO packet_work_orders (packet_id, wo_base, lot, split, sub, seq, qty, target_hrs, op," + " description, visual_status, partspecs, dwg, rev, sort)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ( + packet_id, + lg.key.base, + lg.key.lot, + lg.key.split, + lg.key.sub, + lg.seq, + lg.qty, + lg.tgt, + clean_opt(lg.op, LIMITS["op"]), + clean_opt(c.title, LIMITS["description"]), + lg.st or None, + clean_opt(lg.partspecs, LIMITS["notes"]), + clean_opt(lg.dwg, LIMITS["drawing"]), + clean_opt(lg.rev, LIMITS["rev"]), + sort, + ), + ) + + +def _replace_legs(conn: Any, packet_id: int, c: Candidate) -> None: + """Keep rows whose key and seq still match (their ids and missing_runs), update their fields, drop the rest, + add the new ones.""" + with conn.cursor(row_factory=dict_row) as cur: + old = cur.execute("SELECT * FROM packet_work_orders WHERE packet_id = %s", (packet_id,)).fetchall() + by = {(r["wo_base"], r["lot"], r["split"], r["sub"], r["seq"]): r for r in old} + keep: set[int] = set() + for sort, lg in enumerate(c.legs): + r = by.get((lg.key.base, lg.key.lot, lg.key.split, lg.key.sub, lg.seq)) + vals = ( + lg.qty, + lg.tgt, + clean_opt(lg.op, LIMITS["op"]), + clean_opt(c.title, LIMITS["description"]), + lg.st or None, + clean_opt(lg.partspecs, LIMITS["notes"]), + clean_opt(lg.dwg, LIMITS["drawing"]), + clean_opt(lg.rev, LIMITS["rev"]), + sort, + ) + if r is not None: + keep.add(int(r["id"])) + conn.execute( + "UPDATE packet_work_orders SET qty = %s, target_hrs = %s, op = %s, description = %s," + " visual_status = %s, partspecs = %s, dwg = %s, rev = %s, sort = %s, row_version = row_version + 1" + " WHERE id = %s", + (*vals, r["id"]), + ) + gone = [int(r["id"]) for r in old if int(r["id"]) not in keep] + if gone: + conn.execute("DELETE FROM packet_work_orders WHERE id = ANY(%s)", (gone,)) + for sort, lg in enumerate(c.legs): + if (lg.key.base, lg.key.lot, lg.key.split, lg.key.sub, lg.seq) in by: + continue + conn.execute( + "INSERT INTO packet_work_orders (packet_id, wo_base, lot, split, sub, seq, qty, target_hrs, op," + " description, visual_status, partspecs, dwg, rev, sort)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ( + packet_id, + lg.key.base, + lg.key.lot, + lg.key.split, + lg.key.sub, + lg.seq, + lg.qty, + lg.tgt, + clean_opt(lg.op, LIMITS["op"]), + clean_opt(c.title, LIMITS["description"]), + lg.st or None, + clean_opt(lg.partspecs, LIMITS["notes"]), + clean_opt(lg.dwg, LIMITS["drawing"]), + clean_opt(lg.rev, LIMITS["rev"]), + sort, + ), + ) + + +def _visual_state(conn: Any, packet_id: int, legs: list[dict[str, Any]], hours: Decimal | None, ref: Any) -> None: + conn.execute( + "INSERT INTO packet_visual_state (packet_id, legs, ticket_hours, order_ref, observed_at)" + " VALUES (%s, %s, %s, %s, now()) ON CONFLICT (packet_id) DO UPDATE SET legs = EXCLUDED.legs," + " ticket_hours = coalesce(EXCLUDED.ticket_hours, packet_visual_state.ticket_hours)," + " order_ref = coalesce(EXCLUDED.order_ref, packet_visual_state.order_ref), observed_at = now()", + (packet_id, Jsonb(legs), hours, None if ref is None else Jsonb(packets_jsonable(ref))), + ) + + +# ---------------------------------------------------------------- 3. new drafts + + +def _create( + run: _Run, c: Candidate, unit: Mapping[str, Any], project: Mapping[str, Any], ref: Any, labor: LaborMap | None +) -> int: + conn = run.conn + fp = compute(c) + has_dwg = bool(c.drawing) + flags = ["drawing_needed"] if has_dwg else ["no_drawing"] + pid = int( + conn.execute( + "INSERT INTO packets (unit_id, dept_code, op, title, customer, drawing, rev, source_kind, source," + " visual_key, visual_rel, visual_want, drawing_state, flags, visual_fp_applied, visual, created_by)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, 'visual', %s, %s, %s, %s, %s::drawing_state, %s, %s, %s, NULL)" + " RETURNING id", + ( + unit["id"], + c.dept, + clean_opt(c.op, LIMITS["op"]), + clean_opt(c.title, LIMITS["title"]), + clean_opt(project.get("customer"), LIMITS["customer"]), + clean_opt(c.drawing, LIMITS["drawing"]), + clean_opt(c.rev, LIMITS["rev"]), + Jsonb({"kind": "visual", "batch": run.batch_id}), + c.key, + c.rel, + c.want, + "needed" if has_dwg else "none", + flags, + fp.full, + Jsonb(packets_jsonable(_visual_json(c, fp))), + ), + ).fetchone()[0] + ) + _insert_legs(conn, pid, c) + if c.bom is not None: + materials.insert_bom(conn, pid, c.bom) + _visual_state(conn, pid, _legs_state(c), _hours(labor, ((lg.key, lg.seq) for lg in c.legs)), ref) + n = len(c.legs) + note = f"{n} work order{'' if n == 1 else 's'} · {c.drawing or NO_DRAWING}" + (f" rev {c.rev}" if c.rev else "") + conn.execute( + "INSERT INTO packet_log (packet_id, by, by_name, role, action, note) VALUES (%s, NULL, 'VISUAL feed'," + " 'system', 'Packet created from VISUAL', %s)", + (pid, note), + ) + _audit_system_packet(conn, "packet.created", pid, {"visual_key": c.key, "dept": c.dept, "batch": run.batch_id}) + events.publish(conn, {"event": "changed", "packet_id": pid, "version": 1, "status": "draft", "dept": c.dept}) + run.counts["new_drafts"] += 1 + return pid + + +def _audit_system_packet(conn: Any, action: str, packet_id: int, detail: Any) -> None: + audit.append( + conn, + action=action, + role_snapshot="system", + subject_type="packet", + subject_id=packet_id, + packet_id=packet_id, + detail=packets_jsonable(detail), + ) + + +# ---------------------------------------------------------------- 4. drafts updated in place + + +def _update_draft(run: _Run, packet: Mapping[str, Any], c: Candidate) -> None: + visual = dict(packet.get("visual") or {}) + old_parts = visual.get("fp_parts") + fp = merge(old_parts, compute(c)) + if fp.full == packet.get("visual_fp_applied"): + return + with system_write(run.conn, int(packet["id"]), A.VIS_UPDATE_DRAFT) as sw: + conn = sw.conn + before = changes.packet_view(conn, sw.id) + diff = changes.diff_text(before, c) + row = sw.packet + visual = dict(row.get("visual") or {}) + _replace_legs(conn, sw.id, c) + sw.changed("work_orders") + if c.bom is not None and not visual.get("manual_bom") and bom_sig(changes.view_bom(before)) != bom_sig(c.bom): + conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (sw.id,)) + materials.insert_bom(conn, sw.id, c.bom) + sw.changed("bom") + moved = str(row.get("rev") or "") != str(c.rev or "") or drawing_key(row.get("drawing")) != drawing_key( + c.drawing + ) + flags = list(row.get("flags") or []) + state = str(row.get("drawing_state")) + if moved: + conn.execute("DELETE FROM packet_pages WHERE packet_id = %s AND type = 'drawing'", (sw.id,)) + for k in ("rev_ok", "manual_dwg", "vault", "appr"): + visual.pop(k, None) + visual["rev_confirmed"] = False + flags = [f for f in flags if f not in DRAWING_FLAGS] + flags.append("drawing_needed" if c.drawing else "no_drawing") + state = "needed" if c.drawing else "none" + sw.changed("drawing", "rev", "drawing_state", "flags", "pages") + visual.update(_visual_json(c, fp)) + conn.execute( + "UPDATE packets SET drawing = %s, rev = %s, drawing_state = %s::drawing_state, flags = %s," + " visual_fp_applied = %s, visual = %s, visual_rel = %s, visual_want = %s WHERE id = %s", + ( + clean_opt(c.drawing, LIMITS["drawing"]), + clean_opt(c.rev, LIMITS["rev"]), + state, + flags, + fp.full, + Jsonb(packets_jsonable(visual)), + c.rel, + c.want, + sw.id, + ), + ) + label = "Updated from VISUAL" + if moved and before.get("rev") and c.rev and str(before.get("rev")) != str(c.rev): + label = f"Updated from VISUAL: drawing rev {before.get('rev')} taken off, waiting on rev {c.rev}" + sw.log(label, "; ".join(diff) or None) + run.counts["drafts_updated"] += 1 + + +# ---------------------------------------------------------------- 5. change orders past draft + + +def _past_draft(run: _Run, packet: Mapping[str, Any], c: Candidate) -> None: + visual = dict(packet.get("visual") or {}) + old_parts = visual.get("fp_parts") + fp = merge(old_parts, compute(c)) + applied = packet.get("visual_fp_applied") + pending = changes.pending_visual(run.conn, int(packet["id"])) + if fp.full == applied and pending is None: + return + if pending is not None and pending.get("target_fp") == fp.full: + run.counts["cos_unchanged"] += 1 + return + with system_write(run.conn, int(packet["id"]), A.VIS_OPEN_CO) as sw: + conn = sw.conn + pend = changes.pending_visual(conn, sw.id) + if fp.full == applied: + # VISUAL went back to what the packet holds: withdraw a pending VISUAL change order. + if pend is not None and pend["origin"] == "visual": + changes.withdraw(sw, int(pend["id"]), "VISUAL went back to what the packet already holds.") + if not sw.noop: + run.counts["cos_withdrawn"] += 1 + else: + sw.skip() + return + change = classify(old_parts, fp.parts) + if not change.changed: + sw.skip() + return + last = changes.last_feed_co(conn, sw.id) + if ( + pend is None + and last is not None + and last.get("target_fp") == fp.full + and last["review"] + in ( + "rejected", + "approved", + ) + ): + sw.skip() # Clerical rejected exactly this, or already approved it + return + before = changes.packet_view(conn, sw.id) + diff = changes.diff_text(before, c) + vis = changes.vis_for(c, fp.parts, before, diff, fp.full) + waiting = ( + {"dwg": vis["to_dwg"], "rev": vis["to_rev"], "since": run.now.isoformat(timespec="seconds")} + if vis["dwg_change"] + else None + ) + res = changes.open_or_reaim( + sw, + origin="visual", + reason=changes.co_reason(vis), + vis=vis, + waiting=waiting, + staged=None, + target_fp=fp.full, + minor=change.is_minor, + ) + sw.detail["changes"] = diff[:20] + if res.fresh: + run.counts["cos_opened"] += 1 + elif res.reaimed: + run.counts["cos_reaimed"] += 1 + + +# ---------------------------------------------------------------- flag writes (steps 7 and 8) + + +def _flag_write( + run: _Run, packet_id: int, status: str, label: str, note: str | None, body: Callable[[SysWrite], bool] +) -> bool: + """A flag or observation change on a packet: VIS_UPDATE_DRAFT on a draft, VIS_OPEN_CO past draft (the only + system action open there). `body` returns False to write nothing.""" + action = A.VIS_UPDATE_DRAFT if status == "draft" else A.VIS_OPEN_CO + with system_write(run.conn, packet_id, action, label=label, note=note) as sw: + if not body(sw): + sw.skip() + return False + return True + + +def _set_flags( + conn: Any, packet_id: int, add: Collection[str] = (), remove: Collection[str] = (), extra_sql: str = "" +) -> None: + row = conn.execute("SELECT flags FROM packets WHERE id = %s", (packet_id,)).fetchone() + flags = [f for f in (row[0] or []) if f not in remove] + for f in add: + if f not in flags: + flags.append(f) + conn.execute( + f"UPDATE packets SET flags = %s{extra_sql} WHERE id = %s", # noqa: S608 (fixed fragments) + (flags, packet_id), + ) + + +# ---------------------------------------------------------------- the apply + + +def apply( + conn: Any, + *, + ctx: Any, + batch: Any, + feed_settings: Any, + built: Built, + orders: tuple[Mapping[str, list[Order]], Mapping[str, list[Order]]], + labor: LaborMap | None, + today: date, + seen: Mapping[WoKey, str] | None = None, + missing: Collection[WoKey] = (), + paused: bool | None = None, +) -> Counts: + """Apply one build. `seen` is every work order key this read returned with its status code (closed, + cancelled, back in VISUAL); `missing` the tracked keys VISUAL lacks (work order gone).""" + run = _Run(conn, ctx, batch, feed_settings, today) + suspect = bool(getattr(batch, "suspect", False)) + run.counts["suspect"] = int(suspect) + run.counts["unmapped_resources"] = sum(built.unmapped.values()) + if built.unmapped: + run.counts["unmapped"] = _top(built.unmapped) + run.counts["skipped_lots"] = sum(built.skipped_lots.values()) + if built.skipped_lots: + run.counts["skipped_lots_detail"] = _top(built.skipped_lots) + seen = dict(seen or {}) + with conn.transaction(): + conn.execute("SET LOCAL lock_timeout = '5s'") + _apply(run, built, orders, labor, seen, set(missing), suspect, paused) + _audit_system(conn, "feed.run", "job_run", getattr(ctx, "run_id", None), {"batch": run.batch_id, **run.counts}) + events.publish(conn, {"event": "feed", "run_id": getattr(ctx, "run_id", None), "batch": run.batch_id}) + return run.counts + + +def _apply( + run: _Run, + built: Built, + orders: tuple[Mapping[str, list[Order]], Mapping[str, list[Order]]], + labor: LaborMap | None, + seen: Mapping[WoKey, str], + missing: set[WoKey], + suspect: bool, + paused: bool | None, +) -> None: + conn = run.conn + by_base, by_project = orders + golive = getattr(run.fs, "golive_date", None) + depts = {r[0] for r in conn.execute("SELECT code FROM departments")} + + # Go-live boundary: rows the build left out, and any candidate with no tracked leg released before go-live. + tracked_keys = { + (r[0], r[1], r[2], r[3]) + for r in conn.execute( + "SELECT w.wo_base, w.lot, w.split, w.sub FROM packet_work_orders w JOIN packets p ON p.id = w.packet_id" + " WHERE p.removed_at IS NULL AND p.status <> 'filed'" + ) + } + pulled = { + (r[0], r[1], r[2], r[3]) + for r in conn.execute("SELECT wo_base, lot, split, sub FROM visual_ignored_wos WHERE pulled_in_at IS NOT NULL") + } + tracked_keys |= pulled + ignored_rows: list[Row] = list(built.ignored) + cands: list[Candidate] = [] + for c in built.candidates.values(): + if ( + golive is not None + and c.rel is not None + and c.rel < golive + and not any(lg.key.as_tuple() in tracked_keys for lg in c.legs) + ): + ignored_rows.extend( + Row( + lg.key.base, + lg.key.lot, + lg.key.split, + lg.key.sub, + lg.seq, + lg.op, + lg.dwg, + lg.rev, + "", + "", + lg.qty, + lg.tgt, + c.rel, + c.want, + lg.st, + ) + for lg in c.legs + ) + continue + if c.dept not in depts: + run.error(f"candidate {c.key}", f"department {c.dept} does not exist") + continue + cands.append(c) + if ignored_rows: + run.counts["ignored_golive"] = run.guarded("go-live list", _upsert_ignored, run, ignored_rows) or 0 + + # 1. Projects and units, then 2. match, 3-5 per candidate. + by_proj: dict[str, list[Candidate]] = {} + for c in cands: + by_proj.setdefault(c.project, []).append(c) + live_keys: set[str] = set() + for number, group in sorted(by_proj.items()): + if not RX_PROJECT.match(number): + run.counts["skipped_conflict"] += len(group) + continue + due = max((c.want for c in group if c.want), default=None) + project = run.guarded(f"project {number}", _ensure_project, run, number, due) + if project is None: + continue + if str(project["status"]) == "closed": + run.counts["skipped_closed"] += len(group) + continue + pchosen = orders_mod.choose( + unit_number="", + base=number, + by_base=by_base, + by_project=by_project, + project=number, + pin=project.get("order_pin"), + ) + run.guarded(f"project {number} order", _derive, run, "projects", project, pchosen) + project = _one(conn, "SELECT * FROM projects WHERE id = %s", (project["id"],)) or project + by_unit: dict[str, list[Candidate]] = {} + for c in group: + by_unit.setdefault(c.unit, []).append(c) + for unit_no, ucands in sorted(by_unit.items()): + udue = max((c.want for c in ucands if c.want), default=None) + unit = run.guarded( + f"unit {unit_no}", + _ensure_unit, + run, + project, + unit_no, + ucands[0].lot, + udue, + ) + if unit is None: + continue + if str(unit["status"]) == "closed": + run.counts["skipped_closed"] += len(ucands) + continue + uchosen = orders_mod.choose( + unit_number=unit_no, + base=ucands[0].base, + by_base=by_base, + by_project=by_project, + project=number, + pin=unit.get("order_pin"), + ) + ref = run.guarded(f"unit {unit_no} order", _derive, run, "units", unit, uchosen) + if ref is None: + ref = unit.get("order_ref") + _match_and_apply(run, project, unit, ucands, ref, labor, live_keys) + + # 6. Retire drafts whose work orders moved. + _retire_moved(run, built, live_keys) + # 7. Work orders gone. + if not suspect: + _wo_gone(run, missing) + _wo_back(run, seen) + # 8. Closed and cancelled observed. + _observe(run, seen, labor) + # 9. Auto-release. + is_paused = bool(getattr(run.fs, "paused", False)) if paused is None else paused + if getattr(run.fs, "auto_release", False) and not is_paused: + _auto_release(run) + + +def _upsert_ignored(run: _Run, rows: list[Row]) -> int: + keys: dict[WoKey, Row] = {} + for r in rows: + keys.setdefault(r.key, r) + for k, r in keys.items(): + project = project_of(r.wo) + run.conn.execute( + "INSERT INTO visual_ignored_wos (wo_base, lot, split, sub, rel, project, unit_number, res, dwg," + " first_seen_batch) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + " ON CONFLICT (wo_base, lot, split, sub) DO UPDATE SET rel = EXCLUDED.rel, res = EXCLUDED.res," + " dwg = EXCLUDED.dwg, last_seen_at = now()", + (k.base, k.lot, k.split, k.sub, r.rel, project, unit_number(project, r.lot), r.res, r.dwg, run.batch_id), + ) + return len(keys) + + +def _match_and_apply( + run: _Run, + project: Mapping[str, Any], + unit: Mapping[str, Any], + cands: list[Candidate], + ref: Any, + labor: LaborMap | None, + live_keys: set[str], +) -> None: + conn = run.conn + with conn.cursor(row_factory=dict_row) as cur: + found = cur.execute( + "SELECT id, visual_key, status::text AS status, removed_at, visual, visual_fp_applied, source_kind::text" + " AS source_kind FROM packets WHERE visual_key = ANY(%s) ORDER BY id", + ([c.key for c in cands],), + ).fetchall() + live: dict[str, dict[str, Any]] = {} + removed: set[str] = set() + for r in found: + if r["removed_at"] is None: + live[r["visual_key"]] = dict(r) + else: + removed.add(r["visual_key"]) + for c in cands: + p = live.get(c.key) + if p is None: + if c.key in removed: + run.counts["skipped_removed"] += 1 + continue + if _on_hand_packet(run.conn, c): + run.counts["skipped_hand"] += 1 + continue + pid = run.guarded(f"new draft {c.key}", _create, run, c, unit, project, ref, labor) + if pid is not None: + live_keys.add(c.key) + continue + live_keys.add(c.key) + if p["source_kind"] != "visual" or p["status"] == "filed": + continue + if p["status"] == "draft": + run.guarded(f"update draft {c.key}", _update_draft, run, p, c) + else: + run.guarded(f"change order {c.key}", _past_draft, run, p, c) + + +def _on_hand_packet(conn: Any, c: Candidate) -> bool: + """The old visMatch "exists" rule: every leg's work order (base, lot, split, sub) already sits, with the same + operation, on one live packet made by hand. Such a candidate is not created.""" + keys = sorted({lg.key.as_tuple() for lg in c.legs}) + if not keys: + return False + row = conn.execute( + "SELECT p.id FROM packets p JOIN packet_work_orders w ON w.packet_id = p.id" + " JOIN unnest(%s::text[], %s::text[], %s::text[], %s::text[]) AS k(b, l, sp, su)" + " ON w.wo_base = k.b AND w.lot = k.l AND w.split = k.sp AND w.sub = k.su" + " WHERE p.removed_at IS NULL AND p.source_kind = 'hand' AND upper(coalesce(w.op, p.op, '')) = upper(%s)" + " GROUP BY p.id HAVING count(DISTINCT (w.wo_base, w.lot, w.split, w.sub)) = %s LIMIT 1", + ([k[0] for k in keys], [k[1] for k in keys], [k[2] for k in keys], [k[3] for k in keys], c.op, len(keys)), + ).fetchone() + return row is not None + + +def _retire_moved(run: _Run, built: Built, live_keys: set[str]) -> None: + owner: dict[tuple[str, str, str, str, int], str] = {} + for c in built.candidates.values(): + if c.key not in live_keys: + continue + for lg in c.legs: + owner[(*lg.key.as_tuple(), lg.seq)] = c.key + if not owner: + return + with run.conn.cursor(row_factory=dict_row) as cur: + drafts = cur.execute( + "SELECT p.id, p.visual_key, p.visual FROM packets p WHERE p.removed_at IS NULL AND p.status = 'draft'" + " AND p.source_kind = 'visual' AND NOT (p.visual_key = ANY(%s))" + " AND NOT (p.visual ? 'manual_dwg') AND NOT (p.visual ? 'manual_bom') AND NOT (p.visual ? 'moved')" + " AND NOT EXISTS (SELECT 1 FROM packet_pages g WHERE g.packet_id = p.id AND g.type = 'drawing')" + " AND EXISTS (SELECT 1 FROM packet_work_orders w WHERE w.packet_id = p.id)", + (sorted(built.candidates),), + ).fetchall() + for d in drafts: + with run.conn.cursor(row_factory=dict_row) as cur: + legs = cur.execute( + "SELECT wo_base, lot, split, sub, seq FROM packet_work_orders WHERE packet_id = %s", (d["id"],) + ).fetchall() + to = [owner.get((w["wo_base"], w["lot"], w["split"], w["sub"], w["seq"])) for w in legs] + if any(t is None or t == d["visual_key"] for t in to): + continue + dest = sorted({t for t in to if t}) + drawings = sorted({t.split("|", 2)[2] for t in dest}) + + def retire( + d: Mapping[str, Any] = d, + dest: list[str] = dest, + drawings: list[str] = drawings, + legs: list[dict[str, Any]] = legs, + ) -> None: + note = ( + f"{', '.join(sorted({w['wo_base'] for w in legs}))} now on drawing {', '.join(drawings)} in VISUAL" + f" ({', '.join(dest)}). Kept for the record, hidden from the boards." + ) + with system_write(run.conn, int(d["id"]), A.VIS_RETIRE_DRAFT, note=note) as sw: + visual = dict(sw.packet.get("visual") or {}) + visual["moved"] = {"to": dest, "drawing": ", ".join(drawings), "at": run.now.isoformat()} + sw.conn.execute( + "UPDATE packets SET removed_at = now(), removed_by = NULL, removed_reason = 'moved', visual = %s" + " WHERE id = %s", + (Jsonb(packets_jsonable(visual)), sw.id), + ) + sw.changed("removed_at", "visual") + run.counts["retired"] += 1 + + run.guarded(f"retire {d['visual_key']}", retire) + + +def _key_arrays(keys: Iterable[WoKey]) -> tuple[list[str], list[str], list[str], list[str]]: + ks = list(dict.fromkeys(keys)) + return [k.base for k in ks], [k.lot for k in ks], [k.split for k in ks], [k.sub for k in ks] + + +_UNNEST = "unnest(%s::text[], %s::text[], %s::text[], %s::text[]) AS k(b, l, sp, su)" +_MATCH = "w.wo_base = k.b AND w.lot = k.l AND w.split = k.sp AND w.sub = k.su" +_LIVE = "p.removed_at IS NULL AND p.status <> 'filed' AND p.source_kind = 'visual'" + + +def _wo_gone(run: _Run, missing: set[WoKey]) -> None: + if not missing: + return + conn = run.conn + with conn.cursor(row_factory=dict_row) as cur: + hit = cur.execute( + f"UPDATE packet_work_orders w SET missing_runs = missing_runs + 1 FROM packets p, {_UNNEST}" # noqa: S608 + f" WHERE p.id = w.packet_id AND {_LIVE} AND {_MATCH}" + " RETURNING w.id, w.packet_id, w.wo_base, w.lot, w.split, w.sub, w.missing_runs, w.gone_at," + " p.status::text AS status", + _key_arrays(missing), + ).fetchall() + newly: dict[int, list[dict[str, Any]]] = {} + for h in hit: + if h["missing_runs"] >= 2 and h["gone_at"] is None: + newly.setdefault(int(h["packet_id"]), []).append(dict(h)) + for pid, legs in newly.items(): + keys = sorted({f"{h['wo_base']}/{h['lot']}/{h['split']}/{h['sub']}" for h in legs}) + + def body(sw: SysWrite, legs: list[dict[str, Any]] = legs) -> bool: + sw.conn.execute( + "UPDATE packet_work_orders SET gone_at = now() WHERE id = ANY(%s) AND gone_at IS NULL", + ([h["id"] for h in legs],), + ) + _set_flags(sw.conn, sw.id, add=("wo_gone",), extra_sql=", wo_gone_at = coalesce(wo_gone_at, now())") + sw.changed("flags") + return True + + label = f"Work order {', '.join(keys)} no longer in VISUAL" + if run.guarded( + f"wo gone {pid}", + _flag_write, + run, + pid, + legs[0]["status"], + label, + None, + body, + ): + run.counts["wo_gone_flagged"] += 1 + + +def _wo_back(run: _Run, seen: Mapping[WoKey, str]) -> None: + if not seen: + return + conn = run.conn + with conn.cursor(row_factory=dict_row) as cur: + back = cur.execute( + f"WITH b AS (SELECT w.id, w.gone_at AS was_gone FROM packet_work_orders w, {_UNNEST}" # noqa: S608 + f" WHERE {_MATCH} AND (w.missing_runs > 0 OR w.gone_at IS NOT NULL) FOR UPDATE OF w)" + " UPDATE packet_work_orders w SET missing_runs = 0, gone_at = NULL FROM b WHERE w.id = b.id" + " RETURNING w.packet_id, b.was_gone", + _key_arrays(seen), + ).fetchall() + packets_back = sorted({int(r["packet_id"]) for r in back if r["was_gone"] is not None}) + for pid in packets_back: + row = _one( + conn, + "SELECT p.status::text AS status, p.flags, p.removed_at," + " EXISTS (SELECT 1 FROM packet_work_orders w WHERE w.packet_id = p.id AND w.gone_at IS NOT NULL) AS still" + " FROM packets p WHERE p.id = %s", + (pid,), + ) + if row is None or row["removed_at"] is not None or row["status"] == "filed" or row["still"]: + continue + if "wo_gone" not in (row["flags"] or []): + continue + + def body(sw: SysWrite) -> bool: + _set_flags(sw.conn, sw.id, remove=("wo_gone",), extra_sql=", wo_gone_at = NULL") + sw.changed("flags") + return True + + if run.guarded( + f"wo back {pid}", + _flag_write, + run, + pid, + row["status"], + "Work order back in VISUAL", + None, + body, + ): + run.counts["wo_back"] += 1 + + +def _observe(run: _Run, seen: Mapping[WoKey, str], labor: LaborMap | None) -> None: + """Per leg visual_status, packet_visual_state, closed in VISUAL, cancelled flag.""" + if not seen: + return + conn = run.conn + with conn.cursor(row_factory=dict_row) as cur: + legs = cur.execute( + "SELECT w.id, w.packet_id, w.wo_base, w.lot, w.split, w.sub, w.seq, w.visual_status," # noqa: S608 + " p.status::text AS status, p.flags, p.visual_closed_at FROM packet_work_orders w" + f" JOIN packets p ON p.id = w.packet_id WHERE {_LIVE} AND w.packet_id IN (SELECT w2.packet_id" + f" FROM packet_work_orders w2, {_UNNEST.replace('k(', 'k2(')} WHERE w2.wo_base = k2.b AND w2.lot = k2.l" + " AND w2.split = k2.sp AND w2.sub = k2.su) ORDER BY w.packet_id, w.sort, w.id", + _key_arrays(seen), + ).fetchall() + per: dict[int, list[dict[str, Any]]] = {} + for lg in legs: + per.setdefault(int(lg["packet_id"]), []).append(dict(lg)) + for pid, pl in per.items(): + sts: list[str] = [] + state_legs: list[dict[str, Any]] = [] + for lg in pl: + k = WoKey(lg["wo_base"], lg["lot"], lg["split"], lg["sub"]) + st = seen.get(k, lg["visual_status"] or "") + sts.append(st) + state_legs.append({"key": str(k), "seq": lg["seq"], "st": st}) + if st and st != (lg["visual_status"] or ""): + conn.execute("UPDATE packet_work_orders SET visual_status = %s WHERE id = %s", (st, lg["id"])) + hours = _hours( + labor, + ( + (WoKey(lg["wo_base"], lg["lot"], lg["split"], lg["sub"]), int(lg["seq"])) + for lg in pl + if lg["seq"] is not None + ), + ) + run.guarded(f"observe {pid}", _visual_state, conn, pid, state_legs, hours, None) + head = pl[0] + status = head["status"] + flags = list(head["flags"] or []) + if sts and all(s == "C" for s in sts) and head["visual_closed_at"] is None: + + def closed(sw: SysWrite) -> bool: + sw.conn.execute("UPDATE packets SET visual_closed_at = now() WHERE id = %s", (sw.id,)) + sw.changed("visual_closed_at") + return True + + if run.guarded( + f"closed {pid}", + _flag_write, + run, + pid, + status, + "Closed in VISUAL (observed)", + None, + closed, + ): + run.counts["closed_observed"] += 1 + cancelled = any(s == "X" for s in sts) + if cancelled and "cancelled" not in flags: + + def cancel(sw: SysWrite) -> bool: + _set_flags( + sw.conn, + sw.id, + add=("cancelled",), + extra_sql=", visual_cancelled_at = coalesce(visual_cancelled_at, now())", + ) + sw.changed("flags") + return True + + if run.guarded( + f"cancelled {pid}", + _flag_write, + run, + pid, + status, + "Work order cancelled in VISUAL", + None, + cancel, + ): + run.counts["cancelled_flagged"] += 1 + elif not cancelled and "cancelled" in flags: + + def uncancel(sw: SysWrite) -> bool: + _set_flags(sw.conn, sw.id, remove=("cancelled",), extra_sql=", visual_cancelled_at = NULL") + sw.changed("flags") + return True + + run.guarded( + f"uncancelled {pid}", + _flag_write, + run, + pid, + status, + "Work order no longer cancelled in VISUAL", + None, + uncancel, + ) + + +HEAT_SQL = ( + "EXISTS (SELECT 1 FROM bom_lines b WHERE b.packet_id = packets.id AND coalesce(b.sub, '') <> 'Y' AND NOT b.na" + " AND coalesce(b.cons, '') <> 'Y')" +) + + +def _auto_release(run: _Run) -> None: + limit = int(getattr(run.fs, "auto_release_max", 25) or 0) + if limit <= 0: + return + ids = [ + int(r[0]) + for r in run.conn.execute( + "SELECT id FROM packets WHERE removed_at IS NULL AND status = 'draft' AND source_kind = 'visual'" + " AND drawing_state = 'attached' AND flags = '{}' AND coalesce((visual->>'rev_confirmed')::boolean, false)" + " ORDER BY id LIMIT %s", + (limit,), + ) + ] + for pid in ids: + + def release(pid: int = pid) -> None: + conn = run.conn + hm = conn.execute( + f"UPDATE packets SET heat_map = coalesce(heat_map, {HEAT_SQL}) WHERE id = %s" # noqa: S608 + " RETURNING heat_map, drawing, rev", + (pid,), + ).fetchone() + if hm is None: + return + note = f"no flags · drawing {hm[1] or '-'} rev {hm[2] or '-'} confirmed in M-Files · heat map " + ( + "required (material lines)" if hm[0] else "not required (sub-assemblies only)" + ) + with system_write(conn, pid, A.VIS_AUTO_RELEASE, note=note): + pass + run.counts["auto_released"] += 1 + + run.guarded(f"auto-release {pid}", release) + + +__all__ = ["COUNT_KEYS", "DRAWING_FLAGS", "HEAT_SQL", "apply"] diff --git a/server/packetstation/feed/cache.py b/server/packetstation/feed/cache.py new file mode 100644 index 0000000..9bdf84e --- /dev/null +++ b/server/packetstation/feed/cache.py @@ -0,0 +1,564 @@ +"""The VISUAL cache in PostgreSQL: one batch per read, the rows and material lines as VISUAL last showed them, +the orders and their links, labor per operation, and a change row for everything VISUAL added, changed, dropped +or brought back. + +`write_batch` runs in one transaction. Rows and materials go in through COPY into a temp table, then an upsert +keyed on the typed key (wo_base, lot, split, sub, seq[, piece]), all text but seq and piece. Rows in play that this +read did not return get `gone_batch`, unless the batch is suspect: no window rows where the previous batch had +some, or more than 5 tracked keys missing and more than 10 % of them. + +The apply reads back from here (`rows_for_apply`, `orders`, `labor_hours`), so it works on one consistent snapshot +and keeps the last good orders when this run's order read failed. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Collection, Iterable, Mapping +from dataclasses import dataclass, field +from datetime import date +from decimal import Decimal +from typing import Any + +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb + +from .fingerprint import canonical, norm_decimal +from .materials import material_from_row +from .types import FeedData, Material, Order, Row, WoKey, num + +ROW_COLS = ( + "wo_base", + "lot", + "split", + "sub", + "seq", + "res", + "dwg", + "rev", + "ps", + "descr", + "qty", + "tgt", + "rel", + "want", + "st", + "part", + "barcode", +) +MAT_COLS = ( + "wo_base", + "lot", + "split", + "sub", + "seq", + "piece", + "part", + "qty", + "issued", + "whs", + "loc", + "st", + "istat", + "dwg", + "descr", + "comm", + "cons", +) +SUSPECT_MISSING_MIN = 5 +SUSPECT_MISSING_SHARE = Decimal("0.10") + + +@dataclass +class Batch: + id: int + suspect: bool + counts: dict[str, Any] = field(default_factory=dict) + reason: str = "" + + +# ---------------------------------------------------------------- row dicts and hashes + + +def _d(v: Any) -> str | None: + return v.isoformat() if isinstance(v, date) else None + + +def row_doc(r: Row) -> dict[str, Any]: + """The row as a change-row dict: text values, numbers as text, dates ISO.""" + return { + "wo": r.wo, + "lot": r.lot, + "split": r.split, + "sub": r.sub, + "seq": r.seq, + "res": r.res, + "dwg": r.dwg, + "rev": r.rev, + "ps": r.ps, + "desc": r.desc, + "qty": norm_decimal(r.qty), + "tgt": norm_decimal(r.tgt), + "rel": _d(r.rel), + "want": _d(r.want), + "st": r.st, + "part": r.part, + "barcode": r.barcode, + } + + +def mat_doc(m: Material) -> dict[str, Any]: + return { + "wo": m.wo, + "lot": m.lot, + "split": m.split, + "sub": m.sub, + "seq": m.seq, + "piece": m.piece, + "part": m.part, + "qty": norm_decimal(m.qty), + "issued": norm_decimal(m.issued), + "whs": m.whs, + "loc": m.loc, + "st": m.st, + "istat": m.istat, + "dwg": m.dwg, + "desc": m.desc, + "comm": m.comm, + "cons": m.cons, + } + + +def doc_hash(doc: Mapping[str, Any]) -> bytes: + return hashlib.sha256(canonical(dict(doc))).digest() + + +# SQL that rebuilds the same dict from a cached row, for the change rows' `before`. +_ROW_BEFORE = ( + "jsonb_build_object('wo', v.wo_base, 'lot', v.lot, 'split', v.split, 'sub', v.sub, 'seq', v.seq, 'res', v.res," + " 'dwg', v.dwg, 'rev', v.rev, 'ps', v.ps, 'desc', v.descr, 'qty', v.qty::text, 'tgt', v.tgt::text," + " 'rel', v.rel, 'want', v.want, 'st', v.st, 'part', v.part, 'barcode', v.barcode)" +) +_MAT_BEFORE = ( + "jsonb_build_object('wo', v.wo_base, 'lot', v.lot, 'split', v.split, 'sub', v.sub, 'seq', v.seq," + " 'piece', v.piece, 'part', v.part, 'qty', v.qty::text, 'issued', v.issued::text, 'whs', v.whs, 'loc', v.loc," + " 'st', v.st, 'istat', v.istat, 'dwg', v.dwg, 'desc', v.descr, 'comm', v.comm, 'cons', v.cons)" +) + + +def _key_arrays(keys: Iterable[WoKey]) -> tuple[list[str], list[str], list[str], list[str]]: + ks = list(dict.fromkeys(keys)) + return [k.base for k in ks], [k.lot for k in ks], [k.split for k in ks], [k.sub for k in ks] + + +_KEYS_UNNEST = "unnest(%s::text[], %s::text[], %s::text[], %s::text[]) AS k(b, l, sp, su)" + + +# ---------------------------------------------------------------- write + + +def is_suspect(window_rows: int, prev_window_rows: int | None, missing: int, tracked: int) -> str: + """The reason a batch is suspect, or ''.""" + if window_rows == 0 and (prev_window_rows or 0) > 0: + return f"the window read returned nothing (the previous read had {prev_window_rows})" + if missing > SUSPECT_MISSING_MIN and Decimal(missing) > SUSPECT_MISSING_SHARE * tracked: + return f"{missing} of {tracked} tracked work orders missing" + return "" + + +def write_batch(conn: Any, run_id: int | None, data: FeedData, source_desc: str, window: tuple[date, date]) -> Batch: + """Record one read: the batch row, rows, materials, orders, labor and the change rows. One transaction.""" + since, until = window + with conn.transaction(): + prev = conn.execute("SELECT window_rows FROM visual_batches ORDER BY id DESC LIMIT 1").fetchone() + missing = len(set(data.tracked_missing)) + tracked_n = len(set(data.tracked)) or data.tracked_count + reason = is_suspect(data.window_count, prev[0] if prev else None, missing, tracked_n) + suspect = bool(reason) + server, _, database = source_desc.partition(" / ") + bid = int( + conn.execute( + "INSERT INTO visual_batches (run_id, server, database, site, since, until, window_rows, tracked_keys," + " missing_keys, suspect) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id", + ( + run_id, + str(data.meta.get("server") or server), + str(data.meta.get("database") or database) or None, + data.meta.get("site"), + since, + until, + data.window_count, + tracked_n, + missing, + suspect, + ), + ).fetchone()[0] + ) + counts: dict[str, Any] = {"rows": len(data.rows), "suspect": int(suspect)} + if reason: + counts["suspect_reason"] = reason + counts.update(_write_rows(conn, bid, data, window, suspect)) + if data.materials is not None: + counts.update(_write_materials(conn, bid, data, suspect)) + counts["materials"] = len(data.materials) + if data.orders_by_base is not None and data.orders_by_project is not None: + counts["orders"] = _write_orders(conn, bid, data) + if data.labor is not None: + counts["labor_legs"] = _write_labor(conn, data) + counts["labor_read"] = 1 + conn.execute("UPDATE visual_batches SET counts = %s WHERE id = %s", (Jsonb(counts), bid)) + return Batch(id=bid, suspect=suspect, counts=counts, reason=reason) + + +def _write_rows(conn: Any, bid: int, data: FeedData, window: tuple[date, date], suspect: bool) -> dict[str, int]: + uniq: dict[tuple[WoKey, int], Row] = {} + for r in data.rows: + uniq[(r.key, r.seq)] = r + conn.execute( + "CREATE TEMP TABLE t_rows (wo_base text, lot text, split text, sub text, seq int, res text, dwg text," + " rev text, ps text, descr text, qty numeric, tgt numeric, rel date, want date, st text, part text," + " barcode text, row_hash bytea, doc jsonb) ON COMMIT DROP" + ) + with ( + conn.cursor() as cur, + cur.copy( + "COPY t_rows (wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st, part," + " barcode, row_hash, doc) FROM STDIN" + ) as cp, + ): + for r in uniq.values(): + doc = row_doc(r) + cp.write_row( + ( + r.wo, + r.lot, + r.split, + r.sub, + r.seq, + r.res, + r.dwg, + r.rev, + r.ps, + r.desc, + r.qty, + r.tgt, + r.rel, + r.want, + r.st, + r.part, + r.barcode or None, + doc_hash(doc), + Jsonb(doc), + ) + ) + out: dict[str, int] = {} + join = ( + "visual_rows v ON v.wo_base = t.wo_base AND v.lot = t.lot AND v.split = t.split AND v.sub = t.sub" + " AND v.seq = t.seq" + ) + out["rows_added"] = conn.execute( + "INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, after)" # noqa: S608 + f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'added', t.doc FROM t_rows t LEFT JOIN {join}" + " WHERE v.id IS NULL", + (bid,), + ).rowcount + out["rows_returned"] = conn.execute( + "INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before, after)" # noqa: S608 + f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'returned', {_ROW_BEFORE}, t.doc FROM t_rows t" + f" JOIN {join} WHERE v.gone_batch IS NOT NULL", + (bid,), + ).rowcount + out["rows_changed"] = conn.execute( + "INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before, after)" # noqa: S608 + f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, 'changed', {_ROW_BEFORE}, t.doc FROM t_rows t" + f" JOIN {join} WHERE v.gone_batch IS NULL AND v.row_hash <> t.row_hash", + (bid,), + ).rowcount + conn.execute( + "INSERT INTO visual_rows (wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st," + " part, barcode, row_hash, first_batch, last_batch)" + " SELECT wo_base, lot, split, sub, seq, res, dwg, rev, ps, descr, qty, tgt, rel, want, st, part, barcode," + " row_hash, %s, %s FROM t_rows" + " ON CONFLICT (wo_base, lot, split, sub, seq) DO UPDATE SET res = EXCLUDED.res, dwg = EXCLUDED.dwg," + " rev = EXCLUDED.rev, ps = EXCLUDED.ps, descr = EXCLUDED.descr, qty = EXCLUDED.qty, tgt = EXCLUDED.tgt," + " rel = EXCLUDED.rel, want = EXCLUDED.want, st = EXCLUDED.st, part = EXCLUDED.part," + " barcode = EXCLUDED.barcode, row_hash = EXCLUDED.row_hash, last_batch = EXCLUDED.last_batch," + " gone_batch = NULL", + (bid, bid), + ) + out["rows_gone"] = 0 + if not suspect: + b, lot, sp, su = _key_arrays(data.tracked) + out["rows_gone"] = conn.execute( + "WITH gone AS (UPDATE visual_rows v SET gone_batch = %s" # noqa: S608 + " WHERE v.gone_batch IS NULL AND v.last_batch <> %s" + f" AND (EXISTS (SELECT 1 FROM {_KEYS_UNNEST} WHERE k.b = v.wo_base AND k.l = v.lot AND k.sp = v.split" + " AND k.su = v.sub) OR (v.st = 'F' AND v.rel BETWEEN %s AND %s)) RETURNING v.*)" + " INSERT INTO visual_row_changes (batch_id, wo_base, lot, split, sub, seq, kind, before)" + f" SELECT %s, v.wo_base, v.lot, v.split, v.sub, v.seq, 'gone', {_ROW_BEFORE} FROM gone v", + (bid, bid, b, lot, sp, su, window[0], window[1], bid), + ).rowcount + return out + + +def _write_materials(conn: Any, bid: int, data: FeedData, suspect: bool) -> dict[str, int]: + uniq: dict[tuple[WoKey, int, int], Material] = {} + for m in data.materials or []: + uniq[(m.key, m.seq, m.piece)] = m + conn.execute( + "CREATE TEMP TABLE t_mats (wo_base text, lot text, split text, sub text, seq int, piece int, part text," + " qty numeric, issued numeric, whs text, loc text, st text, istat int, dwg text, descr text, comm text," + " cons text, mat_hash bytea, doc jsonb) ON COMMIT DROP" + ) + with ( + conn.cursor() as cur, + cur.copy( + "COPY t_mats (wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat, dwg, descr," + " comm, cons, mat_hash, doc) FROM STDIN" + ) as cp, + ): + for m in uniq.values(): + doc = mat_doc(m) + cp.write_row( + ( + m.wo, + m.lot, + m.split, + m.sub, + m.seq, + m.piece, + m.part, + m.qty, + m.issued, + m.whs, + m.loc, + m.st, + m.istat, + m.dwg, + m.desc, + m.comm, + m.cons, + doc_hash(doc), + Jsonb(doc), + ) + ) + join = ( + "visual_materials v ON v.wo_base = t.wo_base AND v.lot = t.lot AND v.split = t.split AND v.sub = t.sub" + " AND v.seq = t.seq AND v.piece = t.piece" + ) + cols = "batch_id, wo_base, lot, split, sub, seq, piece, kind" + out: dict[str, int] = {} + out["mats_added"] = conn.execute( + f"INSERT INTO visual_material_changes ({cols}, after)" # noqa: S608 + f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'added', t.doc FROM t_mats t LEFT JOIN {join}" + " WHERE v.id IS NULL", + (bid,), + ).rowcount + out["mats_returned"] = conn.execute( + f"INSERT INTO visual_material_changes ({cols}, before, after)" # noqa: S608 + f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'returned', {_MAT_BEFORE}, t.doc" + f" FROM t_mats t JOIN {join} WHERE v.gone_batch IS NOT NULL", + (bid,), + ).rowcount + out["mats_changed"] = conn.execute( + f"INSERT INTO visual_material_changes ({cols}, before, after)" # noqa: S608 + f" SELECT %s, t.wo_base, t.lot, t.split, t.sub, t.seq, t.piece, 'changed', {_MAT_BEFORE}, t.doc" + f" FROM t_mats t JOIN {join} WHERE v.gone_batch IS NULL AND v.mat_hash <> t.mat_hash", + (bid,), + ).rowcount + conn.execute( + "INSERT INTO visual_materials (wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat," + " dwg, descr, comm, cons, mat_hash, first_batch, last_batch)" + " SELECT wo_base, lot, split, sub, seq, piece, part, qty, issued, whs, loc, st, istat, dwg, descr, comm," + " cons, mat_hash, %s, %s FROM t_mats" + " ON CONFLICT (wo_base, lot, split, sub, seq, piece) DO UPDATE SET part = EXCLUDED.part, qty = EXCLUDED.qty," + " issued = EXCLUDED.issued, whs = EXCLUDED.whs, loc = EXCLUDED.loc, st = EXCLUDED.st," + " istat = EXCLUDED.istat, dwg = EXCLUDED.dwg, descr = EXCLUDED.descr, comm = EXCLUDED.comm," + " cons = EXCLUDED.cons, mat_hash = EXCLUDED.mat_hash, last_batch = EXCLUDED.last_batch, gone_batch = NULL", + (bid, bid), + ) + out["mats_gone"] = 0 + if not suspect: + bases = sorted({r.wo for r in data.rows} | {m.wo for m in data.materials or []}) + out["mats_gone"] = conn.execute( + "WITH gone AS (UPDATE visual_materials v SET gone_batch = %s WHERE v.gone_batch IS NULL" # noqa: S608 + " AND v.last_batch <> %s AND v.wo_base = ANY(%s) RETURNING v.*)" + f" INSERT INTO visual_material_changes ({cols}, before)" + f" SELECT %s, v.wo_base, v.lot, v.split, v.sub, v.seq, v.piece, 'gone', {_MAT_BEFORE} FROM gone v", + (bid, bid, bases, bid), + ).rowcount + return out + + +def _write_orders(conn: Any, bid: int, data: FeedData) -> int: + by_base = data.orders_by_base or {} + by_project = data.orders_by_project or {} + every: dict[str, Order] = {} + for lists in (by_base, by_project): + for orders in lists.values(): + for o in orders: + every.setdefault(o.cust_order_id, o) + for o in every.values(): + conn.execute( + "INSERT INTO visual_orders (cust_order_id, status, customer_id, customer_name, customer_po_ref," + " order_date, desired_ship_date, promise_date, last_batch) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)" + " ON CONFLICT (cust_order_id) DO UPDATE SET status = EXCLUDED.status, customer_id = EXCLUDED.customer_id," + " customer_name = EXCLUDED.customer_name, customer_po_ref = EXCLUDED.customer_po_ref," + " order_date = coalesce(EXCLUDED.order_date, visual_orders.order_date)," + " desired_ship_date = EXCLUDED.desired_ship_date, promise_date = EXCLUDED.promise_date," + " last_batch = EXCLUDED.last_batch", + ( + o.cust_order_id, + o.status, + o.customer_id, + o.customer_name, + o.customer_po_ref, + o.order_date, + o.desired_ship_date, + o.promise_date, + bid, + ), + ) + bases = sorted({r.wo for r in data.rows} | set(by_base)) + projects = sorted({r.wo.partition("-")[0] or r.wo for r in data.rows} | set(by_project)) + conn.execute( + "DELETE FROM visual_order_links WHERE (kind = 'base' AND key = ANY(%s))" + " OR (kind = 'project' AND key = ANY(%s))", + (bases, projects), + ) + n = 0 + for kind, lists in (("base", by_base), ("project", by_project)): + for key, orders in lists.items(): + seen: set[str] = set() + for rank, o in enumerate(orders): + if o.cust_order_id in seen: + continue + seen.add(o.cust_order_id) + conn.execute( + "INSERT INTO visual_order_links (kind, key, cust_order_id, rank, lines, top_jobs, last_batch)" + " VALUES (%s, %s, %s, %s, %s, %s, %s)", + (kind, key, o.cust_order_id, rank, o.lines, o.top_jobs, bid), + ) + n += 1 + return len(every) + + +def _write_labor(conn: Any, data: FeedData) -> int: + n = 0 + for lg in data.labor or []: + conn.execute( + "INSERT INTO visual_labor (wo_base, lot, split, sub, seq, hours, tickets, last_ticket, observed_at)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now())" + " ON CONFLICT (wo_base, lot, split, sub, seq) DO UPDATE SET hours = EXCLUDED.hours," + " tickets = EXCLUDED.tickets, last_ticket = EXCLUDED.last_ticket, observed_at = now()", + (lg.wo, lg.lot, lg.split, lg.sub, lg.seq, lg.hours, lg.tickets, lg.last_ticket), + ) + n += 1 + return n + + +# ---------------------------------------------------------------- read back + + +def row_from_cache(r: Mapping[str, Any]) -> Row: + return Row( + wo=r["wo_base"], + lot=r["lot"], + split=r["split"], + sub=r["sub"], + seq=int(r["seq"]), + res=r["res"], + dwg=r["dwg"], + rev=r["rev"], + ps=r["ps"], + desc=r["descr"], + qty=num(r["qty"]), + tgt=num(r["tgt"]), + rel=r["rel"], + want=r["want"], + st=r["st"], + part=r["part"], + barcode=r["barcode"] or "", + ) + + +def rows_for_apply(conn: Any, keys_in_play: Collection[WoKey]) -> tuple[list[Row], list[Material], set[WoKey]]: + """Live cached rows and material lines of every work order (base and lot) among these keys, so folded + operations and planner instructions of the same work order come along. material_legs = legs with live + material lines whose status is F or R.""" + pairs = sorted({(k.base, k.lot) for k in keys_in_play}) + if not pairs: + return [], [], set() + bases = [p[0] for p in pairs] + lots = [p[1] for p in pairs] + sql_join = "JOIN unnest(%s::text[], %s::text[]) AS k(b, l) ON v.wo_base = k.b AND v.lot = k.l" + with conn.cursor(row_factory=dict_row) as cur: + rows = cur.execute( + f"SELECT v.* FROM visual_rows v {sql_join} WHERE v.gone_batch IS NULL" # noqa: S608 + " ORDER BY v.wo_base, v.lot, v.split, v.sub, v.seq", + (bases, lots), + ).fetchall() + mats = cur.execute( + f"SELECT v.* FROM visual_materials v {sql_join} WHERE v.gone_batch IS NULL" # noqa: S608 + " ORDER BY v.wo_base, v.lot, v.split, v.sub, v.seq, v.piece", + (bases, lots), + ).fetchall() + materials = [material_from_row(m) for m in mats] + legs = {m.key for m in materials if m.st in ("F", "R")} + return [row_from_cache(r) for r in rows], materials, legs + + +def orders( + conn: Any, *, bases: Iterable[str] | None = None, projects: Iterable[str] | None = None +) -> tuple[dict[str, list[Order]], dict[str, list[Order]]]: + """The cached order links as (by_base, by_project), in rank order. Without filters: every link.""" + params: list[Any] = [] + where = "" + if bases is not None or projects is not None: + where = " WHERE (l.kind = 'base' AND l.key = ANY(%s)) OR (l.kind = 'project' AND l.key = ANY(%s))" + params = [sorted(set(bases or ())), sorted(set(projects or ()))] + with conn.cursor(row_factory=dict_row) as cur: + rows = cur.execute( + "SELECT l.kind, l.key, l.rank, l.lines, l.top_jobs, o.* FROM visual_order_links l" # noqa: S608 + f" JOIN visual_orders o ON o.cust_order_id = l.cust_order_id{where} ORDER BY l.kind, l.key, l.rank", + params, + ).fetchall() + by_base: dict[str, list[Order]] = {} + by_project: dict[str, list[Order]] = {} + for r in rows: + o = Order( + cust_order_id=r["cust_order_id"], + status=r["status"], + customer_id=r["customer_id"], + customer_name=r["customer_name"], + customer_po_ref=r["customer_po_ref"], + order_date=r["order_date"], + desired_ship_date=r["desired_ship_date"], + promise_date=r["promise_date"], + lines=list(r["lines"] or []), + top_jobs=list(r["top_jobs"] or []), + ) + (by_base if r["kind"] == "base" else by_project).setdefault(r["key"], []).append(o) + return by_base, by_project + + +def labor_hours(conn: Any) -> dict[tuple[WoKey, int], Decimal]: + """Cached ticket hours per (leg, seq).""" + out: dict[tuple[WoKey, int], Decimal] = {} + for r in conn.execute("SELECT wo_base, lot, split, sub, seq, hours FROM visual_labor"): + out[(WoKey(r[0], r[1], r[2], r[3]), int(r[4]))] = num(r[5]) + return out + + +def last_labor_read(conn: Any) -> Any: + """When labor was last read (a batch that carried labor), or None.""" + r = conn.execute("SELECT max(started_at) FROM visual_batches WHERE counts ? 'labor_read'").fetchone() + return r[0] if r else None + + +def finish_batch(conn: Any, batch_id: int, counts: Mapping[str, Any]) -> None: + """Merge the apply's counts into the batch row and set finished_at.""" + conn.execute( + "UPDATE visual_batches SET finished_at = now(), counts = counts || %s WHERE id = %s", + (Jsonb(dict(counts)), batch_id), + ) diff --git a/server/packetstation/feed/candidates.py b/server/packetstation/feed/candidates.py new file mode 100644 index 0000000..2659187 --- /dev/null +++ b/server/packetstation/feed/candidates.py @@ -0,0 +1,243 @@ +"""VISUAL rows to packet candidates: the old visBuild, typed. + +One candidate = unit + resource + drawing, holding every work order operation on that drawing. The key is +`f"{unit}|{res}|{dwg or '(no drawing)'}"`, which is `packets.visual_key`. Project = the work order base before its +first dash; unit = project + the lot zero-padded to 3. A non-numeric lot (7A) makes no candidate, as the old +build: it is counted in `Built.skipped_lots`. Resources route through the resource map: a shop department code, +`fold` (rides along on the packet of the same work order and lot), `ignore`, or no entry (counted as unmapped). +`MAT_*` with no map entry folds. +""" + +from __future__ import annotations + +from collections.abc import Collection, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import date +from decimal import Decimal + +from ..text import RX_LOT, RX_PROJECT +from . import materials as mats +from .materials import BomLine +from .types import Material, Row, WoKey + +FOLD = "fold" +IGNORE = "ignore" +NO_DRAWING = "(no drawing)" + +__all__ = [ + "FOLD", + "IGNORE", + "NO_DRAWING", + "BomLine", + "Built", + "Candidate", + "Folded", + "Leg", + "build", + "project_of", + "route", + "unit_number", +] + + +@dataclass(slots=True) +class Leg: + """One work order operation on the packet.""" + + key: WoKey + seq: int + op: str + qty: Decimal + tgt: Decimal + dwg: str + rev: str + partspecs: str + st: str + + +@dataclass(slots=True) +class Folded: + key: WoKey + seq: int + op: str + tgt: Decimal + + +@dataclass(slots=True) +class Candidate: + key: str # f"{unit}|{res}|{dwg or '(no drawing)'}" == packets.visual_key + project: str + unit: str + lot: str + dept: str + op: str + drawing: str + rev: str + title: str + legs: list[Leg] + folded: list[Folded] + partspecs: list[str] + rel: date | None + want: date | None + bom: list[BomLine] | None # None when materials are unknown for this candidate + statuses: dict[WoKey, str] # header status per leg key + base: str # the first leg's wo base, for the order lookup + in_window: bool # at least one leg came from the window read + + @property + def mat_known(self) -> bool: + return self.bom is not None + + @property + def leg_keys(self) -> set[WoKey]: + return {lg.key for lg in self.legs} + + +@dataclass(slots=True) +class Built: + candidates: dict[str, Candidate] + unmapped: dict[str, int] + ignored: list[Row] # rows below the go-live date (not tracked) + skipped_lots: dict[str, int] = field(default_factory=dict) # f"{wo}/{lot}" -> rows (non-numeric lot) + + +def route(res: str, resource_map: Mapping[str, str]) -> str | None: + """'fold' | 'ignore' | a dept code | None (no map entry). MAT_* with no entry folds.""" + if res in resource_map: + return resource_map[res] + if res.upper().startswith("MAT_"): + return FOLD + return None + + +def project_of(wo: str) -> str: + """'L489-05-5GA' -> 'L489' (upper-cased); a base with no dash is its own project.""" + head = wo.partition("-")[0].strip() + return (head or wo).upper() + + +def unit_number(project: str, lot: str) -> str | None: + """project + lot zero-padded to 3 when the lot is numeric; None for a non-numeric lot (the old unit_no).""" + lot = lot.strip() + if not lot.isdigit(): + return None + padded = lot.zfill(3) + if not RX_LOT.match(padded): + return None + return project + padded + + +def _rev_wins(new: str, cur: str) -> bool: + """The old rule: a numeric rev higher than the current wins; anything wins over an empty current.""" + if not new or new == cur: + return False + if not cur: + return True + try: + return float(new) > float(cur) + except ValueError: + return False + + +def build( + rows: Sequence[Row], + materials: Iterable[Material] | None, + material_legs: Collection[WoKey] | None, + *, + resource_map: Mapping[str, str], + window: tuple[date, date], + golive: date | None, + departments: Collection[str], + tracked_keys: Collection[WoKey], + today: date | None = None, +) -> Built: + """Group rows into candidates. See the module docstring and phase3-design "Candidates".""" + del today # kept for the contract signature; the window already carries it + tracked = set(tracked_keys) + depts = set(departments) + by_wo_lot: dict[tuple[str, str], list[Row]] = {} + for r in rows: + by_wo_lot.setdefault((r.wo, r.lot), []).append(r) + + cands: dict[str, Candidate] = {} + unmapped: dict[str, int] = {} + ignored: list[Row] = [] + skipped_lots: dict[str, int] = {} + leg_seen: dict[str, set[tuple[WoKey, int]]] = {} + fold_seen: dict[str, set[tuple[WoKey, int]]] = {} + + for r in rows: + target = route(r.res, resource_map) + if target in (FOLD, IGNORE): + continue + is_tracked = r.key in tracked + in_window = r.st == "F" and r.rel is not None and window[0] <= r.rel <= window[1] + if not is_tracked and not in_window: + continue + if target is None: + unmapped[r.res] = unmapped.get(r.res, 0) + 1 + continue + if depts and target not in depts: + continue + if not is_tracked and golive is not None and r.rel is not None and r.rel < golive: + ignored.append(r) + continue + project = project_of(r.wo) + unit = unit_number(project, r.lot) if RX_PROJECT.match(project) else None + if unit is None: + k = f"{r.wo}/{r.lot}" + skipped_lots[k] = skipped_lots.get(k, 0) + 1 + continue + key = f"{unit}|{r.res}|{r.dwg or NO_DRAWING}" + c = cands.get(key) + if c is None: + c = Candidate( + key=key, + project=project, + unit=unit, + lot=unit[len(project) :], + dept=target, + op=r.res, + drawing=r.dwg, + rev=r.rev, + title=r.desc, + legs=[], + folded=[], + partspecs=[], + rel=r.rel, + want=r.want, + bom=None, + statuses={}, + base=r.wo, + in_window=False, + ) + cands[key] = c + leg_seen[key] = set() + fold_seen[key] = set() + c.in_window = c.in_window or in_window + if (r.key, r.seq) not in leg_seen[key]: + leg_seen[key].add((r.key, r.seq)) + c.legs.append( + Leg(key=r.key, seq=r.seq, op=r.res, qty=r.qty, tgt=r.tgt, dwg=r.dwg, rev=r.rev, partspecs=r.ps, st=r.st) + ) + c.statuses[r.key] = r.st + if r.rel is not None and (c.rel is None or r.rel < c.rel): + c.rel = r.rel + if r.want is not None and (c.want is None or r.want > c.want): + c.want = r.want + if _rev_wins(r.rev, c.rev): + c.rev = r.rev + for o in by_wo_lot.get((r.wo, r.lot), []): + if route(o.res, resource_map) == FOLD and (o.key, o.seq) not in fold_seen[key]: + fold_seen[key].add((o.key, o.seq)) + c.folded.append(Folded(key=o.key, seq=o.seq, op=o.res, tgt=o.tgt)) + if o.ps and o.ps not in c.partspecs: + c.partspecs.append(o.ps) + + if material_legs is not None: + idx = mats.index(materials) + known = set(material_legs) + for c in cands.values(): + if any(lg.key in known for lg in c.legs): + c.bom = mats.bom_for(c.legs, idx) + return Built(candidates=cands, unmapped=unmapped, ignored=ignored, skipped_lots=skipped_lots) diff --git a/server/packetstation/feed/changes.py b/server/packetstation/feed/changes.py new file mode 100644 index 0000000..828e483 --- /dev/null +++ b/server/packetstation/feed/changes.py @@ -0,0 +1,382 @@ +"""Change orders opened by the feed (origin visual) and by the drawing job (origin drawing). The old visCoNeed, +coAutoChanges and visDiff. + +At most one pending feed change order per packet, whatever its origin (visual or drawing): when one is pending +it is re-aimed (reason, vis, waiting, staged, target, minor) instead of a second one being opened. A drawing +re-release that meets a pending visual change order stages the new drawing on it and adds its line to the reason. +The required departments are filled at approval (the API's co_touched), not here. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, date, datetime +from typing import TYPE_CHECKING, Any + +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb + +from ..domain.pipeline import next_counter +from ..text import LIMITS, clean_text +from .fingerprint import bom_sig, drawing_key, leg_key, norm_decimal +from .writes import SysWrite, packets_jsonable + +if TYPE_CHECKING: + from .candidates import Candidate + +FEED_ORIGINS = ("visual", "drawing") +ECN = {"visual": "VISUAL", "drawing": "M-FILES"} + + +@dataclass +class CoResult: + co_id: int + co_no: int + fresh: bool + reaimed: bool + + +def _now() -> str: + return datetime.now(UTC).isoformat(timespec="seconds") + + +def pending_visual(conn: Any, packet_id: int, *, lock: bool = False) -> dict[str, Any] | None: + """The single pending change order with origin visual or drawing, or None.""" + sql = ( + "SELECT *, origin::text AS origin, review::text AS review FROM change_orders" + " WHERE packet_id = %s AND origin IN ('visual', 'drawing') AND review = 'pending' ORDER BY co_no DESC LIMIT 1" + ) + if lock: + sql += " FOR UPDATE" + with conn.cursor(row_factory=dict_row) as cur: + r = cur.execute(sql, (packet_id,)).fetchone() + return dict(r) if r else None + + +def last_feed_co(conn: Any, packet_id: int) -> dict[str, Any] | None: + """The newest change order with origin visual, any review.""" + with conn.cursor(row_factory=dict_row) as cur: + r = cur.execute( + "SELECT *, origin::text AS origin, review::text AS review FROM change_orders" + " WHERE packet_id = %s AND origin = 'visual' ORDER BY co_no DESC LIMIT 1", + (packet_id,), + ).fetchone() + return dict(r) if r else None + + +def _j(value: Any) -> Any: + return None if value is None else Jsonb(packets_jsonable(value)) + + +def _same_target(old_vis: Mapping[str, Any] | None, vis: Mapping[str, Any]) -> bool: + o = old_vis or {} + return drawing_key(o.get("to_dwg")) == drawing_key(vis.get("to_dwg")) and str(o.get("to_rev") or "") == str( + vis.get("to_rev") or "" + ) + + +def open_or_reaim( + sw: SysWrite, + *, + origin: str, + reason: str, + vis: dict[str, Any], + waiting: dict[str, Any] | None, + staged: dict[str, Any] | None, + target_fp: str | None, + minor: bool, + required: list[str] | None = None, +) -> CoResult: + """Open the packet's feed change order, or re-aim the pending one. Call it inside `system_write`.""" + del required # filled at approval by the API (co_touched) + if origin not in FEED_ORIGINS: + raise ValueError(f"origin must be visual or drawing, not {origin!r}") + conn = sw.conn + reason = clean_text(reason, LIMITS["reason"]) + pend = pending_visual(conn, sw.id, lock=True) + if pend is not None: + old_vis = dict(pend.get("vis") or {}) + moved = list(old_vis.get("moved") or []) + if origin == "drawing" and pend["origin"] == "visual": + # A drawing re-release meets a pending VISUAL change: stage the drawing on it, keep its content. + new_vis = old_vis + new_vis["moved"] = [*moved, {"at": _now(), "rev": vis.get("to_rev"), "dwg": vis.get("to_dwg")}] + old_reason = str(pend.get("reason") or "") + new_reason = ( + old_reason if reason in old_reason else clean_text(f"{old_reason} · {reason}", LIMITS["reason"]) + ) + conn.execute( + "UPDATE change_orders SET reason = %s, vis = %s, staged = coalesce(%s, staged)," + " waiting = CASE WHEN %s::jsonb IS NULL THEN waiting ELSE NULL END, row_version = row_version + 1" + " WHERE id = %s", + (new_reason, _j(new_vis), _j(staged), _j(staged), pend["id"]), + ) + label = f"Change order {pend['co_no']}: drawing from M-Files staged" + else: + new_vis = dict(vis) + new_vis["moved"] = [*moved, {"at": _now(), "rev": vis.get("to_rev"), "dwg": vis.get("to_dwg")}] + keep = staged is None and pend.get("staged") is not None and _same_target(old_vis, vis) + new_staged = staged if staged is not None else (pend.get("staged") if keep else None) + new_waiting = None if (keep or staged is not None) else waiting + conn.execute( + "UPDATE change_orders SET origin = %s::co_origin, reason = %s, vis = %s, waiting = %s, staged = %s," + " target_fp = %s, minor = %s, ecn = %s, row_version = row_version + 1 WHERE id = %s", + ( + origin, + reason, + _j(new_vis), + _j(new_waiting), + _j(new_staged), + target_fp, + minor, + ECN[origin], + pend["id"], + ), + ) + label = ( + f"Change order {pend['co_no']} updated: VISUAL changed again before Clerical approved" + if origin == "visual" + else f"Change order {pend['co_no']} updated: new drawing from M-Files" + ) + sw.log(label, reason) + sw.changed("change_orders") + sw.detail.update({"co_no": pend["co_no"], "co_id": pend["id"], "reaimed": True}) + return CoResult(co_id=int(pend["id"]), co_no=int(pend["co_no"]), fresh=False, reaimed=True) + + co_no = next_counter(conn, sw.id, "co") + row = conn.execute( + "INSERT INTO change_orders (packet_id, co_no, origin, review, minor, ecn, reason, issued_by, vis, waiting," + " staged, target_fp) VALUES (%s, %s, %s::co_origin, 'pending', %s, %s, %s, NULL, %s, %s, %s, %s) RETURNING id", + (sw.id, co_no, origin, minor, ECN[origin], reason, _j(vis), _j(waiting), _j(staged), target_fp), + ).fetchone() + label = ( + f"Change order {co_no}: change from VISUAL waiting on Clerical approval" + if origin == "visual" + else f"Change order {co_no}: new drawing from M-Files waiting on Clerical approval" + ) + sw.log(label, reason) + sw.changed("change_orders") + sw.detail.update({"co_no": co_no, "co_id": int(row[0]), "fresh": True}) + return CoResult(co_id=int(row[0]), co_no=co_no, fresh=True, reaimed=False) + + +def withdraw(sw: SysWrite, co_id: int, reason: str) -> None: + """A pending feed change order VISUAL no longer asks for: review -> rejected by nobody, with the reason.""" + reason = clean_text(reason, LIMITS["reason"]) + r = sw.conn.execute( + "UPDATE change_orders SET review = 'rejected', rejected_at = now(), rejected_by = NULL, rejected_reason = %s," + " row_version = row_version + 1 WHERE id = %s AND packet_id = %s AND review = 'pending' RETURNING co_no", + (reason, co_id, sw.id), + ).fetchone() + if r is None: + sw.skip() + return + sw.log(f"Change order {r[0]} withdrawn", reason) + sw.changed("change_orders") + sw.detail.update({"co_no": int(r[0]), "co_id": co_id, "withdrawn": True}) + + +# ---------------------------------------------------------------- the human diff (the old visDiff + matDiff) + + +def packet_view(conn: Any, packet_id: int) -> dict[str, Any]: + """What the packet holds now, for `diff_text`: legs, rev, drawing, want, the VISUAL BOM and manual_bom.""" + with conn.cursor(row_factory=dict_row) as cur: + p = cur.execute( + "SELECT rev, drawing, visual_want, visual, status::text AS status FROM packets WHERE id = %s", + (packet_id,), + ).fetchone() + wos = cur.execute( + "SELECT wo_base, lot, split, sub, seq, qty, target_hrs FROM packet_work_orders WHERE packet_id = %s" + " ORDER BY sort, id", + (packet_id,), + ).fetchall() + bom = cur.execute( + "SELECT coalesce(cl_item, item::text) AS item, part_id, qty, src FROM bom_lines WHERE packet_id = %s" + " ORDER BY sort, item", + (packet_id,), + ).fetchall() + p = dict(p or {}) + visual = p.get("visual") or {} + return { + "status": p.get("status"), + "rev": p.get("rev"), + "drawing": p.get("drawing"), + "want": p.get("visual_want"), + "legs": [dict(w) for w in wos], + "bom": [dict(b) for b in bom], + "manual_bom": bool(visual.get("manual_bom")), + } + + +def _n(v: Any) -> str: + return norm_decimal(v) or "0" + + +def _items(items: Sequence[str]) -> str: + head = ", ".join(items[:8]) + return head + ("..." if len(items) > 8 else "") + + +def mat_diff(old: Sequence[Mapping[str, Any]], new: Sequence[Any]) -> list[str]: + """The old matDiff: materials added, removed and changed, item by item.""" + o = {str(b["item"]): b for b in old if b.get("src") == "visual"} + n = {b.item: b for b in new} + hand = sum(1 for b in old if b.get("src") != "visual") + if not o: + if not (hand or n): + return [] + if hand: + return [f"BOM by hand ({hand} line{'' if hand == 1 else 's'}) replaced by VISUAL ({len(n)} lines)"] + return [f"materials: {len(n)} line{'' if len(n) == 1 else 's'} from VISUAL"] + add = [k for k in n if k not in o] + rem = [k for k in o if k not in n] + chg = [ + k + for k in n + if k in o and (str(o[k].get("part_id") or "") != n[k].part_id or _n(o[k].get("qty")) != _n(n[k].qty)) + ] + d: list[str] = [] + if add: + d.append(f"materials added: item{'s' if len(add) > 1 else ''} {_items(add)}") + if rem: + d.append(f"materials removed in VISUAL: item{'s' if len(rem) > 1 else ''} {_items(rem)}") + for k in chg[:8]: + a, b = o[k], n[k] + if str(a.get("part_id") or "") != b.part_id: + d.append(f"item {k}: {a.get('part_id') or '-'} -> {b.part_id or '-'}") + else: + d.append(f"item {k}: qty {_n(a.get('qty'))} -> {_n(b.qty)}") + if len(chg) > 8: + more = len(chg) - 8 + d.append(f"{more} more material change{'' if more == 1 else 's'}") + return d + + +def diff_text(before: Mapping[str, Any], c: Candidate) -> list[str]: + """Human lines for what VISUAL changed on a packet (`before` is `packet_view`).""" + d: list[str] = [] + old = { + leg_key(w["wo_base"], w["lot"], w["split"], w["sub"], w["seq"]): w + for w in before.get("legs") or [] + if w.get("seq") is not None + } + new_keys: set[str] = set() + for lg in c.legs: + k = leg_key(lg.key.base, lg.key.lot, lg.key.split, lg.key.sub, lg.seq) + new_keys.add(k) + o = old.get(k) + if o is None: + d.append(f"work order {lg.key.base} added") + continue + if _n(o.get("target_hrs")) != _n(lg.tgt): + d.append(f"{lg.key.base} target {_n(o.get('target_hrs'))} h -> {_n(lg.tgt)} h") + if _n(o.get("qty")) != _n(lg.qty): + d.append(f"{lg.key.base} qty {_n(o.get('qty'))} -> {_n(lg.qty)}") + for k, o in old.items(): + if k not in new_keys: + d.append(f"work order {o['wo_base']} no longer in VISUAL") + if drawing_key(before.get("drawing")) != drawing_key(c.drawing) and c.drawing: + d.append(f"drawing {before.get('drawing') or '-'} -> {c.drawing}") + if str(before.get("rev") or "") != str(c.rev or ""): + d.append(f"drawing rev {before.get('rev') or '-'} -> {c.rev or '-'}") + want_old = before.get("want") + want_old_s = want_old.isoformat() if isinstance(want_old, date) else str(want_old or "") + want_new_s = c.want.isoformat() if c.want else "" + if want_old_s != want_new_s: + d.append(f"want date {want_old_s or '-'} -> {want_new_s or '-'}") + hand = bool(before.get("manual_bom")) and (before.get("status") == "draft" or not c.bom) + if c.bom is not None and not hand and bom_sig(view_bom(before)) != bom_sig(c.bom): + d.extend(mat_diff(before.get("bom") or [], c.bom)) + return d + + +def view_bom(before: Mapping[str, Any]) -> list[dict[str, Any]]: + """The packet's BOM in matSig shape. Hand lines get a prefix so they always differ from VISUAL's.""" + out = [] + for b in before.get("bom") or []: + item = str(b.get("item")) + out.append( + { + "item": item if b.get("src") == "visual" else f"dwg:{item}", + "part_id": b.get("part_id"), + "qty": b.get("qty"), + } + ) + return out + + +def vis_for( + c: Candidate, fp_parts: Mapping[str, str], before: Mapping[str, Any], changes: list[str], target_fp: str +) -> dict[str, Any]: + """The change order `vis` payload that approval applies.""" + from_dwg = str(before.get("drawing") or "") + from_rev = str(before.get("rev") or "") + to_dwg = c.drawing or from_dwg + to_rev = c.rev or "" + dwg_change = (bool(to_rev) and to_rev != from_rev) or ( + bool(c.drawing) and drawing_key(c.drawing) != drawing_key(from_dwg) + ) + return { + "from_dwg": from_dwg, + "from_rev": from_rev, + "to_dwg": to_dwg, + "to_rev": to_rev, + "dwg_change": dwg_change, + "changes": [x for x in changes if not x.startswith("drawing rev ")], + "sig": target_fp, + "legs": legs_json(c), + "bom": [b.as_dict() for b in c.bom] if c.bom is not None else None, + "want": c.want.isoformat() if c.want else None, + "rel": c.rel.isoformat() if c.rel else None, + "folded": folded_json(c), + "partspecs": list(c.partspecs), + "fp_parts": dict(fp_parts), + } + + +def co_reason(vis: Mapping[str, Any]) -> str: + """The change order reason, as the old coAutoChanges wrote it.""" + if vis.get("dwg_change"): + fd, td = str(vis.get("from_dwg") or ""), str(vis.get("to_dwg") or "") + what = f"{fd} -> {td}" if fd and drawing_key(fd) != drawing_key(td) else td + return f"VISUAL changed the drawing: {what} Rev {vis.get('from_rev') or '?'} -> Rev {vis.get('to_rev')}" + other = list(vis.get("changes") or []) + more = f" (+{len(other) - 3} more)" if len(other) > 3 else "" + return f"VISUAL changed the packet: {'; '.join(other[:3])}{more}" + + +def legs_json(c: Candidate) -> list[dict[str, Any]]: + return [ + { + "wo_base": lg.key.base, + "lot": lg.key.lot, + "split": lg.key.split, + "sub": lg.key.sub, + "seq": lg.seq, + "op": lg.op, + "qty": str(lg.qty), + "target_hrs": str(lg.tgt), + "dwg": lg.dwg, + "rev": lg.rev, + "partspecs": lg.partspecs, + "st": lg.st, + } + for lg in c.legs + ] + + +def folded_json(c: Candidate) -> list[dict[str, Any]]: + return [ + { + "wo_base": f.key.base, + "lot": f.key.lot, + "split": f.key.split, + "sub": f.key.sub, + "seq": f.seq, + "op": f.op, + "target_hrs": str(f.tgt), + } + for f in c.folded + ] diff --git a/server/packetstation/feed/config.py b/server/packetstation/feed/config.py new file mode 100644 index 0000000..223a1e5 --- /dev/null +++ b/server/packetstation/feed/config.py @@ -0,0 +1,226 @@ +"""Feed settings: one `settings` row with key `feed`, a JSON object read by the worker on every fire. + +Unknown keys are ignored on load and missing keys take the default, so the row may be partial. A save +validates, bumps the row version, writes an audit row and NOTIFYs `ps_settings` so the worker reschedules. +""" + +from __future__ import annotations + +import json +import re +from datetime import date, datetime, time, timedelta +from typing import Any + +from psycopg.types.json import Jsonb +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator + +from .. import audit + +KEY = "feed" +SETTINGS_CHANNEL = "ps_settings" +_HHMM = re.compile(r"^([01][0-9]|2[0-3]):[0-5][0-9]$") +_DEPT = re.compile(r"^[A-Za-z0-9]{1,10}$") + + +class SettingsError(ValueError): + """A patch the model refuses. `errors` holds one short line per problem.""" + + def __init__(self, errors: list[str]): + super().__init__("; ".join(errors)) + self.errors = errors + + +class FeedSettings(BaseModel): + model_config = ConfigDict(extra="ignore") + + interval_shop_minutes: int = Field(default=5, ge=1, le=1440) + interval_off_minutes: int = Field(default=30, ge=1, le=1440) + shop_days: list[int] = Field(default_factory=lambda: [0, 1, 2, 3, 4]) # Monday = 0, site time + shop_start: str = "06:00" # HH:MM site time + shop_end: str = "18:00" + window_back_days: int = Field(default=14, ge=0, le=120) + window_ahead_days: int = Field(default=30, ge=0, le=365) + golive_date: date | None = None # work orders released before this date are ignored + departments: list[str] = Field(default_factory=list) # shop codes; empty = every department the map routes to + drawing_recheck_hours: int = Field(default=4, ge=1, le=168) + drawing_budget: int = Field(default=60, ge=1, le=500) + drawing_interval_minutes: int = Field(default=30, ge=1, le=1440) + labor_tickets: bool = True # read labor tickets (at most every 30 minutes) + auto_release: bool = True + auto_release_max: int = Field(default=25, ge=0, le=500) + paused: bool = False # stops scheduled feed, drawing and auto-release runs; "run now" still works + + @field_validator("shop_days") + @classmethod + def _days(cls, v: list[int]) -> list[int]: + if any(d < 0 or d > 6 for d in v): + raise ValueError("shop days are 0 (Monday) to 6 (Sunday)") + return sorted(set(v)) + + @field_validator("shop_start", "shop_end") + @classmethod + def _hhmm(cls, v: str) -> str: + v = v.strip() + if not _HHMM.match(v): + raise ValueError("a time is HH:MM, 00:00 to 23:59") + return v + + @field_validator("departments") + @classmethod + def _depts(cls, v: list[str]) -> list[str]: + out: list[str] = [] + for d in v: + d = str(d).strip() + if not _DEPT.match(d): + raise ValueError("a department is its short code, like 03") + if d not in out: + out.append(d) + return out + + @model_validator(mode="after") + def _hours(self) -> FeedSettings: + if self.shop_start == self.shop_end: + raise ValueError("shop_start and shop_end must differ") + return self + + +FIELDS = tuple(FeedSettings.model_fields) + + +def _clock(hhmm: str) -> time: + h, m = hhmm.split(":") + return time(int(h), int(m)) + + +def _errors(e: ValidationError) -> list[str]: + out = [] + for err in e.errors(): + loc = ".".join(str(x) for x in err.get("loc", ())) + msg = str(err.get("msg", "invalid")).removeprefix("Value error, ") + out.append(f"{loc}: {msg}" if loc else msg) + return out + + +def from_value(value: Any) -> FeedSettings: + """The model from a stored value. A bad stored value keeps its good fields; the rest take the default.""" + if not isinstance(value, dict): + return FeedSettings() + try: + return FeedSettings.model_validate(value) + except ValidationError: + good: dict[str, Any] = {} + for k in FIELDS: + if k not in value: + continue + try: + FeedSettings.model_validate({**good, k: value[k]}) + except ValidationError: + continue + good[k] = value[k] + return FeedSettings.model_validate(good) + + +def load(conn: Any) -> FeedSettings: + """The current feed settings; the defaults when the row is missing.""" + row = conn.execute("SELECT value FROM settings WHERE key = %s", (KEY,)).fetchone() + return from_value(row[0] if row else None) + + +def load_versioned(conn: Any) -> tuple[FeedSettings, int, Any, Any]: + """(settings, version, updated_at, updated_by); version 0 when the row is missing.""" + row = conn.execute("SELECT value, version, updated_at, updated_by FROM settings WHERE key = %s", (KEY,)).fetchone() + if row is None: + return FeedSettings(), 0, None, None + return from_value(row[0]), int(row[1]), row[2], row[3] + + +def seed(conn: Any) -> bool: + """Write the defaults when the row is missing. True when it wrote.""" + cur = conn.execute( + "INSERT INTO settings (key, value, version, updated_at) VALUES (%s, %s, 1, now()) ON CONFLICT (key) DO NOTHING", + (KEY, Jsonb(FeedSettings().model_dump(mode="json"))), + ) + return bool(cur.rowcount) + + +def validate_patch(current: FeedSettings, patch: dict[str, Any]) -> FeedSettings: + """The settings after a partial change. Raises SettingsError, also for a key that is not a setting.""" + unknown = sorted(k for k in patch if k not in FIELDS) + if unknown: + raise SettingsError([f"{k}: not a feed setting" for k in unknown]) + try: + return FeedSettings.model_validate({**current.model_dump(mode="json"), **patch}) + except ValidationError as e: + raise SettingsError(_errors(e)) from None + + +def save(conn: Any, *, who: Any, patch: dict[str, Any]) -> FeedSettings: + """Validate and store a partial change. Raises SettingsError. A patch that changes nothing writes nothing. + + `who` has user_id, session_id, role_snapshot, ip and it_reason (domain.projects.Who). + """ + with conn.transaction(): + row = conn.execute("SELECT value FROM settings WHERE key = %s FOR UPDATE", (KEY,)).fetchone() + current = from_value(row[0] if row else None) + new = validate_patch(current, patch) + before, after = current.model_dump(mode="json"), new.model_dump(mode="json") + changed = {k: {"from": before[k], "to": after[k]} for k in FIELDS if before[k] != after[k]} + if not changed and row is not None: + return new + version = conn.execute( + "INSERT INTO settings (key, value, version, updated_at, updated_by) VALUES (%s, %s, 1, now(), %s)" + " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1," + " updated_at = now(), updated_by = EXCLUDED.updated_by RETURNING version", + (KEY, Jsonb(after), who.user_id), + ).fetchone()[0] + audit.append( + conn, + action="feed.settings_changed", + user_id=who.user_id, + session_id=getattr(who, "session_id", None), + role_snapshot=getattr(who, "role_snapshot", None), + subject_type="settings", + subject_id=KEY, + detail={"changed": changed}, + it_reason=getattr(who, "it_reason", None), + ip=getattr(who, "ip", None), + ) + notify_settings(conn, int(version)) + return new + + +def notify_settings(conn: Any, version: int | None = None) -> None: + """Tell the worker the settings changed. Inside a transaction it is delivered on commit.""" + payload = json.dumps({"key": KEY, "version": version}, separators=(",", ":")) + conn.execute("SELECT ps_notify(%s, %s)", (SETTINGS_CHANNEL, payload)) + + +# ---------------------------------------------------------------- derived values (pure) + + +def window(settings: FeedSettings, today: date) -> tuple[date, date]: + """(since, until) for the released read, from the site-local date.""" + return today - timedelta(days=settings.window_back_days), today + timedelta(days=settings.window_ahead_days) + + +def in_shop_hours(settings: FeedSettings, now_site: datetime) -> bool: + """True on a shop day from shop_start up to shop_end, site time. + + An end before the start spans midnight; the hours after midnight belong to the day the shift started. + """ + start, end = _clock(settings.shop_start), _clock(settings.shop_end) + t = now_site.time().replace(tzinfo=None) + day = now_site.weekday() + if start < end: + return day in settings.shop_days and start <= t < end + if t >= start: + return day in settings.shop_days + if t < end: + return (day - 1) % 7 in settings.shop_days + return False + + +def interval_seconds(settings: FeedSettings, now_site: datetime) -> int: + """The feed's cadence now: the shop interval inside shop hours, the off interval outside.""" + minutes = settings.interval_shop_minutes if in_shop_hours(settings, now_site) else settings.interval_off_minutes + return minutes * 60 diff --git a/server/packetstation/feed/fingerprint.py b/server/packetstation/feed/fingerprint.py new file mode 100644 index 0000000..e661f0f --- /dev/null +++ b/server/packetstation/feed/fingerprint.py @@ -0,0 +1,210 @@ +"""The candidate fingerprint: what VISUAL says about a packet, hashed per part, to decide update, CO or nothing. + +Parts (each the sha256 hex of its canonical JSON): + +- major: `legs` (the sorted leg keys), `qty` (per leg), `rev`, `drawing` (upper-cased, spaces removed), `bom` + (sorted `[item, part_id, qty]`, the old matSig); +- minor: `tgt` (target hours per leg), `want` (the want date). + +Never in it: rel, folded, partspecs, title, statuses, issued, whs, loc, orders, labor, dept, op. A candidate whose +materials are unknown (`bom is None`) has no `bom` part; `classify` skips a part the new side lacks and `merge` +carries the old one forward, so a failed materials read never looks like a BOM change. +""" + +from __future__ import annotations + +import hashlib +import json +import unicodedata +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .candidates import Candidate + +MAJOR = ("legs", "qty", "rev", "drawing", "bom") +MINOR = ("tgt", "want") +PARTS = MAJOR + MINOR + + +# ---------------------------------------------------------------- normalisation + + +def norm_text(value: Any) -> str | None: + """NFC, CRLF and CR to LF, trimmed; '' and None both None.""" + if value is None: + return None + s = value if isinstance(value, str) else str(value) + s = unicodedata.normalize("NFC", s.replace("\r\n", "\n").replace("\r", "\n")).strip() + return s or None + + +def norm_id(value: Any) -> str | None: + """An ID (base, lot, split, sub, part, drawing): normalised text, upper-cased. Stays text.""" + s = norm_text(value) + return s.upper() if s is not None else None + + +def norm_decimal(value: Any) -> str | None: + """A number as text with no exponent and no trailing zeros: 2, 2.0 and Decimal('2.00') are all '2'.""" + if value is None or value == "": + return None + d = value if isinstance(value, Decimal) else Decimal(str(value)) + if d.is_nan() or d.is_infinite(): + return None + if d == 0: + return "0" + s = format(d.normalize(), "f") + if "." in s: + s = s.rstrip("0").rstrip(".") + return s + + +def normalise(value: Any) -> Any: + """The canonical form of a JSON-like value.""" + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + return norm_text(value) + if isinstance(value, int | float | Decimal): + return norm_decimal(value) + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, Mapping): + return {str(k): normalise(v) for k, v in value.items()} + if isinstance(value, list | tuple | set | frozenset): + return [normalise(v) for v in value] + return norm_text(str(value)) + + +def canonical(value: Any) -> bytes: + """json.dumps(sort_keys, compact separators, ensure_ascii False) of the normalised value, UTF-8.""" + return json.dumps(normalise(value), sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def sha256_hex(value: Any) -> str: + return hashlib.sha256(canonical(value)).hexdigest() + + +def drawing_key(dwg: Any) -> str | None: + """A drawing number for comparison: upper-cased, every whitespace removed.""" + s = norm_text(dwg) + if s is None: + return None + s = "".join(s.split()).upper() + return s or None + + +def leg_key(base: Any, lot: Any, split: Any, sub: Any, seq: Any) -> str: + """'BASE|LOT|SPLIT|SUB|SEQ' with the ID parts upper-cased (as text) and seq as an integer.""" + parts = [norm_id(x) or "" for x in (base, lot, split, sub)] + return "|".join([*parts, str(int(seq)) if seq is not None and seq != "" else ""]) + + +# ---------------------------------------------------------------- the fingerprint + + +@dataclass(frozen=True) +class Fingerprint: + full: str + parts: dict[str, str] + + +@dataclass(frozen=True) +class Change: + major: frozenset[str] + minor: frozenset[str] + + @property + def changed(self) -> bool: + return bool(self.major or self.minor) + + @property + def is_minor(self) -> bool: + return bool(self.minor) and not self.major + + +def part_values(c: Candidate) -> dict[str, Any]: + """The normalised value of every part, before hashing (tests and the diff text read these).""" + legs = sorted( + ( + [ + norm_id(lg.key.base) or "", + norm_id(lg.key.lot) or "", + norm_id(lg.key.split) or "", + norm_id(lg.key.sub) or "", + int(lg.seq), + ] + for lg in c.legs + ), + key=lambda x: (x[0], x[1], x[2], x[3], x[4]), + ) + qty: dict[str, Any] = {} + tgt: dict[str, Any] = {} + for lg in c.legs: + k = leg_key(lg.key.base, lg.key.lot, lg.key.split, lg.key.sub, lg.seq) + qty[k] = norm_decimal(lg.qty) + tgt[k] = norm_decimal(lg.tgt) + values: dict[str, Any] = { + "legs": legs, + "qty": qty, + "rev": norm_text(c.rev), + "drawing": drawing_key(c.drawing), + "tgt": tgt, + "want": c.want.isoformat() if c.want else None, + } + if c.bom is not None: + values["bom"] = bom_sig(c.bom) + return values + + +def bom_sig(bom: Any) -> list[list[Any]]: + """The old matSig: sorted [item, part_id, qty] of every line (BomLine objects or dicts).""" + out: list[list[Any]] = [] + for b in bom or []: + get = b.get if isinstance(b, Mapping) else lambda k, _b=b: getattr(_b, k, None) + out.append([norm_text(get("item")) or "", norm_id(get("part_id")) or "", norm_decimal(get("qty"))]) + out.sort(key=lambda x: (x[0], x[1], x[2] or "")) + return out + + +def _full(parts: Mapping[str, str]) -> str: + return sha256_hex(dict(parts)) + + +def compute(c: Candidate) -> Fingerprint: + parts = {name: sha256_hex(v) for name, v in part_values(c).items()} + return Fingerprint(full=_full(parts), parts=parts) + + +def merge(before: Mapping[str, str] | None, after: Fingerprint) -> Fingerprint: + """Carry forward parts the new side does not know (the BOM when materials are unknown this run).""" + if not before: + return after + parts = dict(after.parts) + for name in PARTS: + if name not in parts and name in before: + parts[name] = str(before[name]) + if parts == after.parts: + return after + return Fingerprint(full=_full(parts), parts=parts) + + +def classify(before: Mapping[str, str] | None, after: Mapping[str, str]) -> Change: + """Which parts changed. A part the new side lacks (unknown this run) is skipped. No `before`: all changed.""" + major: set[str] = set() + minor: set[str] = set() + for name in PARTS: + if name not in after: + continue + if before is not None and before.get(name) == after[name]: + continue + (major if name in MAJOR else minor).add(name) + return Change(frozenset(major), frozenset(minor)) diff --git a/server/packetstation/feed/materials.py b/server/packetstation/feed/materials.py new file mode 100644 index 0000000..ea10af6 --- /dev/null +++ b/server/packetstation/feed/materials.py @@ -0,0 +1,291 @@ +"""A packet's BOM from VISUAL's material lines: the old matBom, typed. + +Rules (Seth, 2026-10-08, as the old build): + +- a line belongs to a leg when its work order key and operation seq match the leg and its drawing is blank or + equals the leg's drawing ignoring spaces and case; +- the same piece can come back more than once (one per header status): keep the best, R before F before U before + anything else; +- lines without a part are not BOM lines; a part-less line whose piece is a multiple of 10 is the group header of + the next nine pieces of the same work order (`grp`); +- the item is the piece number, suffixed `-` when the piece repeats; +- WIP sub-assemblies (commodity WIP or warehouse ASY) are listed, marked sub and n/a (no heat needed); +- `loc` is the warehouse (unless MAIN) and the location, distinct and non-empty, joined by a space. + +`bom_lines.item` is an int in the database while the BOM item label is text ("10", "10-5GA"). `db_items` gives +the int per line: the piece number when the label is a plain number used once, else the next free number above +every piece. The text label goes into `cl_item`. `insert_bom` writes lines that way; agent D's "use VISUAL's BOM +again" calls `bom_for_packet` and `insert_bom` so both paths store the same thing. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from decimal import Decimal +from typing import Any + +from psycopg.rows import dict_row + +from .types import Material, WoKey, integer, num, text + +STATUS_RANK = {"R": 0, "F": 1, "U": 2} + +MatIndex = dict[tuple[WoKey, int], list[Material]] + + +@dataclass(slots=True) +class BomLine: + item: str + qty: Decimal + part_id: str + desc: str + loc: str + whs: str + comm: str + cons: bool + sub: bool + na: bool + grp: str | None + vkey: str + issued: Decimal + istat: int | None + + def as_dict(self) -> dict[str, Any]: + """JSON-ready (numbers as text) for change order `vis.bom`.""" + return { + "item": self.item, + "qty": str(self.qty), + "part_id": self.part_id, + "desc": self.desc, + "loc": self.loc, + "whs": self.whs, + "comm": self.comm, + "cons": self.cons, + "sub": self.sub, + "na": self.na, + "grp": self.grp, + "vkey": self.vkey, + "issued": str(self.issued), + "istat": self.istat, + } + + @classmethod + def from_dict(cls, d: Mapping[str, Any]) -> BomLine: + return cls( + item=text(d.get("item")), + qty=num(d.get("qty")), + part_id=text(d.get("part_id")), + desc=text(d.get("desc")), + loc=text(d.get("loc")), + whs=text(d.get("whs")), + comm=text(d.get("comm")), + cons=bool(d.get("cons")), + sub=bool(d.get("sub")), + na=bool(d.get("na")), + grp=text(d.get("grp")) or None, + vkey=text(d.get("vkey")), + issued=num(d.get("issued")), + istat=None if d.get("istat") is None else integer(d.get("istat")), + ) + + +def dwg_norm(dwg: Any) -> str: + return "".join(text(dwg).split()).upper() + + +def index(materials: Iterable[Material] | None) -> MatIndex: + """Material lines by (leg key, seq).""" + idx: MatIndex = {} + for m in materials or (): + idx.setdefault((m.key, m.seq), []).append(m) + return idx + + +def _leg_parts(leg: Any) -> tuple[WoKey, int, str]: + """(key, seq, dwg) from a candidates.Leg, or from a packet_work_orders row dict (wo_base, lot, split, sub, seq, + dwg).""" + if isinstance(leg, Mapping): + key = WoKey.of(leg.get("wo_base"), leg.get("lot"), leg.get("split"), leg.get("sub")) + return key, integer(leg.get("seq")), text(leg.get("dwg")) + return leg.key, int(leg.seq), text(leg.dwg) + + +def _rank(m: Material) -> int: + return STATUS_RANK.get(text(m.st).upper()[:1], 3) + + +def bom_for(legs: Sequence[Any], idx: MatIndex) -> list[BomLine]: + """The BOM of these legs (candidates.Leg objects or packet_work_orders dicts) from the material index.""" + lines: list[Material] = [] + seen: set[tuple[WoKey, int]] = set() + for leg in legs: + key, seq, dwg = _leg_parts(leg) + k = (key, seq) + if k in seen: + continue + seen.add(k) + best: dict[int, Material] = {} + for m in idx.get(k, []): + if m.dwg and dwg and dwg_norm(m.dwg) != dwg_norm(dwg): + continue + o = best.get(m.piece) + if o is None or _rank(m) < _rank(o): + best[m.piece] = m + lines.extend(best.values()) + lines.sort(key=lambda m: (m.wo, m.piece)) # stable: ties keep leg order + out: list[BomLine] = [] + used: set[str] = set() + for m in lines: + if not m.part: + continue + grp = next( + ( + h + for h in lines + if not h.part and h.wo == m.wo and h.piece % 10 == 0 and h.piece < m.piece < h.piece + 10 + ), + None, + ) + item = str(m.piece) + if item in used: + item = f"{item}-{m.wo.split('-')[-1]}" + used.add(item) + sub = m.comm.upper() == "WIP" or m.whs.upper() == "ASY" + locs: list[str] = [] + for part in ("" if m.whs.upper() == "MAIN" else m.whs, m.loc): + if part and part not in locs: + locs.append(part) + out.append( + BomLine( + item=item, + qty=m.qty, + part_id=m.part, + desc=m.desc, + loc=" ".join(locs), + whs=m.whs, + comm=m.comm, + cons=m.cons.upper() == "Y", + sub=sub, + na=sub, + grp=str(grp.piece) if grp is not None else None, + vkey=f"{m.wo}|{m.lot}|{m.sub}|{m.seq}|{m.piece}", + issued=m.issued, + istat=m.istat, + ) + ) + return out + + +def db_items(bom: Sequence[BomLine]) -> list[int]: + """The int `bom_lines.item` per line: the piece when the label is a plain number used once, else the next + free number above every piece number.""" + plain = [b.item for b in bom if b.item.isdigit()] + taken: set[int] = set() + out: list[int | None] = [] + for b in bom: + if b.item.isdigit() and plain.count(b.item) == 1 and int(b.item) not in taken: + n = int(b.item) + taken.add(n) + out.append(n) + else: + out.append(None) + nxt = max(taken, default=0) + for b in bom: + head = b.item.split("-", 1)[0] + if head.isdigit(): + nxt = max(nxt, int(head)) + result: list[int] = [] + for slot in out: + if slot is not None: + result.append(slot) + continue + nxt += 1 + while nxt in taken: + nxt += 1 + taken.add(nxt) + result.append(nxt) + return result + + +def insert_bom(conn: Any, packet_id: int, bom: Sequence[BomLine]) -> int: + """Insert VISUAL BOM lines (src 'visual'). cons is stored 'Y'/'N', sub 'Y' or NULL. Returns the count.""" + items = db_items(bom) + for sort, (b, item) in enumerate(zip(bom, items, strict=True)): + conn.execute( + "INSERT INTO bom_lines (packet_id, item, cl_item, qty, part_id, description, loc, whs, comm, cons, sub," + " grp, vkey, na, issued, src, sort) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s," + " 'visual', %s)", + ( + packet_id, + item, + b.item, + b.qty, + b.part_id[:60], + b.desc[:240] or None, + b.loc[:120] or None, + b.whs or None, + b.comm or None, + "Y" if b.cons else "N", + "Y" if b.sub else None, + b.grp, + b.vkey, + b.na, + b.issued, + sort, + ), + ) + return len(bom) + + +# ---------------------------------------------------------------- from the cache + + +def materials_for(conn: Any, keys: Iterable[WoKey]) -> list[Material]: + """Live cached material lines (gone_batch NULL) of these leg keys.""" + ks = list(dict.fromkeys(keys)) + if not ks: + return [] + with conn.cursor(row_factory=dict_row) as cur: + rows = cur.execute( + "SELECT m.* FROM visual_materials m" + " JOIN unnest(%s::text[], %s::text[], %s::text[], %s::text[]) AS k(b, l, sp, su)" + " ON m.wo_base = k.b AND m.lot = k.l AND m.split = k.sp AND m.sub = k.su" + " WHERE m.gone_batch IS NULL ORDER BY m.wo_base, m.lot, m.split, m.sub, m.seq, m.piece", + ([k.base for k in ks], [k.lot for k in ks], [k.split for k in ks], [k.sub for k in ks]), + ).fetchall() + return [material_from_row(r) for r in rows] + + +def material_from_row(r: Mapping[str, Any]) -> Material: + return Material( + wo=r["wo_base"], + lot=r["lot"], + split=r["split"], + sub=r["sub"], + seq=int(r["seq"]), + piece=int(r["piece"]), + part=r["part"], + qty=num(r["qty"]), + issued=num(r["issued"]), + whs=r["whs"], + loc=r["loc"], + st=r["st"], + istat=r["istat"], + dwg=r["dwg"], + desc=r["descr"], + comm=r["comm"], + cons=r["cons"], + ) + + +def bom_for_packet(conn: Any, packet_id: int) -> list[BomLine]: + """VISUAL's BOM for a packet's legs, rebuilt from the cache (for "use VISUAL's BOM again").""" + with conn.cursor(row_factory=dict_row) as cur: + legs = cur.execute( + "SELECT wo_base, lot, split, sub, seq, coalesce(dwg, '') AS dwg FROM packet_work_orders" + " WHERE packet_id = %s AND seq IS NOT NULL ORDER BY sort, id", + (packet_id,), + ).fetchall() + keys = [WoKey.of(r["wo_base"], r["lot"], r["split"], r["sub"]) for r in legs] + return bom_for(legs, index(materials_for(conn, keys))) diff --git a/server/packetstation/feed/orders.py b/server/packetstation/feed/orders.py new file mode 100644 index 0000000..5014205 --- /dev/null +++ b/server/packetstation/feed/orders.py @@ -0,0 +1,104 @@ +"""Which customer order a project or unit takes its customer, PO and dates from, when VISUAL has several. + +The one rule, in order: + +1. a Clerical pin that names one of the project's orders (`pin`); +2. the base's own linked order, the first of `by_base[base]` (`base`); +3. the project order whose top jobs name the unit, open orders first (`unit_top_job`); +4. the project's first open order, status F, H or R (`project_open`); +5. the project's first order, the newest (`project_newest`). + +`derive` maps the chosen order to the fields a project or unit keeps: customer, po, order_date, ship_date. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date, datetime +from typing import Any + +from .types import Order + +OPEN_STATUSES = ("F", "H", "R") +VIA = ("pin", "base", "unit_top_job", "project_open", "project_newest") +FIELDS = ("customer", "po", "order_date", "ship_date") + + +@dataclass(frozen=True) +class Chosen: + order: Order + via: str # pin | base | unit_top_job | project_open | project_newest + + +def _is_open(o: Order) -> bool: + return (o.status or "").strip().upper() in OPEN_STATUSES + + +def choose( + *, + unit_number: str, + base: str, + by_base: Mapping[str, list[Order]], + by_project: Mapping[str, list[Order]], + project: str, + pin: str | None, +) -> Chosen | None: + proj = list(by_project.get(project) or []) + own = list(by_base.get(base) or []) if base else [] + if pin: + p = pin.strip().upper() + for o in [*proj, *own]: + if o.cust_order_id.strip().upper() == p: + return Chosen(o, "pin") + if own: + return Chosen(own[0], "base") + if unit_number: + u = unit_number.upper() + hits = [o for o in proj if any(t.upper() == u for t in o.top_jobs)] + hits.sort(key=lambda o: not _is_open(o)) # stable: open first, the list's own order within + if hits: + return Chosen(hits[0], "unit_top_job") + for o in proj: + if _is_open(o): + return Chosen(o, "project_open") + if proj: + return Chosen(proj[0], "project_newest") + return None + + +def derive(order: Order) -> dict[str, Any]: + """customer <- customer_name, po <- customer_po_ref, order_date, ship_date <- promise or desired ship date.""" + return { + "customer": order.customer_name, + "po": order.customer_po_ref, + "order_date": order.order_date, + "ship_date": order.promise_date or order.desired_ship_date, + } + + +def _iso(v: date | None) -> str | None: + return v.isoformat() if v else None + + +def order_ref(chosen: Chosen, at: datetime) -> dict[str, Any]: + """The jsonb kept on projects.order_ref and units.order_ref.""" + o = chosen.order + return { + "cust_order_id": o.cust_order_id, + "via": chosen.via, + "status": o.status, + "customer_name": o.customer_name, + "customer_po_ref": o.customer_po_ref, + "order_date": _iso(o.order_date), + "desired_ship_date": _iso(o.desired_ship_date), + "promise_date": _iso(o.promise_date), + "chosen_at": at.isoformat(), + } + + +def same_ref(a: Mapping[str, Any] | None, b: Mapping[str, Any] | None) -> bool: + """Two order_refs are the same when everything but chosen_at matches.""" + if a is None or b is None: + return a is None and b is None + return {k: v for k, v in a.items() if k != "chosen_at"} == {k: v for k, v in b.items() if k != "chosen_at"} diff --git a/server/packetstation/feed/source.py b/server/packetstation/feed/source.py new file mode 100644 index 0000000..1e156ab --- /dev/null +++ b/server/packetstation/feed/source.py @@ -0,0 +1,491 @@ +"""Where the feed reads VISUAL from: VisualKit (DEV unless the stack says otherwise) or a synthetic fixture file. + +`VisualKitSource` copies the read calls and output shapes of VisualKit's examples/packet_station_feed.py: the +window read, the tracked keys by key, operations, operation text and parts for rows; for_bases, requirements, +requirement text, parts and part sites for materials; DEMAND_SUPPLY_LINK orders by base and by project; labor +tickets summed per operation. Read-only: VisualKit enforces it. No cost, price, amount or pay column is read. + +`FixtureSource` reads `/state` and `/feed..json` in the same contract shape (CI and tests). + +Every value goes through `types.parse_row` / `parse_material` / `parse_order`, so both sources give identical +FeedData. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Iterable, Sequence +from datetime import date, datetime +from decimal import Decimal +from pathlib import Path +from typing import Any, Protocol + +from .types import ( + FeedData, + LaborLeg, + Material, + Order, + Row, + WoKey, + parse_labor, + parse_material, + parse_order, + parse_orders, + parse_row, + status_code, + text, +) + +STATUS_WORDS = {"F": "Firmed", "R": "Released", "U": "Unreleased", "C": "Closed", "X": "Cancelled"} +MATERIAL_STATUSES = ("F", "R") +OPEN_ORDER_STATUSES = ("F", "H", "R") +QTY_EMPLOYEE = "9999" + +_SECRETISH = re.compile(r"(?i)(password|pwd|secret|token)\s*[=:]\s*\S+") + + +def redact(message: object) -> str: + """An exception as one short line with anything password-like removed.""" + s = _SECRETISH.sub( + r"\1=***", f"{type(message).__name__}: {message}" if isinstance(message, BaseException) else str(message) + ) + return s.replace("\r", " ").replace("\n", " ")[:500] + + +class FeedSource(Protocol): + def read( + self, + *, + site: str, + since: date, + until: date, + tracked: Sequence[WoKey], + with_materials: bool, + with_orders: bool, + with_labor: bool, + labor_since: date | None, + ) -> FeedData: ... + + def describe(self) -> str: ... + + +# ---------------------------------------------------------------- value helpers (as the VisualKit example) + + +def _to_number(v: Any) -> Any: + if v is None or isinstance(v, bool): + return v + if isinstance(v, Decimal): + return v + return Decimal(str(v)) + + +def _fmt_date(v: date | datetime | None) -> str | None: + if v is None: + return None + if isinstance(v, datetime): + v = v.date() + return v.isoformat() + + +def _clean(s: str | None) -> str: + if not s: + return "" + return s.replace("\r\n", "\n").replace("\r", "\n") + + +def _istat(calc_qty: Any, issued_qty: Any, line_status: str | None) -> int: + """Job Pull's issue state: 3 over-issued, 0 line closed, 2 nothing issued, 1 partly.""" + calc = Decimal(str(calc_qty or 0)) + issued = Decimal(str(issued_qty or 0)) + if issued > calc: + return 3 + if (line_status or "").strip().upper() == "C": + return 0 + if issued == 0: + return 2 + return 1 + + +def _yes_no(flag: bool | None) -> str: + return "" if flag is None else ("Y" if flag else "N") + + +def project_of(base: str) -> str: + """VisualKit's project_of: the base before its first dash, or the whole base.""" + head = base.partition("-")[0] + return head or base + + +# ---------------------------------------------------------------- VisualKit + + +class VisualKitSource: + """The five reads through one `visualkit.Visual` (made by the job per run, closed after).""" + + def __init__(self, visual: Any) -> None: + self.v = visual + + def describe(self) -> str: + s = self.v.settings + return f"{s.server} / {s.database}" + + def _key(self, k: WoKey) -> Any: + from visualkit import WorkOrderKey + + return WorkOrderKey(k.base, k.lot, k.split, k.sub, "W") + + def _load_parts(self, part_ids: Iterable[str | None], cache: dict[str, Any]) -> None: + missing = sorted({p for p in part_ids if p and p not in cache}) + if missing: + cache.update(self.v.parts.get_many(missing)) + + def read( + self, + *, + site: str, + since: date, + until: date, + tracked: Sequence[WoKey], + with_materials: bool, + with_orders: bool, + with_labor: bool, + labor_since: date | None, + ) -> FeedData: + v = self.v + s = v.settings + meta: dict[str, Any] = { + "source": "visualkit", + "server": s.server, + "database": s.database, + "site": site, + "since": since.isoformat(), + "until": until.isoformat(), + "generated_at": datetime.now().isoformat(timespec="seconds"), + } + data = FeedData(meta=meta, rows=[], tracked=list(tracked), tracked_count=len(tracked)) + parts: dict[str, Any] = {} + + # 1. The window: firmed TYPE W work orders released in [since, until], paged. + wos: list[Any] = [] + offset: int | None = 0 + while offset is not None: + page = v.work_orders.released(site, since, until, status=("F",), offset=offset) + wos.extend(page.items) + offset = page.next_offset + data.window_count = len(wos) + + # 2. The tracked keys, whatever their status or date. Keys VISUAL lacks are missing. + asked = {self._key(k): k for k in dict.fromkeys(tracked)} + found = v.work_orders.get_many(list(asked)) if asked else {} + data.tracked_missing = [k for vk, k in asked.items() if vk not in found] + have = {wo.key for wo in wos} + for wo in found.values(): + if wo.key not in have: + wos.append(wo) + have.add(wo.key) + + # 3. Rows for the union. + keys = [wo.key for wo in wos] + ops = v.work_orders.operations_for(keys) if keys else {} + specs = v.work_orders.operation_text_for(keys) if keys else {} + self._load_parts((wo.part_id for wo in wos), parts) + for wo in wos: + part = parts.get(wo.part_id) if wo.part_id else None + desc = _clean(part.description if part else None) + qty = wo.desired_qty or Decimal(0) + for op in ops.get(wo.key, []): + data.rows.append( + parse_row( + { + "wo": wo.base_id, + "lot": wo.lot_id, + "split": wo.split_id, + "sub": wo.sub_id, + "seq": op.sequence_no, + "res": op.resource_id or "", + "dwg": wo.drawing_id or "", + "rev": wo.drawing_rev_no or "", + "ps": specs.get((wo.key, op.sequence_no), ""), + "desc": desc, + "qty": _to_number(qty), + "tgt": _to_number((op.run or Decimal(0)) * qty), + "rel": _fmt_date(wo.desired_rls_date), + "want": _fmt_date(wo.desired_want_date), + "st": STATUS_WORDS.get((wo.status or "").strip().upper(), wo.status or ""), + "barcode": op.barcode(), + "part": wo.part_id or "", + } + ) + ) + + bases = sorted({wo.base_id for wo in wos}) + if with_materials: + try: + data.materials, data.material_legs = self._materials(bases, site, parts) + except Exception as e: # a failed materials read keeps the BOMs as they are + data.materials, data.material_legs = None, None + data.errors.append("materials: " + redact(e)) + if with_orders: + try: + data.orders_by_base = self._orders_by_base(bases) + data.orders_by_project = self._orders_by_project({project_of(b) for b in bases}) + except Exception as e: # the apply keeps the last cached orders + data.orders_by_base, data.orders_by_project = None, None + data.errors.append("orders: " + redact(e)) + if with_labor: + try: + data.labor = self._labor(list(asked) or keys) + except Exception as e: + data.labor = None + data.errors.append("labor: " + redact(e)) + return data + + def _materials(self, bases: Sequence[str], site: str, parts: dict[str, Any]) -> tuple[list[Material], set[WoKey]]: + v = self.v + if not bases: + return [], set() + legs = v.work_orders.for_bases(bases, type="W", status=MATERIAL_STATUSES, site_id=site) + keys = [wo.key for wo in legs] + reqs = v.work_orders.requirements_for(keys) if keys else {} + text_keys = [k for k in keys if any(not r.part_id for r in reqs.get(k, []))] + texts = v.work_orders.requirement_text_for(text_keys) if text_keys else {} + part_ids = {r.part_id for rs in reqs.values() for r in rs if r.part_id} + self._load_parts(part_ids, parts) + sites = v.parts.sites_for(sorted(part_ids), site) if part_ids else {} + out: list[Material] = [] + covered: set[WoKey] = set() + for wo in legs: + covered.add(WoKey.of(wo.base_id, wo.lot_id, wo.split_id, wo.sub_id)) + for r in reqs.get(wo.key, []): + part = parts.get(r.part_id) if r.part_id else None + ps = sites.get(r.part_id) if r.part_id else None + desc = ( + _clean(part.description if part else None) + if r.part_id + else _clean(texts.get((wo.key, r.operation_seq_no, r.piece_no))) + ) + out.append( + parse_material( + { + "wo": wo.base_id, + "lot": wo.lot_id, + "split": wo.split_id, + "sub": wo.sub_id, + "seq": r.operation_seq_no, + "piece": r.piece_no, + "part": r.part_id or None, + "qty": _to_number(r.calc_qty or Decimal(0)), + "issued": _to_number(r.issued_qty or Decimal(0)), + "whs": (ps.primary_whs_id or "").upper() if ps else "", + "loc": (ps.primary_loc_id or "").upper() if ps else "", + "st": wo.status, + "istat": _istat(r.calc_qty, r.issued_qty, r.status), + "dwg": wo.drawing_id or "", + "desc": desc, + "comm": (part.commodity_code or "") if part else "", + "cons": _yes_no(part.consumable) if part else "", + } + ) + ) + return out, covered + + def _orders_by_base(self, bases: Sequence[str]) -> dict[str, list[Order]]: + v = self.v + if not bases: + return {} + pairs = v.links.orders_for_bases(list(bases)) + order_ids = sorted({oid for ps in pairs.values() for oid, _ in ps}) + orders = v.orders.get_many(order_ids) if order_ids else {} + customers = v.orders.customers(sorted({o.customer_id for o in orders.values()})) if orders else {} + out: dict[str, list[Order]] = {} + for base in bases: + lines_by_order: dict[str, list[int]] = {} + for oid, line_no in pairs.get(base, []): + lines_by_order.setdefault(oid, []) + if line_no is not None: + lines_by_order[oid].append(line_no) + if not lines_by_order: + continue + out[base] = [] + for oid, lines in sorted(lines_by_order.items()): + o = orders.get(oid) + cust = customers.get(o.customer_id) if o else None + out[base].append( + parse_order( + { + "cust_order_id": oid, + "lines": sorted(set(lines)), + "status": o.status if o else None, + "customer_id": o.customer_id if o else None, + "customer_name": cust.name if cust else None, + "customer_po_ref": o.customer_po_ref if o else None, + "order_date": _fmt_date(getattr(o, "order_date", None)) if o else None, + "desired_ship_date": _fmt_date(o.desired_ship_date) if o else None, + "promise_date": _fmt_date(o.promise_date) if o else None, + } + ) + ) + return out + + def _orders_by_project(self, projects: Iterable[str]) -> dict[str, list[Order]]: + v = self.v + wanted = sorted(set(projects)) + if not wanted: + return {} + found = v.links.orders_for_projects(wanted) + cust_ids = sorted({po.customer_id for pos in found.values() for po in pos if po.customer_id}) + customers = v.orders.customers(cust_ids) if cust_ids else {} + out: dict[str, list[Order]] = {} + for project in wanted: + entries: dict[str, dict[str, Any]] = {} + for po in found.get(project, []): + e = entries.get(po.cust_order_id) + if e is None: + cust = customers.get(po.customer_id) if po.customer_id else None + e = entries[po.cust_order_id] = { + "cust_order_id": po.cust_order_id, + "lines": [], + "top_jobs": [], + "status": po.status, + "customer_id": po.customer_id, + "customer_name": cust.name if cust else None, + "customer_po_ref": po.customer_po_ref, + "order_date": _fmt_date(po.order_date), + "desired_ship_date": _fmt_date(po.desired_ship_date), + "promise_date": _fmt_date(po.promise_date), + } + if po.line_no is not None and po.line_no not in e["lines"]: + e["lines"].append(po.line_no) + if po.supply_base_id not in e["top_jobs"]: + e["top_jobs"].append(po.supply_base_id) + if entries: + for e in entries.values(): + e["lines"].sort() + e["top_jobs"].sort() + ordered = sorted(entries.values(), key=lambda e: e["status"] not in OPEN_ORDER_STATUSES) + out[project] = [parse_order(e) for e in ordered] + return out + + def _labor(self, keys: Sequence[Any]) -> list[LaborLeg]: + """Hours and ticket counts per (leg, seq). Employee 9999's quantity-only tickets and indirect tickets + are left out. Hours only: no cost or pay column is read. VisualKit has no date filter on this call, so + every ticket of the legs is read and summed (the totals need them all).""" + if not keys: + return [] + tickets = self.v.labor.tickets_for_work_orders(list(keys)) + sums: dict[tuple[WoKey, int], LaborLeg] = {} + for vk, ts in tickets.items(): + k = WoKey.of(vk.base_id, vk.lot_id, vk.split_id, vk.sub_id) + for t in ts: + if t.operation_seq_no is None or t.is_quantity_only or t.is_indirect: + continue + seq = int(t.operation_seq_no) + leg = sums.get((k, seq)) + if leg is None: + leg = sums[(k, seq)] = LaborLeg(k.base, k.lot, k.split, k.sub, seq, Decimal(0), 0, None) + leg.hours += Decimal(str(t.hours_worked or 0)) + leg.tickets += 1 + d = t.transaction_date.date() if isinstance(t.transaction_date, datetime) else t.transaction_date + if d is not None and (leg.last_ticket is None or d > leg.last_ticket): + leg.last_ticket = d + return list(sums.values()) + + +# ---------------------------------------------------------------- fixtures + + +class FixtureSource: + """Synthetic VISUAL: `/state` (default "1") picks `/feed..json`.""" + + def __init__(self, directory: Path) -> None: + self.dir = Path(directory) + + def state(self) -> str: + try: + s = (self.dir / "state").read_text(encoding="utf-8").strip() + except OSError: + s = "" + return s or "1" + + def describe(self) -> str: + return f"fixture state {self.state()}" + + def read( + self, + *, + site: str, + since: date, + until: date, + tracked: Sequence[WoKey], + with_materials: bool, + with_orders: bool, + with_labor: bool, + labor_since: date | None, + ) -> FeedData: + state = self.state() + raw = json.loads((self.dir / f"feed.{state}.json").read_text(encoding="utf-8")) + all_rows: list[Row] = [parse_row(d) for d in raw.get("rows") or []] + tracked_set = set(tracked) + window_keys = {r.key for r in all_rows if r.st == "F" and r.rel is not None and since <= r.rel <= until} + keep = window_keys | (tracked_set & {r.key for r in all_rows}) + rows = [r for r in all_rows if r.key in keep] + present = {r.key for r in all_rows} + meta = dict(raw.get("meta") or {}) + meta.update( + {"source": "fixture", "state": state, "site": site, "since": since.isoformat(), "until": until.isoformat()} + ) + data = FeedData( + meta=meta, + rows=rows, + tracked=list(tracked), + tracked_missing=[k for k in dict.fromkeys(tracked) if k not in present], + window_count=len(window_keys), + tracked_count=len(tracked), + ) + bases = {r.wo for r in rows} + if with_materials: + if raw.get("materials") is None: + data.materials, data.material_legs = None, None + else: + lines = [parse_material(d) for d in raw.get("materials") or []] + data.materials = [m for m in lines if m.wo in bases] + data.material_legs = {m.key for m in data.materials if m.st in MATERIAL_STATUSES} + # A leg the file lists with status F or R counts as covered even when it has no lines. + for r in all_rows: + if r.wo in bases and r.st in MATERIAL_STATUSES: + data.material_legs.add(r.key) + if with_orders: + obb = parse_orders(raw.get("orders_by_base")) + obp = parse_orders(raw.get("orders_by_project")) + if obb is None or obp is None: + data.orders_by_base, data.orders_by_project = None, None + else: + projects = {project_of(b) for b in bases} + data.orders_by_base = {b: v for b, v in obb.items() if b in bases} + data.orders_by_project = {p: v for p, v in obp.items() if p in projects} + if with_labor and raw.get("labor") is not None: + data.labor = [parse_labor(d) for d in raw.get("labor") or []] + return data + + +def make(settings: Any, visual: Any = None) -> FeedSource: + """The source the settings name. `visualkit` needs the open `visual` the job made.""" + if settings.feed_source == "fixture": + return FixtureSource(Path(settings.fixture_dir)) + if visual is None: + raise ValueError("the visualkit feed source needs an open visualkit.Visual") + return VisualKitSource(visual) + + +__all__ = [ + "FeedSource", + "FixtureSource", + "VisualKitSource", + "make", + "project_of", + "redact", + "status_code", + "text", +] diff --git a/server/packetstation/feed/types.py b/server/packetstation/feed/types.py new file mode 100644 index 0000000..bc9f389 --- /dev/null +++ b/server/packetstation/feed/types.py @@ -0,0 +1,306 @@ +"""The feed's typed records: one VISUAL operation row, one material line, one customer order, labor per leg. + +Both feed sources (VisualKit and the fixture files) build these through `parse_row` and `parse_material`, so they +produce identical data. Every ID part (work order base, lot, split, sub, part id, order id) is text and is never +cast to a number: leading zeros stay and lot `7A` occurs. `seq` and `piece` are int. Quantities are Decimal. +`None` and `""` are the same for text. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import date, datetime +from decimal import Decimal, InvalidOperation +from typing import Any + +STATUS_CODES = { + "FIRMED": "F", + "RELEASED": "R", + "UNRELEASED": "U", + "CLOSED": "C", + "CANCELLED": "X", + "CANCELED": "X", +} + + +@dataclass(frozen=True, slots=True) +class WoKey: + """A work order leg key, text exactly as VISUAL has it ('' never None).""" + + base: str + lot: str + split: str + sub: str + + def __str__(self) -> str: + return f"{self.base}/{self.lot}/{self.split}/{self.sub}" + + def as_tuple(self) -> tuple[str, str, str, str]: + return (self.base, self.lot, self.split, self.sub) + + @classmethod + def of(cls, base: Any, lot: Any, split: Any, sub: Any) -> WoKey: + return cls(text(base), text(lot), text(split), text(sub)) + + +@dataclass(slots=True) +class Row: + """One TYPE W operation, as Job Release Information carries it.""" + + wo: str + lot: str + split: str + sub: str + seq: int + res: str + dwg: str + rev: str + ps: str + desc: str + qty: Decimal + tgt: Decimal + rel: date | None + want: date | None + st: str # the status CODE: F R U C X, or '' + part: str = "" + barcode: str = "" + + @property + def key(self) -> WoKey: + return WoKey(self.wo, self.lot, self.split, self.sub) + + +@dataclass(slots=True) +class Material: + """One requirement line, as Job Pull carries it. part is None for a line with no part (a group header).""" + + wo: str + lot: str + split: str + sub: str + seq: int + piece: int + part: str | None + qty: Decimal + issued: Decimal + whs: str + loc: str + st: str + istat: int | None + dwg: str + desc: str + comm: str + cons: str + + @property + def key(self) -> WoKey: + return WoKey(self.wo, self.lot, self.split, self.sub) + + +@dataclass(slots=True) +class Order: + cust_order_id: str + status: str | None + customer_id: str | None + customer_name: str | None + customer_po_ref: str | None + order_date: date | None + desired_ship_date: date | None + promise_date: date | None + lines: list[int] = field(default_factory=list) + top_jobs: list[str] = field(default_factory=list) + + +@dataclass(slots=True) +class LaborLeg: + wo: str + lot: str + split: str + sub: str + seq: int + hours: Decimal + tickets: int + last_ticket: date | None + + @property + def key(self) -> WoKey: + return WoKey(self.wo, self.lot, self.split, self.sub) + + +@dataclass(slots=True) +class FeedData: + """One read of VISUAL (or of a fixture file). None on a section means that read failed or was not asked for.""" + + meta: dict[str, Any] + rows: list[Row] + materials: list[Material] | None = None + material_legs: set[WoKey] | None = None # legs the materials read covered (header status F or R) + orders_by_base: dict[str, list[Order]] | None = None + orders_by_project: dict[str, list[Order]] | None = None + labor: list[LaborLeg] | None = None + tracked: list[WoKey] = field(default_factory=list) # the tracked keys this read was asked for + tracked_missing: list[WoKey] = field(default_factory=list) + window_count: int = 0 # work orders the window read returned + tracked_count: int = 0 + errors: list[str] = field(default_factory=list) + + +# ---------------------------------------------------------------- value helpers + + +def text(value: Any) -> str: + """Text as stored: None and '' alike become ''. CRLF and CR become LF. Never a number cast.""" + if value is None: + return "" + s = value if isinstance(value, str) else str(value) + return s.replace("\r\n", "\n").replace("\r", "\n").strip() + + +def opt_text(value: Any) -> str | None: + s = text(value) + return s or None + + +def num(value: Any) -> Decimal: + """A quantity as Decimal, through str so 0.1 stays 0.1. None, '' and junk become 0.""" + if value is None or value == "" or isinstance(value, bool): + return Decimal(0) + if isinstance(value, Decimal): + return value + try: + return Decimal(str(value)) + except (InvalidOperation, ValueError): + return Decimal(0) + + +def opt_num(value: Any) -> Decimal | None: + if value is None or value == "": + return None + return num(value) + + +def day(value: Any) -> date | None: + """A date from a date, datetime or ISO text (the first 10 characters). Anything else is None.""" + if value is None or value == "": + return None + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + try: + return date.fromisoformat(str(value).strip()[:10]) + except ValueError: + return None + + +def integer(value: Any, default: int = 0) -> int: + """seq and piece: whole numbers. 10, 10.0, '10' and Decimal('10') are all 10.""" + if value is None or value == "" or isinstance(value, bool): + return default + if isinstance(value, int): + return value + try: + return int(Decimal(str(value).strip())) + except (InvalidOperation, ValueError): + return default + + +def opt_int(value: Any) -> int | None: + if value is None or value == "": + return None + return integer(value) + + +def status_code(word_or_code: Any) -> str: + """'Firmed' or 'F' -> 'F', and so on. Unknown -> its first letter upper-cased. None and '' -> ''.""" + s = text(word_or_code).upper() + if not s: + return "" + if s in STATUS_CODES: + return STATUS_CODES[s] + return s[0] + + +# ---------------------------------------------------------------- parsing contract dicts + + +def parse_row(d: Mapping[str, Any]) -> Row: + """A Row from a feed-contract dict (examples/packet_station_feed.py `rows`).""" + return Row( + wo=text(d.get("wo")), + lot=text(d.get("lot")), + split=text(d.get("split")), + sub=text(d.get("sub")), + seq=integer(d.get("seq")), + res=text(d.get("res")), + dwg=text(d.get("dwg")), + rev=text(d.get("rev")), + ps=text(d.get("ps")), + desc=text(d.get("desc")), + qty=num(d.get("qty")), + tgt=num(d.get("tgt")), + rel=day(d.get("rel")), + want=day(d.get("want")), + st=status_code(d.get("st")), + part=text(d.get("part")), + barcode=text(d.get("barcode")), + ) + + +def parse_material(d: Mapping[str, Any]) -> Material: + """A Material from a feed-contract dict (`materials`). A missing part stays None.""" + return Material( + wo=text(d.get("wo")), + lot=text(d.get("lot")), + split=text(d.get("split")), + sub=text(d.get("sub")), + seq=integer(d.get("seq")), + piece=integer(d.get("piece")), + part=opt_text(d.get("part")), + qty=num(d.get("qty")), + issued=num(d.get("issued")), + whs=text(d.get("whs")), + loc=text(d.get("loc")), + st=status_code(d.get("st")), + istat=opt_int(d.get("istat")), + dwg=text(d.get("dwg")), + desc=text(d.get("desc")), + comm=text(d.get("comm")), + cons=text(d.get("cons")), + ) + + +def parse_order(d: Mapping[str, Any]) -> Order: + """An Order from an orders_by_base or orders_by_project entry.""" + return Order( + cust_order_id=text(d.get("cust_order_id")), + status=opt_text(d.get("status")), + customer_id=opt_text(d.get("customer_id")), + customer_name=opt_text(d.get("customer_name")), + customer_po_ref=opt_text(d.get("customer_po_ref")), + order_date=day(d.get("order_date")), + desired_ship_date=day(d.get("desired_ship_date")), + promise_date=day(d.get("promise_date")), + lines=[integer(x) for x in (d.get("lines") or []) if x is not None and x != ""], + top_jobs=[text(x) for x in (d.get("top_jobs") or []) if text(x)], + ) + + +def parse_orders(m: Mapping[str, Any] | None) -> dict[str, list[Order]] | None: + if m is None: + return None + return {text(k): [parse_order(e) for e in (v or [])] for k, v in m.items() if text(k)} + + +def parse_labor(d: Mapping[str, Any]) -> LaborLeg: + return LaborLeg( + wo=text(d.get("wo")), + lot=text(d.get("lot")), + split=text(d.get("split")), + sub=text(d.get("sub")), + seq=integer(d.get("seq")), + hours=num(d.get("hours")), + tickets=integer(d.get("tickets")), + last_ticket=day(d.get("last_ticket")), + ) diff --git a/server/packetstation/feed/writes.py b/server/packetstation/feed/writes.py new file mode 100644 index 0000000..24399a0 --- /dev/null +++ b/server/packetstation/feed/writes.py @@ -0,0 +1,216 @@ +"""Packet writes by the worker: the pipeline's packet_write for the system principal, with no session. + +`system_write` locks the packet row (FOR UPDATE SKIP LOCKED: a row a person is writing right now is skipped and +retried next run, `Busy`), builds the policy's PacketState from the same child rows the API uses, asks the policy +as `Principal(user_id=None, system="visual-feed")` (`Refused` when it says no), yields a `SysWrite`, then bumps +the version, writes the packet_log row (by NULL, by_name 'VISUAL feed', role 'system'), the audit row +(`packet.`, role_snapshot 'system') and the NOTIFY. Everything runs in a savepoint (or a transaction when +none is open), so a refusal or an error inside the body leaves nothing behind. + +VIS_AUTO_RELEASE carries the SEND_TO_WAREHOUSE side effects here, locally: status warehouse, status_at, +wh_skipped false, wh_sent_at, a packet_wh incoming row (by NULL, auto true) and a packet_transitions row (by NULL, +action 'vis_auto_release'). +""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import Any + +from psycopg.rows import dict_row + +from .. import audit, events, policy +from ..domain import packets +from ..text import LIMITS, clean_opt + +A = policy.Action +SYSTEM = policy.Principal(user_id=None, system="visual-feed") +BY_NAME = "VISUAL feed" +ROLE = "system" + +LABELS: dict[policy.Action, str] = { + A.VIS_UPDATE_DRAFT: "Updated from VISUAL", + A.VIS_OPEN_CO: "Change from VISUAL", + A.VIS_AUTO_RELEASE: "Sent to warehouse automatically", + A.VIS_RETIRE_DRAFT: "Retired: work orders moved to another packet", +} + +_SELECT = ( + "SELECT *, status::text AS status, source_kind::text AS source_kind, routed_to::text AS routed_to," + " drawing_state::text AS drawing_state FROM packets WHERE id = %s FOR UPDATE SKIP LOCKED" +) + +EMPTY: Mapping[str, Any] = {} + + +class Busy(Exception): + """Someone else holds the packet row right now. Retry next run.""" + + +class Missing(LookupError): + """No such packet.""" + + +class Refused(Exception): + """The policy said no.""" + + def __init__(self, decision: policy.Decision) -> None: + super().__init__(f"{decision.code}: {decision.why}") + self.decision = decision + + +@dataclass +class SysWrite: + conn: Any + packet: dict[str, Any] # the packets row, locked + state: policy.PacketState + action: policy.Action + label: str + note: str | None = None + detail: dict[str, Any] = field(default_factory=dict) + fields: list[str] = field(default_factory=list) + noop: bool = False + version: int = 0 + transition_id: int | None = None + + @property + def id(self) -> int: + return int(self.packet["id"]) + + def changed(self, *names: str) -> None: + for n in names: + if n not in self.fields: + self.fields.append(n) + + def log(self, label: str, note: str | None = None) -> None: + """Override the packet_log label (and note).""" + self.label = label + if note is not None: + self.note = note + + def skip(self) -> None: + """Nothing changed: no version bump, no log, audit or event.""" + self.noop = True + + +def lock(conn: Any, packet_id: int) -> dict[str, Any]: + """The packets row locked FOR UPDATE SKIP LOCKED. Busy when another transaction holds it.""" + with conn.cursor(row_factory=dict_row) as cur: + row = cur.execute(_SELECT, (packet_id,)).fetchone() + if row is not None: + return dict(row) + if conn.execute("SELECT 1 FROM packets WHERE id = %s", (packet_id,)).fetchone() is None: + raise Missing(packet_id) + raise Busy(packet_id) + + +@contextmanager +def system_write( + conn: Any, + packet_id: int, + action: policy.Action, + *, + label: str | None = None, + note: str | None = None, + detail: dict[str, Any] | None = None, + extra: Mapping[str, Any] = EMPTY, +) -> Iterator[SysWrite]: + with conn.transaction(): + row = lock(conn, packet_id) + ch = packets.load_children(conn, row, full=False) + state = packets.state_from(row, ch) + decision = policy.check(SYSTEM, state, action, extra=extra) + if not decision.ok: + raise Refused(decision) + sw = SysWrite( + conn=conn, + packet=row, + state=state, + action=action, + label=label or LABELS.get(action, str(action)), + note=note, + detail=dict(detail or {}), + version=int(row["version"]), + ) + yield sw + _finish(sw) + + +def _finish(sw: SysWrite) -> None: + if sw.noop: + return + conn = sw.conn + from_status = str(sw.packet["status"]) + from_dept = str(sw.packet["dept_code"]) + if sw.action is A.VIS_AUTO_RELEASE: + _auto_release(sw) + sw.version = int( + conn.execute("UPDATE packets SET version = version + 1 WHERE id = %s RETURNING version", (sw.id,)).fetchone()[0] + ) + after = conn.execute("SELECT status::text, dept_code FROM packets WHERE id = %s", (sw.id,)).fetchone() + status, dept = str(after[0]), str(after[1]) + note = clean_opt(sw.note, LIMITS["note"]) if sw.note else None + conn.execute( + "INSERT INTO packet_log (packet_id, by, by_name, role, action, note, transition_id)" + " VALUES (%s, NULL, %s, %s, %s, %s, %s)", + (sw.id, BY_NAME, ROLE, sw.label, note, sw.transition_id), + ) + detail: dict[str, Any] = {"from": from_status, "to": status, "dept": dept} + if from_dept != dept: + detail["from_dept"] = from_dept + if sw.fields: + detail["fields"] = sw.fields + if note: + detail["note"] = note + detail.update(sw.detail) + audit.append( + conn, + action=f"packet.{sw.action}", + role_snapshot=ROLE, + subject_type="packet", + subject_id=sw.id, + packet_id=sw.id, + detail=packets_jsonable(detail), + ) + events.publish( + conn, {"event": "changed", "packet_id": sw.id, "version": sw.version, "status": status, "dept": dept} + ) + + +def _auto_release(sw: SysWrite) -> None: + conn = sw.conn + conn.execute( + "UPDATE packets SET status = 'warehouse', status_at = now(), wh_skipped = false, wh_sent_at = now()," + " return_note = NULL WHERE id = %s", + (sw.id,), + ) + conn.execute("UPDATE packet_wh SET closed_at = now() WHERE packet_id = %s AND closed_at IS NULL", (sw.id,)) + conn.execute( + "INSERT INTO packet_wh (packet_id, stage, by, note, auto) VALUES (%s, 'incoming', NULL, %s, true)", + (sw.id, "Sent automatically: no flags"), + ) + dept = str(sw.packet["dept_code"]) + tid = conn.execute( + "INSERT INTO packet_transitions (packet_id, by, action, from_status, to_status, from_dept, to_dept, reason)" + " VALUES (%s, NULL, 'vis_auto_release', %s::packet_status, 'warehouse', %s, %s, %s) RETURNING id", + (sw.id, str(sw.packet["status"]), dept, dept, sw.note), + ).fetchone() + sw.transition_id = int(tid[0]) + sw.changed("status") + + +def packets_jsonable(value: Any) -> Any: + """Audit detail as plain JSON (dates and Decimals as text).""" + from fastapi.encoders import jsonable_encoder + + return jsonable_encoder(value) + + +def log_only(conn: Any, packet_id: int, label: str, note: str | None = None) -> None: + """A packet_log row by the feed with no version bump (for observations that change nothing a user edits).""" + conn.execute( + "INSERT INTO packet_log (packet_id, by, by_name, role, action, note) VALUES (%s, NULL, %s, %s, %s, %s)", + (packet_id, BY_NAME, ROLE, label, clean_opt(note, LIMITS["note"]) if note else None), + ) diff --git a/server/packetstation/main.py b/server/packetstation/main.py index bc29136..4fade7a 100644 --- a/server/packetstation/main.py +++ b/server/packetstation/main.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio import contextlib +import importlib import logging from collections.abc import AsyncIterator from pathlib import Path @@ -93,13 +94,48 @@ def create_app( app.state.groups_cache = None api_errors.install(app) - from .api import account, admin, assets, auth, departments, packet_children, packets, projects, units + from .api import ( + account, + admin, + assets, + auth, + departments, + feed_settings, + jobs, + packet_children, + packets, + projects, + units, + ) from .api import events as events_api # Routers go in before the static catch-all below, which answers every other GET. - for module in (auth, account, admin, departments, events_api, projects, units, packets, packet_children, assets): + for module in ( + auth, + account, + admin, + departments, + events_api, + projects, + units, + packets, + packet_children, + assets, + jobs, + feed_settings, + ): app.include_router(module.router) app.include_router(assets.blob_router) + for name in ("feed", "packet_visual"): + modname = f"{__package__}.api.{name}" + try: + extra = importlib.import_module(modname) + except ModuleNotFoundError as e: + if e.name != modname: + raise + log.warning("router api.%s not there yet; not loaded", name) + continue + app.include_router(extra.router) @app.get("/healthz") def healthz() -> dict[str, Any]: diff --git a/server/packetstation/migrations/0004_feed.sql b/server/packetstation/migrations/0004_feed.sql new file mode 100644 index 0000000..779a2b5 --- /dev/null +++ b/server/packetstation/migrations/0004_feed.sql @@ -0,0 +1,422 @@ +-- 0004_feed: the VISUAL and drawing caches, the go-live ignored list, the resource map, the worker's +-- job tables and the order-derived columns on projects and units. Runs as ps_migrate; 0001's default +-- privileges give ps_app SELECT, INSERT, UPDATE, DELETE on every table here. Every work order ID part +-- (base, lot, split, sub) and every part ID is text: leading zeros are kept and lot 7A occurs. + +-- ---------------------------------------------------------------- VISUAL cache + +-- One row per feed read. suspect batches skip the "gone" marking. +CREATE TABLE visual_batches ( + id bigserial PRIMARY KEY, + run_id bigint, + started_at timestamptz NOT NULL DEFAULT now(), + finished_at timestamptz, + server text, + database text, + site text, + since date, + until date, + window_rows int, + tracked_keys int, + missing_keys int, + suspect bool NOT NULL DEFAULT false, + counts jsonb NOT NULL DEFAULT '{}' +); + +-- One row per TYPE W operation as VISUAL last showed it. st is the status code (F R U C X) or ''. +CREATE TABLE visual_rows ( + id bigserial PRIMARY KEY, + wo_base text NOT NULL, + lot text NOT NULL DEFAULT '', + split text NOT NULL DEFAULT '', + sub text NOT NULL DEFAULT '', + seq int NOT NULL, + res text NOT NULL DEFAULT '', + dwg text NOT NULL DEFAULT '', + rev text NOT NULL DEFAULT '', + ps text NOT NULL DEFAULT '', + descr text NOT NULL DEFAULT '', + qty numeric, + tgt numeric, + rel date, + want date, + st text NOT NULL DEFAULT '', + part text NOT NULL DEFAULT '', + barcode text, + row_hash bytea NOT NULL, + first_batch bigint NOT NULL REFERENCES visual_batches (id), + last_batch bigint NOT NULL REFERENCES visual_batches (id), + gone_batch bigint REFERENCES visual_batches (id), + UNIQUE (wo_base, lot, split, sub, seq) +); + +CREATE INDEX visual_rows_wo ON visual_rows (wo_base, lot); +CREATE INDEX visual_rows_dwg ON visual_rows (dwg); +CREATE INDEX visual_rows_rel_live ON visual_rows (rel) WHERE gone_batch IS NULL; + +-- Material requirement lines. part is NULL on a line with no part (a group header). +CREATE TABLE visual_materials ( + id bigserial PRIMARY KEY, + wo_base text NOT NULL, + lot text NOT NULL DEFAULT '', + split text NOT NULL DEFAULT '', + sub text NOT NULL DEFAULT '', + seq int NOT NULL, + piece int NOT NULL, + part text, + qty numeric, + issued numeric, + whs text NOT NULL DEFAULT '', + loc text NOT NULL DEFAULT '', + st text NOT NULL DEFAULT '', + istat int, + dwg text NOT NULL DEFAULT '', + descr text NOT NULL DEFAULT '', + comm text NOT NULL DEFAULT '', + cons text NOT NULL DEFAULT '', + mat_hash bytea NOT NULL, + first_batch bigint NOT NULL REFERENCES visual_batches (id), + last_batch bigint NOT NULL REFERENCES visual_batches (id), + gone_batch bigint REFERENCES visual_batches (id), + UNIQUE (wo_base, lot, split, sub, seq, piece) +); + +CREATE INDEX visual_materials_wo ON visual_materials (wo_base, lot); + +CREATE TABLE visual_orders ( + cust_order_id text PRIMARY KEY, + status text, + customer_id text, + customer_name text, + customer_po_ref text, + order_date date, + desired_ship_date date, + promise_date date, + last_batch bigint +); + +-- kind base: key is a work order base. kind project: key is a project number. +CREATE TABLE visual_order_links ( + id bigserial PRIMARY KEY, + kind text NOT NULL CHECK (kind IN ('base', 'project')), + key text NOT NULL, + cust_order_id text NOT NULL REFERENCES visual_orders (cust_order_id), + rank int NOT NULL, + lines int[] NOT NULL DEFAULT '{}', + top_jobs text[] NOT NULL DEFAULT '{}', + last_batch bigint, + UNIQUE (kind, key, cust_order_id) +); + +CREATE INDEX visual_order_links_key ON visual_order_links (kind, key); + +-- Hours and ticket counts per operation. No cost columns, ever. +CREATE TABLE visual_labor ( + id bigserial PRIMARY KEY, + wo_base text NOT NULL, + lot text NOT NULL DEFAULT '', + split text NOT NULL DEFAULT '', + sub text NOT NULL DEFAULT '', + seq int NOT NULL, + hours numeric NOT NULL DEFAULT 0, + tickets int NOT NULL DEFAULT 0, + last_ticket date, + observed_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (wo_base, lot, split, sub, seq) +); + +-- What VISUAL changed, per batch, for the Admin view. Pruned at 90 days. +CREATE TABLE visual_row_changes ( + id bigserial PRIMARY KEY, + batch_id bigint NOT NULL REFERENCES visual_batches (id), + wo_base text NOT NULL, + lot text NOT NULL DEFAULT '', + split text NOT NULL DEFAULT '', + sub text NOT NULL DEFAULT '', + seq int, + kind text NOT NULL CHECK (kind IN ('added', 'changed', 'gone', 'returned')), + before jsonb, + after jsonb, + at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX visual_row_changes_batch ON visual_row_changes (batch_id); +CREATE INDEX visual_row_changes_at_brin ON visual_row_changes USING brin (at); + +CREATE TABLE visual_material_changes ( + id bigserial PRIMARY KEY, + batch_id bigint NOT NULL REFERENCES visual_batches (id), + wo_base text NOT NULL, + lot text NOT NULL DEFAULT '', + split text NOT NULL DEFAULT '', + sub text NOT NULL DEFAULT '', + seq int, + piece int, + kind text NOT NULL CHECK (kind IN ('added', 'changed', 'gone', 'returned')), + before jsonb, + after jsonb, + at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX visual_material_changes_batch ON visual_material_changes (batch_id); +CREATE INDEX visual_material_changes_at_brin ON visual_material_changes USING brin (at); + +-- Work orders released before the go-live date. A row pulled in by hand becomes tracked. +CREATE TABLE visual_ignored_wos ( + id bigserial PRIMARY KEY, + wo_base text NOT NULL, + lot text NOT NULL DEFAULT '', + split text NOT NULL DEFAULT '', + sub text NOT NULL DEFAULT '', + rel date, + project text, + unit_number text, + res text, + dwg text, + first_seen_batch bigint, + last_seen_at timestamptz NOT NULL DEFAULT now(), + pulled_in_at timestamptz, + pulled_in_by uuid REFERENCES users (id), + pulled_in_reason text, + UNIQUE (wo_base, lot, split, sub) +); + +-- VISUAL resource to shop department, or fold (an operation folded into the packet) or ignore. +CREATE TABLE visual_resource_map ( + resource text PRIMARY KEY, + target text NOT NULL CHECK (target = 'fold' OR target = 'ignore' OR target ~ '^[0-9]{2}$'), + confirmed bool NOT NULL DEFAULT false, + updated_at timestamptz, + updated_by uuid +); + +-- Seeded from the old build's sync/visual_map.json. "Electical" is spelled that way in VISUAL. +INSERT INTO visual_resource_map (resource, target) VALUES + ('LAYOUT', '01'), + ('LAYOUT-BT', '01'), + ('LAYOUT-SM BAND', '01'), + ('LAYOUT-BEVELER', '01'), + ('LAYOUT-ATT', '01'), + ('LAYOUT-ROLL', '01'), + ('LAYOUT-PRESS', '01'), + ('LAYOUT-THREADER', '01'), + ('LAYOUT-HAND CUT', '01'), + ('LAYOUT-WEST', '02'), + ('LT-SS', '02'), + ('WELD-PART', '03'), + ('NOVARC_1', '03'), + ('WELD-PART-WEST', '04'), + ('PART-WELD-WEST', '04'), + ('WP-SS', '05'), + ('WELD-CODE', '06'), + ('WELD-CODE-BT', '07'), + ('SUBASSEMBLY', '08'), + ('WEST-SA', '08'), + ('HYDROTEST', '09'), + ('WELD-ASY', '10'), + ('FINAL ASY-NORTH', '11'), + ('PREP-NORTH', '11'), + ('FINAL ASSEMBLY', '11'), + ('FINAL ASY-SOUTH', '12'), + ('PREP-SOUTH', '12'), + ('WEST-FA', '12'), + ('PAINT', '13'), + ('SANDBLAST', '13'), + ('FINAL-ASY-ELEC', '14'), + ('Electical', '14'), + ('MFG', 'fold'), + ('INSPECTION', 'fold'), + ('REWORK', 'fold'), + ('REWORK-CUSTOMER', 'fold'), + ('OUTSIDE SERVICE', 'fold'), + ('PTU', 'fold'), + ('FORKLIFT', 'ignore'), + ('STEAM', 'ignore'), + ('WIPLOAD', 'ignore'), + ('FOREMAN', 'ignore'), + ('ENGINEERING', 'ignore'), + ('TRAINING', 'ignore'), + ('F.S. TECH.', 'ignore'), + ('F.S. TECH. ASST', 'ignore'), + ('F.S. STEAM', 'ignore'), + ('F.S. WELDER', 'ignore'), + ('F.S. SANDBLAST', 'ignore'), + ('F.S. PAINT', 'ignore'), + ('F.S. SAFETY', 'ignore'), + ('REP. TECH. A', 'ignore'), + ('REP. TECH. B', 'ignore'), + ('REP. WELDER A', 'ignore'), + ('REP. WELDER B', 'ignore'); + +-- ---------------------------------------------------------------- drawing cache + +-- One row per drawing number as VISUAL writes it (trimmed). asset_id is the last fetched copy. +CREATE TABLE drawing_lookups ( + dwg text PRIMARY KEY, + checked_at timestamptz, + ok bool NOT NULL DEFAULT true, + rev int, + name text, + obj_id int, + obj_version int, + file_id int, + file_version int, + file_size bigint, + base text, + draft_rev int, + draft_where text, + in_drafting bool NOT NULL DEFAULT false, + note text, + fail_count int NOT NULL DEFAULT 0, + failing_since timestamptz, + asset_id uuid REFERENCES assets (id), + sha256 bytea, + visual_rev_seen text, + next_check_at timestamptz +); + +CREATE INDEX drawing_lookups_next_check ON drawing_lookups (next_check_at) WHERE next_check_at IS NOT NULL; + +-- ---------------------------------------------------------------- worker jobs + +-- "Run now": the app inserts a row and NOTIFYs ps_job_request; one run claims every pending row. +CREATE TABLE job_requests ( + id bigserial PRIMARY KEY, + job text NOT NULL, + requested_at timestamptz NOT NULL DEFAULT now(), + requested_by uuid REFERENCES users (id), + note text, + claimed_run_id bigint, + claimed_at timestamptz +); + +CREATE INDEX job_requests_pending ON job_requests (job) WHERE claimed_at IS NULL; + +CREATE TABLE job_runs ( + id bigserial PRIMARY KEY, + job text NOT NULL, + trigger text NOT NULL CHECK (trigger IN ('schedule', 'request', 'chain', 'cli')), + requested_by uuid[] NOT NULL DEFAULT '{}', + started_at timestamptz NOT NULL DEFAULT now(), + finished_at timestamptz, + status text NOT NULL DEFAULT 'running' + CHECK (status IN ('running', 'ok', 'degraded', 'failed', 'skipped', 'abandoned')), + counts jsonb NOT NULL DEFAULT '{}', + errors jsonb NOT NULL DEFAULT '[]', + build_version text, + visual_target text, + host text +); + +CREATE INDEX job_runs_job ON job_runs (job, id DESC); + +-- The next run per job as the worker last scheduled it, for the Admin and Clerical screens. +CREATE TABLE job_schedule ( + job text PRIMARY KEY, + next_run_at timestamptz, + interval_seconds int, + paused bool NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL DEFAULT now() +); + +INSERT INTO job_schedule (job) VALUES ('visual-feed'), ('drawing-lookup'), ('housekeeping'); + +-- One row, written every 30 seconds by the running worker. +CREATE TABLE worker_heartbeat ( + id int PRIMARY KEY CHECK (id = 1), + at timestamptz NOT NULL, + build text, + host text, + pid int, + running text[] NOT NULL DEFAULT '{}' +); + +-- ---------------------------------------------------------------- column additions + +ALTER TABLE packets + ADD COLUMN visual_cancelled_at timestamptz, + ADD COLUMN wo_gone_at timestamptz; + +-- The leg's own drawing, rev and planner instructions as VISUAL carries them. +ALTER TABLE packet_work_orders + ADD COLUMN partspecs text, + ADD COLUMN rev text, + ADD COLUMN dwg text; + +-- One row per leg: two operations of one work order may sit on one packet. Hand rows keep seq NULL, +-- and NULLS NOT DISTINCT still allows only one hand row per key. +DO $$ +DECLARE + c text; +BEGIN + SELECT con.conname INTO c + FROM pg_constraint con + WHERE con.conrelid = 'packet_work_orders'::regclass + AND con.contype = 'u' + AND ( + SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM unnest(con.conkey) k + JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = k + ) = ARRAY['lot', 'packet_id', 'split', 'sub', 'wo_base']; + IF c IS NULL THEN + RAISE EXCEPTION 'packet_work_orders unique key (packet_id, wo_base, lot, split, sub) not found'; + END IF; + EXECUTE format('ALTER TABLE packet_work_orders DROP CONSTRAINT %I', c); +END; +$$; + +ALTER TABLE packet_work_orders + ADD CONSTRAINT packet_work_orders_leg_key UNIQUE NULLS NOT DISTINCT (packet_id, wo_base, lot, split, sub, seq); + +-- One pending feed change order per packet, whether VISUAL or M-Files opened it. +DROP INDEX change_orders_visual_pending; +CREATE UNIQUE INDEX change_orders_feed_pending ON change_orders (packet_id) + WHERE origin IN ('visual', 'drawing') AND review = 'pending'; + +-- order_ref is the order the feed chose; order_pin is one Clerical pinned (with a reason). +ALTER TABLE projects + ADD COLUMN order_date_visual date, + ADD COLUMN ship_date_visual date, + ADD COLUMN order_ref jsonb, + ADD COLUMN order_pin text, + ADD COLUMN order_pin_by uuid REFERENCES users (id), + ADD COLUMN order_pin_at timestamptz, + ADD COLUMN order_pin_reason text; + +ALTER TABLE units + ADD COLUMN customer text, + ADD COLUMN po text, + ADD COLUMN customer_visual text, + ADD COLUMN po_visual text, + ADD COLUMN order_date_visual date, + ADD COLUMN ship_date_visual date, + ADD COLUMN order_ref jsonb, + ADD COLUMN order_pin text, + ADD COLUMN order_pin_by uuid REFERENCES users (id), + ADD COLUMN order_pin_at timestamptz, + ADD COLUMN order_pin_reason text; + +-- v_units expanded u.* when it was made, so it is rebuilt to carry the new unit columns. +DROP VIEW v_units; +CREATE VIEW v_units AS +SELECT u.*, + p.number || u.lot AS number, + p.number AS project_number, + p.name AS project_name, + p.status AS project_status, + coalesce(u.due, p.due) AS due_effective +FROM units u +JOIN projects p ON p.id = u.project_id; + +-- Explicit, after the default privileges: the view is read-only to the app. +REVOKE INSERT, UPDATE, DELETE, TRUNCATE ON v_units FROM ps_app; +GRANT SELECT ON v_units TO ps_app; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ps_app; + +-- The feed COPYs each batch into temp tables before the MERGE. ps_migrate owns the database (initdb and +-- bootstrap), so it may grant TEMP; stacks made before this migration get the grant here. +DO $$ +BEGIN + EXECUTE format('GRANT TEMPORARY ON DATABASE %I TO ps_app', current_database()); +END; +$$; diff --git a/server/packetstation/settings.py b/server/packetstation/settings.py index f3c1157..fdc1109 100644 --- a/server/packetstation/settings.py +++ b/server/packetstation/settings.py @@ -12,6 +12,8 @@ from pydantic_settings import BaseSettings, SettingsConfigDict ENV_FILE_VAR = "PS_ENV_FILE" Stack = Literal["prod", "test", "ci", "dev"] +FeedSource = Literal["visualkit", "fixture"] +DrawingSource = Literal["mfiles", "fixture"] class SecretFileError(Exception): @@ -101,6 +103,11 @@ class Settings(BaseSettings): pin_max_digits: int = Field(default=12, ge=4) upload_max_bytes: int = Field(default=25 * 1024 * 1024, ge=1024) + # Where the worker reads VISUAL and M-Files from. `fixture` reads synthetic files under FIXTURE_DIR (CI and tests). + feed_source: FeedSource = "visualkit" + drawing_source: DrawingSource = "mfiles" + fixture_dir: Path = Path("/fixtures") + @model_validator(mode="before") @classmethod def _blank_is_unset(cls, data: Any) -> Any: @@ -110,7 +117,7 @@ class Settings(BaseSettings): if key.lower() in _OPTIONAL_PATHS and data[key] in ("", None): data[key] = None for key in list(data): - if key.lower() == "stack" and isinstance(data[key], str): + if key.lower() in ("stack", "feed_source", "drawing_source") and isinstance(data[key], str): data[key] = data[key].strip().lower() return data @@ -128,6 +135,12 @@ class Settings(BaseSettings): raise ValueError("LDAP_ALLOW_INSECURE=true is refused on STACK=prod") return self + @model_validator(mode="after") + def _fixture_sources(self) -> Settings: + if self.stack == "prod" and (self.feed_source == "fixture" or self.drawing_source == "fixture"): + raise ValueError("FEED_SOURCE=fixture and DRAWING_SOURCE=fixture are refused on STACK=prod") + return self + @model_validator(mode="after") def _pin_digits(self) -> Settings: if self.pin_min_digits > self.pin_max_digits: diff --git a/server/packetstation/text.py b/server/packetstation/text.py index 60d271c..da95d52 100644 --- a/server/packetstation/text.py +++ b/server/packetstation/text.py @@ -46,6 +46,9 @@ LIMITS: dict[str, int] = { "doc_title": 160, "ecn": 40, "name": 160, + "order_pin": 40, + "file_name": 160, + "job_note": 200, } REASON_MIN = 8 MAX_BODY_BYTES = 200_000 diff --git a/server/packetstation/worker.py b/server/packetstation/worker.py deleted file mode 100644 index 9795a62..0000000 --- a/server/packetstation/worker.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Phase 0 worker stub: a heartbeat line every 30 s and a touched beat file; exits on SIGTERM.""" - -from __future__ import annotations - -import logging -import os -import signal -import threading -import time -from pathlib import Path - -log = logging.getLogger("packetstation.worker") - -BEAT_SECONDS = 30 -STALE_SECONDS = 90 - - -def touch(path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.touch(exist_ok=True) - now = time.time() - os.utime(path, (now, now)) - - -def run(beat_file: Path, build: str, stop: threading.Event | None = None, interval: float = BEAT_SECONDS) -> int: - stop = stop or threading.Event() - - def _stop(signum: int, _frame: object) -> None: - log.info("worker stopping on signal %s", signum) - stop.set() - - if threading.current_thread() is threading.main_thread(): - signal.signal(signal.SIGTERM, _stop) - signal.signal(signal.SIGINT, _stop) - log.info("worker started", extra={"ps": {"build": build, "beat_file": str(beat_file)}}) - while not stop.is_set(): - try: - touch(beat_file) - except OSError as e: - log.error("can't touch beat file %s: %s", beat_file, e.strerror) - log.info("heartbeat", extra={"ps": {"build": build}}) - stop.wait(interval) - log.info("worker stopped") - return 0 - - -def beat_age(beat_file: Path) -> float | None: - try: - return time.time() - beat_file.stat().st_mtime - except OSError: - return None diff --git a/server/packetstation/worker/__init__.py b/server/packetstation/worker/__init__.py new file mode 100644 index 0000000..88de462 --- /dev/null +++ b/server/packetstation/worker/__init__.py @@ -0,0 +1,63 @@ +"""The background worker: the job scheduler plus a heartbeat (beat file and worker_heartbeat row). + +`run(beat_file, build, stop=None, interval=BEAT_SECONDS, settings=None)` blocks until SIGTERM, SIGINT or `stop`. +With settings it builds and starts a Scheduler; without (the CLI tests) it only beats the file. +""" + +from __future__ import annotations + +import logging +import signal +import threading +from pathlib import Path +from typing import TYPE_CHECKING + +from .heartbeat import BEAT_SECONDS, STALE_SECONDS, beat_age, touch + +if TYPE_CHECKING: + from ..settings import Settings + +log = logging.getLogger("packetstation.worker") + +__all__ = ["BEAT_SECONDS", "STALE_SECONDS", "beat_age", "run", "touch"] + + +def run( + beat_file: Path, + build: str, + stop: threading.Event | None = None, + interval: float = BEAT_SECONDS, + settings: Settings | None = None, +) -> int: + stop = stop or threading.Event() + + def _stop(signum: int, _frame: object) -> None: + log.info("worker stopping on signal %s", signum) + stop.set() + + if threading.current_thread() is threading.main_thread(): + signal.signal(signal.SIGTERM, _stop) + signal.signal(signal.SIGINT, _stop) + log.info("worker started", extra={"ps": {"build": build, "beat_file": str(beat_file)}}) + sched = None + if settings is not None: + from .scheduler import Scheduler + + sched = Scheduler(settings, build=build) + sched.start() + try: + if sched is not None: + sched.serve(beat_file, stop, interval) + else: + while not stop.is_set(): + try: + touch(beat_file) + except OSError as e: + log.error("can't touch beat file %s: %s", beat_file, e.strerror) + log.info("heartbeat", extra={"ps": {"build": build}}) + stop.wait(interval) + finally: + if sched is not None: + sched.shutdown(wait=True) + log.info("worker stopped") + return 0 diff --git a/server/packetstation/worker/heartbeat.py b/server/packetstation/worker/heartbeat.py new file mode 100644 index 0000000..b7bc776 --- /dev/null +++ b/server/packetstation/worker/heartbeat.py @@ -0,0 +1,36 @@ +"""The worker's heartbeat: a touched beat file for the container healthcheck and a worker_heartbeat row.""" + +from __future__ import annotations + +import os +import time +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +BEAT_SECONDS = 30 +STALE_SECONDS = 90 + + +def touch(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.touch(exist_ok=True) + now = time.time() + os.utime(path, (now, now)) + + +def beat_age(beat_file: Path) -> float | None: + try: + return time.time() - beat_file.stat().st_mtime + except OSError: + return None + + +def write_row(conn: Any, *, build: str, host: str, pid: int, running: Iterable[str]) -> None: + """Upsert the single worker_heartbeat row.""" + conn.execute( + "INSERT INTO worker_heartbeat (id, at, build, host, pid, running) VALUES (1, now(), %s, %s, %s, %s)" + " ON CONFLICT (id) DO UPDATE SET at = EXCLUDED.at, build = EXCLUDED.build, host = EXCLUDED.host," + " pid = EXCLUDED.pid, running = EXCLUDED.running", + (build, host, pid, sorted(running)), + ) diff --git a/server/packetstation/worker/jobs/__init__.py b/server/packetstation/worker/jobs/__init__.py new file mode 100644 index 0000000..a899f6f --- /dev/null +++ b/server/packetstation/worker/jobs/__init__.py @@ -0,0 +1,146 @@ +"""The job registry and the shapes every job uses. + +Jobs import `from packetstation.worker.jobs import JobContext, JobSpec, register` and expose `SPEC`. The feed and +drawing jobs are imported lazily by `load_jobs`, so the worker runs before they exist. +""" + +from __future__ import annotations + +import importlib +import logging +import threading +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import UTC, date, datetime, time, timedelta +from typing import Any +from zoneinfo import ZoneInfo + +from ...feed.config import FeedSettings +from ...settings import Settings +from ..redaction import MAX_ERRORS, clip, redact + +log = logging.getLogger(__name__) + +Counts = dict[str, Any] +Cadence = Callable[[FeedSettings, datetime], int | None] +TRIGGERS = ("schedule", "request", "chain", "cli") + + +@dataclass +class Stack: + """Per-process state shared by every run: the startup guard result and long-lived objects (the M-Files vault).""" + + guard: Any = None + cache: dict[str, Any] = field(default_factory=dict) + lock: threading.Lock = field(default_factory=threading.Lock) + + def cached(self, key: str, factory: Callable[[], Any]) -> Any: + """The object under `key`, made once by `factory`. A factory that raises leaves nothing cached.""" + with self.lock: + if key not in self.cache: + self.cache[key] = factory() + return self.cache[key] + + def drop(self, key: str) -> None: + with self.lock: + self.cache.pop(key, None) + + +@dataclass +class JobContext: + """What a run gets. `conn` is the run's own autocommit connection as the worker role.""" + + conn: Any + settings: Settings + feed_settings: FeedSettings + run_id: int + trigger: str + started_at: datetime + log: logging.Logger + stack: Stack + degraded: bool = False + errors: list[str] = field(default_factory=list) + visual_target: str | None = None + + def error(self, text: str, *, degrade: bool = True) -> None: + """Record one error (redacted, clipped). It makes the run degraded unless degrade is False.""" + if len(self.errors) < MAX_ERRORS: + self.errors.extend(clip([text])) + if degrade: + self.degraded = True + + def degrade(self, reason: str | None = None) -> None: + self.degraded = True + if reason: + self.error(reason) + + @property + def tz(self) -> ZoneInfo: + return ZoneInfo(self.settings.site_tz) + + def now_site(self) -> datetime: + return datetime.now(self.tz) + + def today_site(self) -> date: + return self.now_site().date() + + +@dataclass(frozen=True) +class JobSpec: + name: str + run: Callable[[JobContext], Counts] + cadence: Cadence # seconds to the next run, or None for "no schedule" + pausable: bool # the feed pause skips scheduled fires (requests still run) + chain: tuple[str, ...] = () # moved to now after a run that did not fail + + +JOBS: dict[str, JobSpec] = {} +_loaded = False +_load_lock = threading.Lock() +LAZY = ("visual_feed", "drawing_lookup") + + +def register(spec: JobSpec) -> JobSpec: + JOBS[spec.name] = spec + return spec + + +def load_jobs() -> dict[str, JobSpec]: + """Register every job once and return the registry. A job module that cannot import is logged and left out.""" + global _loaded + with _load_lock: + if not _loaded: + from . import housekeeping + + register(housekeeping.SPEC) + for mod in LAZY: + try: + m = importlib.import_module(f"{__name__}.{mod}") + register(m.SPEC) + except ImportError as e: + log.warning("job module %s not loaded: %s", mod, redact(str(e))) + _loaded = True + return JOBS + + +def seconds_until(now_site: datetime, at: time) -> int: + """Seconds from now_site to the next `at` (site wall clock), at least 1.""" + target = datetime.combine(now_site.date(), at, tzinfo=now_site.tzinfo) + if target <= now_site: + target = datetime.combine(now_site.date() + timedelta(days=1), at, tzinfo=now_site.tzinfo) + # Aware datetimes sharing a tzinfo subtract as wall clock; through UTC a DST change counts right + return max(1, int((target.astimezone(UTC) - now_site.astimezone(UTC)).total_seconds())) + + +__all__ = [ + "JOBS", + "TRIGGERS", + "Cadence", + "Counts", + "JobContext", + "JobSpec", + "Stack", + "load_jobs", + "register", + "seconds_until", +] diff --git a/server/packetstation/worker/jobs/drawing_lookup.py b/server/packetstation/worker/jobs/drawing_lookup.py new file mode 100644 index 0000000..759b35f --- /dev/null +++ b/server/packetstation/worker/jobs/drawing_lookup.py @@ -0,0 +1,536 @@ +"""The drawing-lookup job: look drawings up in M-Files (read-only), copy released PDFs into assets, and attach, +flag, stage or open a change order on the packets that use them. + +Per run: the queue (drawings/queue.py) cut to `drawing_budget`; per drawing one `Lookup.status`, a fetch when a +released file exists (drawings/fetch.py downloads only when its M-Files identity changed), the `drawing_lookups` +row, then `decide` for every live packet that uses the drawing (its own drawing, or the target of its pending feed +change order), applied through `feed.writes.system_write`. A busy packet is left for the next run. + +Never writes to M-Files. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb + +from ... import audit, policy +from ...domain import pipeline +from ...drawings import DRAWING_FLAGS, decide, queue +from ...drawings import lookup as lookups +from ...drawings.fetch import FetchError, FetchResult, fetch +from ...drawings.lookup import Found, Lookup +from ...feed import changes, writes +from . import Counts, JobContext, JobSpec + +log = logging.getLogger(__name__) + +A = policy.Action +D = decide + +FEED_ORIGINS = ("visual", "drawing") + + +class _Stale(Exception): + """The packet changed between the read and the lock; the next run decides again.""" + + +def new_counts() -> Counts: + return { + "checked": 0, + "fetched": 0, + "unchanged": 0, + "attached": 0, + "flagged": {}, + "lookup_failed": 0, + "cos_opened": 0, + "cos_reaimed": 0, + "staged": 0, + "budget_left": 0, + "fetch_failed": 0, + "busy": 0, + "refused": 0, + "no_file": 0, + } + + +# ---------------------------------------------------------------- the run + + +def run(ctx: JobContext) -> Counts: + counts = new_counts() + fs = ctx.feed_settings + budget = int(getattr(fs, "drawing_budget", 60)) + try: + lk = lookups.make(ctx.settings, ctx.stack.cache) + except Exception as e: # any failure to open M-Files skips the run + ctx.error(f"M-Files could not be opened: {type(e).__name__}: {e}") + counts["reason"] = "mfiles unavailable" + counts["budget_left"] = budget + return counts + now = datetime.now(UTC) + items = queue.queue(ctx.conn, fs, now, limit=budget) + counts["budget_left"] = max(0, budget - len(items)) + counts["source"] = lk.describe() + for item in items: + try: + _drawing(ctx, lk, item, counts, now) + except Exception as e: # one bad drawing must not stop the others + log.exception("drawing %s failed", item.dwg) + ctx.error(f"drawing {item.dwg}: {type(e).__name__}: {e}") + return counts + + +def _drawing(ctx: JobContext, lk: Lookup, item: queue.QueueItem, counts: Counts, now: datetime) -> None: + conn = ctx.conn + try: + found = lk.status(item.dwg) + except Exception as e: # a lookup error is a failed lookup, never a crash + found = lookups.failed(item.dwg, f"M-Files lookup failed: {type(e).__name__}: {e}") + counts["checked"] += 1 + fetched: FetchResult | None = None + fetch_failed = False + if found.ok and found.rev is not None: + try: + fetched = fetch(conn, ctx.settings, lk, found, dwg=item.dwg) + except FetchError as e: + fetch_failed = True + counts["fetch_failed"] += 1 + ctx.error(str(e)) + else: + if fetched.downloaded: + counts["fetched"] += 1 + audit.append( + conn, + action="drawing.fetched", + role_snapshot="system", + subject_type="asset", + subject_id=str(fetched.asset_id), + detail={ + "dwg": item.dwg, + "rev": found.rev, + "sha256": fetched.sha256.hex(), + "reused": fetched.reused, + }, + ) + if not found.ok: + counts["lookup_failed"] += 1 + ctx.error(f"drawing {item.dwg}: {D.failed_note(found)}") + vrev_seen = item.visual_rev if item.co_id is None else own_rev(conn, item.dwg) + save_lookup(conn, item, found, fetched, now, fetch_failed=fetch_failed, visual_rev_seen=vrev_seen) + scope = [str(d) for d in (getattr(ctx.feed_settings, "departments", None) or [])] + for pid, relations in sharing(conn, item.dwg, scope): + _packet(ctx, item.dwg, found, fetched, pid, relations, counts) + + +# ---------------------------------------------------------------- drawing_lookups + + +def save_lookup( + conn: Any, + item: queue.QueueItem, + found: Found, + fetched: FetchResult | None, + now: datetime, + *, + fetch_failed: bool, + visual_rev_seen: str | None, +) -> None: + """Upsert the drawing_lookups row for this lookup. + + A failed lookup keeps everything known before (rev, file, asset) and backs off. A good one stores what M-Files + has; the file identity and the asset are stored together, and cleared when there is no released file or the + copy failed (so the next run downloads again). + """ + with conn.transaction(): + prev = conn.execute("SELECT fail_count FROM drawing_lookups WHERE dwg = %s FOR UPDATE", (item.dwg,)).fetchone() + if not found.ok: + fails = (int(prev[0]) if prev else 0) + 1 + conn.execute( + "INSERT INTO drawing_lookups (dwg, checked_at, ok, note, fail_count, failing_since, next_check_at," + " visual_rev_seen) VALUES (%(dwg)s, %(now)s, false, %(note)s, %(fails)s, %(now)s, %(next)s, %(vrev)s)" + " ON CONFLICT (dwg) DO UPDATE SET checked_at = EXCLUDED.checked_at, ok = false, note = EXCLUDED.note," + " fail_count = EXCLUDED.fail_count," + " failing_since = coalesce(drawing_lookups.failing_since, EXCLUDED.failing_since)," + " next_check_at = EXCLUDED.next_check_at, visual_rev_seen = EXCLUDED.visual_rev_seen", + { + "dwg": item.dwg, + "now": now, + "note": D.failed_note(found), + "fails": fails, + "next": now + queue.backoff(fails), + "vrev": visual_rev_seen, + }, + ) + return + has_file = fetched is not None + conn.execute( + "INSERT INTO drawing_lookups (dwg, checked_at, ok, rev, name, obj_id, obj_version, file_id, file_version," + " file_size, base, draft_rev, draft_where, in_drafting, note, fail_count, failing_since, asset_id, sha256," + " visual_rev_seen, next_check_at) VALUES (%(dwg)s, %(now)s, true, %(rev)s, %(name)s, %(obj_id)s," + " %(obj_version)s, %(file_id)s, %(file_version)s, %(file_size)s, %(base)s, %(draft_rev)s," + " %(draft_where)s, %(in_drafting)s, %(note)s, 0, NULL, %(asset_id)s, %(sha)s, %(vrev)s, %(next)s)" + " ON CONFLICT (dwg) DO UPDATE SET checked_at = EXCLUDED.checked_at, ok = true, rev = EXCLUDED.rev," + " name = EXCLUDED.name, obj_id = EXCLUDED.obj_id, obj_version = EXCLUDED.obj_version," + " file_id = EXCLUDED.file_id, file_version = EXCLUDED.file_version, file_size = EXCLUDED.file_size," + " base = EXCLUDED.base, draft_rev = EXCLUDED.draft_rev, draft_where = EXCLUDED.draft_where," + " in_drafting = EXCLUDED.in_drafting, note = EXCLUDED.note, fail_count = 0, failing_since = NULL," + " asset_id = EXCLUDED.asset_id, sha256 = EXCLUDED.sha256," + " visual_rev_seen = coalesce(EXCLUDED.visual_rev_seen, drawing_lookups.visual_rev_seen)," + " next_check_at = EXCLUDED.next_check_at", + { + "dwg": item.dwg, + "now": now, + "rev": found.rev, + "name": found.name, + "obj_id": found.obj_id if has_file else None, + "obj_version": found.obj_version if has_file else None, + "file_id": found.file_id if has_file else None, + "file_version": found.file_version if has_file else None, + "file_size": found.file_size, + "base": found.base, + "draft_rev": found.draft_rev, + "draft_where": found.draft_where or None, + "in_drafting": found.in_drafting, + "note": found.note or None, + "asset_id": fetched.asset_id if fetched else None, + "sha": fetched.sha256 if fetched else None, + "vrev": visual_rev_seen, + "next": now + queue.backoff(1) if fetch_failed else None, + }, + ) + + +# ---------------------------------------------------------------- packets + + +def own_rev(conn: Any, dwg: str) -> str | None: + """The rev a live VISUAL packet with this drawing carries (for visual_rev_seen when the item came from a CO).""" + row = conn.execute( + "SELECT coalesce(btrim(rev), '') FROM packets WHERE removed_at IS NULL AND source_kind = 'visual'" + " AND status <> 'filed' AND btrim(drawing) = %s ORDER BY id LIMIT 1", + (dwg,), + ).fetchone() + return None if row is None else str(row[0]) + + +SHARING_SQL = """ +SELECT p.id, 'own' AS rel FROM packets p +WHERE p.removed_at IS NULL AND p.source_kind = 'visual' AND p.status <> 'filed' AND btrim(p.drawing) = %(dwg)s + AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[])) +UNION +SELECT p.id, 'co' AS rel FROM change_orders c JOIN packets p ON p.id = c.packet_id +WHERE c.review = 'pending' AND c.origin IN ('visual', 'drawing') AND jsonb_typeof(c.waiting) = 'object' + AND btrim(c.waiting->>'dwg') = %(dwg)s AND p.removed_at IS NULL AND p.status NOT IN ('draft', 'filed') + AND (cardinality(%(depts)s::text[]) = 0 OR p.dept_code = ANY(%(depts)s::text[])) +ORDER BY 1 +""" + + +def sharing(conn: Any, dwg: str, depts: list[str]) -> list[tuple[int, set[str]]]: + """Live packets that use the drawing: as their own drawing ('own') or as a pending feed CO's target ('co').""" + out: dict[int, set[str]] = {} + for pid, rel in conn.execute(SHARING_SQL, {"dwg": dwg, "depts": depts}).fetchall(): + out.setdefault(int(pid), set()).add(str(rel)) + return sorted(out.items()) + + +@dataclass +class PacketFacts: + row: dict[str, Any] + visual: dict[str, Any] + attached_rev: str | None + attached_shas: set[bytes] + co: dict[str, Any] | None + rejected_sha: bytes | None + + +def _sha(value: Any) -> bytes | None: + if not isinstance(value, str) or not value: + return None + try: + return bytes.fromhex(value) + except ValueError: + return None + + +def facts(conn: Any, pid: int) -> PacketFacts | None: + with conn.cursor(row_factory=dict_row) as cur: + row = cur.execute( + "SELECT id, status::text AS status, drawing, rev, drawing_state::text AS drawing_state, flags, visual," + " version FROM packets WHERE id = %s AND removed_at IS NULL", + (pid,), + ).fetchone() + if row is None: + return None + pages = cur.execute( + "SELECT pg.rev, a.sha256 FROM packet_pages pg LEFT JOIN assets a ON a.id = pg.asset_id" + " WHERE pg.packet_id = %s AND pg.type = 'drawing' AND pg.void_at IS NULL ORDER BY pg.n", + (pid,), + ).fetchall() + co = cur.execute( + "SELECT id, co_no, origin::text AS origin, waiting, staged FROM change_orders" + " WHERE packet_id = %s AND review = 'pending' AND origin IN ('visual', 'drawing') ORDER BY id DESC LIMIT 1", + (pid,), + ).fetchone() + rej = cur.execute( + "SELECT staged FROM change_orders WHERE packet_id = %s AND origin = 'drawing' AND review = 'rejected'" + " ORDER BY id DESC LIMIT 1", + (pid,), + ).fetchone() + visual = row["visual"] if isinstance(row["visual"], dict) else {} + attached_rev = None + if pages: + attached_rev = str(pages[0]["rev"] or "").strip() + shas = {bytes(p["sha256"]) for p in pages if p["sha256"] is not None} + rejected = None + if rej is not None and isinstance(rej["staged"], dict): + rejected = _sha(rej["staged"].get("sha256")) + return PacketFacts(dict(row), dict(visual), attached_rev, shas, dict(co) if co else None, rejected) + + +def _dwg_rev(value: Any) -> tuple[str, str] | None: + if isinstance(value, dict) and value.get("dwg") and value.get("rev") is not None: + return (str(value["dwg"]).strip(), str(value["rev"]).strip()) + return None + + +def target_for(dwg: str, pf: PacketFacts, *, own: bool) -> D.Target: + row, visual, co = pf.row, pf.visual, pf.co + waiting = co.get("waiting") if co else None + staged = co.get("staged") if co else None + co_waiting = _dwg_rev(waiting) + co_staged = _dwg_rev(staged) + packet_rev = str(row["rev"] or "").strip() + rev_ok = None + ro = visual.get("rev_ok") + if isinstance(ro, dict) and ro.get("rev") is not None and str(ro.get("vis_rev") or "").strip() == packet_rev: + rev_ok = str(ro["rev"]).strip() + visual_rev = packet_rev if own or co_waiting is None else co_waiting[1] + return D.Target( + dwg=dwg, + visual_rev=visual_rev, + attached_rev=pf.attached_rev, + attached_sha=next(iter(sorted(pf.attached_shas)), None), + status=str(row["status"]), + drawing_state=str(row["drawing_state"]), + has_hand_drawing=row["drawing_state"] == "hand" or bool(visual.get("manual_dwg")), + rev_ok=rev_ok, + co_waiting=co_waiting, + co_staged=co_staged, + co_pending=co is not None, + co_staged_sha=_sha(staged.get("sha256")) if isinstance(staged, dict) else None, + rejected_sha=pf.rejected_sha, + ) + + +def appr_of(t: D.Target, f: Found, now: datetime) -> dict[str, Any]: + return { + "ok": f.ok, + "rev": f.rev, + "vrev": t.visual_rev, + "name": f.name, + "note": f.note or "", + "base": f.base, + "draft_rev": f.draft_rev, + "draft_where": f.draft_where, + "in_drafting": f.in_drafting, + "at": now.isoformat(), + } + + +def _same_appr(a: Any, b: dict[str, Any]) -> bool: + if not isinstance(a, dict): + return False + return {k: v for k, v in a.items() if k != "at"} == {k: v for k, v in b.items() if k != "at"} + + +def new_flags(flags: list[str], d: D.Decision) -> list[str]: + """Every flag but the drawing ones, then the decision's flag.""" + out = [f for f in flags if f not in DRAWING_FLAGS] + if d.flag and d.flag not in out: + out.append(d.flag) + return out + + +def _packet( + ctx: JobContext, + dwg: str, + found: Found, + fetched: FetchResult | None, + pid: int, + relations: set[str], + counts: Counts, +) -> None: + conn = ctx.conn + pf = facts(conn, pid) + if pf is None: + return + own = "own" in relations + t = target_for(dwg, pf, own=own) + same_file = fetched is not None and fetched.sha256 in pf.attached_shas + d = D.decide(t, found, same_file=same_file, fetched_sha=fetched.sha256 if fetched else None) + if not own and d.kind != D.STAGE_FOR_CO: + return # only the CO's target drawing: nothing to do until it can be staged + if d.kind in (D.ATTACH, D.OPEN_CO, D.STAGE_FOR_CO) and fetched is None: + counts["no_file"] += 1 # the copy failed this run; try again next run + return + now = datetime.now(UTC) + row, visual = pf.row, pf.visual + flags = list(row["flags"] or []) + flags_after = new_flags(flags, d) if own else flags + patch: dict[str, Any] = {} + if own: + appr = appr_of(t, found, now) + if not _same_appr(visual.get("appr"), appr): + patch["appr"] = appr + if found.ok: + confirmed = D.rev_confirmed(t, found) + if bool(visual.get("rev_confirmed")) != confirmed: + patch["rev_confirmed"] = confirmed + state_after = d.drawing_state if own else str(row["drawing_state"]) + material = ( + d.kind in (D.ATTACH, D.OPEN_CO, D.STAGE_FOR_CO) + or flags_after != flags + or state_after != row["drawing_state"] + or bool(patch) + ) + if not material: + counts["unchanged"] += 1 + return + action = A.VIS_UPDATE_DRAFT if row["status"] == "draft" else A.VIS_OPEN_CO + label, note = "Drawing status from M-Files", d.note or None + if d.kind == D.ATTACH and fetched is not None: + label = "Drawing attached from M-Files" + note = "; ".join(x for x in (fetched.name or found.name or "", d.note) if x) or None + detail = {"dwg": dwg, "rev": found.rev, "kind": d.kind, "asset_id": str(fetched.asset_id) if fetched else None} + try: + with writes.system_write(conn, pid, action, label=label, note=note, detail=detail) as sw: + if int(sw.packet["version"]) != int(row["version"]): + raise _Stale + if d.kind == D.ATTACH and fetched is not None: + _attach(sw, pid, dwg, found, fetched, d, flags_after, patch, now) + counts["attached"] += 1 + else: + _status(sw, pid, state_after, flags_after, patch) + if d.flag: + fl = counts["flagged"] + fl[d.kind] = fl.get(d.kind, 0) + 1 + if d.kind == D.OPEN_CO and fetched is not None: + _open_co(sw, t, dwg, found, fetched, d, counts) + elif d.kind == D.STAGE_FOR_CO and fetched is not None and pf.co is not None: + _stage(sw, pf.co, dwg, found, fetched) + counts["staged"] += 1 + elif d.kind == D.UNCHANGED: + counts["unchanged"] += 1 + except (_Stale, writes.Busy, writes.Missing): + counts["busy"] += 1 + except writes.Refused as e: + counts["refused"] += 1 + ctx.error(f"packet {pid}: {d.kind} refused: {e}", degrade=False) + + +def _staged(dwg: str, found: Found, fetched: FetchResult) -> dict[str, Any]: + return { + "asset_id": str(fetched.asset_id), + "name": fetched.name or found.name, + "rev": str(found.rev), + "dwg": dwg, + "sha256": fetched.sha256.hex(), + "pages": fetched.pages, + } + + +def _attach( + sw: Any, + pid: int, + dwg: str, + found: Found, + fetched: FetchResult, + d: D.Decision, + flags: list[str], + patch: dict[str, Any], + now: datetime, +) -> None: + conn = sw.conn + pages = max(1, int(fetched.pages)) + conn.execute("DELETE FROM packet_pages WHERE packet_id = %s AND type = 'drawing'", (pid,)) + n0 = pipeline.next_counter(conn, pid, "page", pages) + rev = str(found.rev) + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO packet_pages (packet_id, n, type, asset_id, pn, dwg, rev) VALUES (%s, %s, 'drawing', %s, %s," + " %s, %s)", + [(pid, n0 + i, fetched.asset_id, i + 1, dwg, rev) for i in range(pages)], + ) + vault = { + "asset_id": str(fetched.asset_id), + "name": fetched.name or found.name, + "rev": found.rev, + "note": d.note, + "at": now.isoformat(), + } + merged = {**patch, "rev_confirmed": True, "vault": vault} + conn.execute( + "UPDATE packets SET drawing_state = 'attached', flags = %s, visual = (visual - 'manual_dwg') || %s" + " WHERE id = %s", + (flags, Jsonb(merged), pid), + ) + sw.changed("pages", "drawing_state", "flags", "visual") + + +def _status(sw: Any, pid: int, state: str, flags: list[str], patch: dict[str, Any]) -> None: + sw.conn.execute( + "UPDATE packets SET drawing_state = %s::drawing_state, flags = %s, visual = visual || %s WHERE id = %s", + (state, flags, Jsonb(patch), pid), + ) + sw.changed("drawing_state", "flags", "visual") + + +def _open_co(sw: Any, t: D.Target, dwg: str, found: Found, fetched: FetchResult, d: D.Decision, counts: Counts) -> None: + vis = { + "from_dwg": dwg, + "to_dwg": dwg, + "dwg": dwg, + "from_rev": t.attached_rev, + "to_rev": str(found.rev), + "dwg_change": True, + "changes": [d.note], + } + res = changes.open_or_reaim( + sw, + origin="drawing", + reason=d.note, + vis=vis, + waiting=None, + staged=_staged(dwg, found, fetched), + target_fp=None, + minor=False, + ) + # open_or_reaim writes the packet_log label itself + counts["cos_opened" if res.fresh else "cos_reaimed"] += 1 + + +def _stage(sw: Any, co: dict[str, Any], dwg: str, found: Found, fetched: FetchResult) -> None: + staged = _staged(dwg, found, fetched) + sw.conn.execute( + "UPDATE change_orders SET staged = %s, waiting = NULL, row_version = row_version + 1" + " WHERE id = %s AND review = 'pending'", + (Jsonb(staged), co["id"]), + ) + sw.changed("change_orders") + sw.log(f"Drawing for change order {co['co_no']} ready for Clerical", staged["name"]) + + +def _cadence(fs: Any, now: datetime) -> int | None: + return int(fs.drawing_interval_minutes) * 60 + + +SPEC = JobSpec(name="drawing-lookup", run=run, cadence=_cadence, pausable=True) + +__all__ = ["SPEC", "new_flags", "run", "save_lookup", "sharing", "target_for"] diff --git a/server/packetstation/worker/jobs/housekeeping.py b/server/packetstation/worker/jobs/housekeeping.py new file mode 100644 index 0000000..87485d6 --- /dev/null +++ b/server/packetstation/worker/jobs/housekeeping.py @@ -0,0 +1,60 @@ +"""Housekeeping, nightly at 03:30 site time: prune the VISUAL change rows and gone cache rows, old job runs and +old claimed run requests. It never touches packets, the audit log or anything append-only.""" + +from __future__ import annotations + +from datetime import datetime, time + +from ...feed.config import FeedSettings +from . import Counts, JobContext, JobSpec, seconds_until + +AT = time(3, 30) +CACHE_DAYS = 90 +RUNS_DAYS = 180 +REQUESTS_DAYS = 30 + +STEPS: tuple[tuple[str, str, int], ...] = ( + ("row_changes", "DELETE FROM visual_row_changes WHERE at < now() - make_interval(days => %s)", CACHE_DAYS), + ( + "material_changes", + "DELETE FROM visual_material_changes WHERE at < now() - make_interval(days => %s)", + CACHE_DAYS, + ), + ( + "gone_rows", + "DELETE FROM visual_rows WHERE gone_batch IN" + " (SELECT id FROM visual_batches WHERE started_at < now() - make_interval(days => %s))", + CACHE_DAYS, + ), + ( + "gone_materials", + "DELETE FROM visual_materials WHERE gone_batch IN" + " (SELECT id FROM visual_batches WHERE started_at < now() - make_interval(days => %s))", + CACHE_DAYS, + ), + ( + "job_runs", + "DELETE FROM job_runs WHERE started_at < now() - make_interval(days => %s) AND status <> 'running'", + RUNS_DAYS, + ), + ( + "job_requests", + "DELETE FROM job_requests WHERE claimed_at IS NOT NULL AND claimed_at < now() - make_interval(days => %s)", + REQUESTS_DAYS, + ), +) + + +def run(ctx: JobContext) -> Counts: + counts: Counts = {} + for name, sql, days in STEPS: + with ctx.conn.transaction(): + counts[name] = ctx.conn.execute(sql, (days,)).rowcount + return counts + + +def cadence(_: FeedSettings, now_site: datetime) -> int: + return seconds_until(now_site, AT) + + +SPEC = JobSpec(name="housekeeping", run=run, cadence=cadence, pausable=False) diff --git a/server/packetstation/worker/jobs/visual_feed.py b/server/packetstation/worker/jobs/visual_feed.py new file mode 100644 index 0000000..ff0f267 --- /dev/null +++ b/server/packetstation/worker/jobs/visual_feed.py @@ -0,0 +1,141 @@ +"""The visual-feed job: read VISUAL (or the fixture files), record the read in the cache, build the candidates +from the cache and apply them. + +Per run: the feed settings; the tracked keys (every work order of a live VISUAL packet that is not filed, plus the +go-live rows Clerical pulled in); the window from today in site time; one read through the source (labor at most +every 30 minutes); `cache.write_batch`; `cache.rows_for_apply`; `candidates.build`; `apply.apply`. The run is +degraded when a read section failed or the batch is suspect. + +VisualKit: one `Visual.from_env()` per run, closed after, and the stack guard checked against its settings before +the first read. Read-only: VisualKit enforces it. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import Any + +from ... import guard +from ...feed import apply as feed_apply +from ...feed import cache, candidates, config +from ...feed.config import FeedSettings +from ...feed.source import FeedSource, FixtureSource, VisualKitSource +from ...feed.types import FeedData, WoKey +from . import Counts, JobContext, JobSpec + +LABOR_EVERY = timedelta(minutes=30) + + +def tracked_keys(conn: Any) -> list[WoKey]: + """Work orders of live VISUAL packets that are not filed, plus pulled-in go-live rows.""" + rows = conn.execute( + "SELECT DISTINCT w.wo_base, w.lot, w.split, w.sub FROM packet_work_orders w JOIN packets p" + " ON p.id = w.packet_id WHERE p.removed_at IS NULL AND p.status <> 'filed' AND p.source_kind = 'visual'" + " UNION SELECT wo_base, lot, split, sub FROM visual_ignored_wos WHERE pulled_in_at IS NOT NULL" + ).fetchall() + return sorted({WoKey(r[0], r[1], r[2], r[3]) for r in rows}, key=lambda k: k.as_tuple()) + + +def resource_map(conn: Any) -> dict[str, str]: + return {r[0]: r[1] for r in conn.execute("SELECT resource, target FROM visual_resource_map")} + + +def _labor_due(conn: Any, fs: FeedSettings) -> bool: + if not fs.labor_tickets: + return False + last = cache.last_labor_read(conn) + return last is None or datetime.now(UTC) - last >= LABOR_EVERY + + +def _read(source: FeedSource, ctx: JobContext, tracked: list[WoKey], window: tuple[Any, Any], labor: bool) -> FeedData: + return source.read( + site=ctx.settings.site_id, + since=window[0], + until=window[1], + tracked=tracked, + with_materials=True, + with_orders=True, + with_labor=labor, + labor_since=None, + ) + + +def run(ctx: JobContext) -> Counts: + conn = ctx.conn + fs = ctx.feed_settings + today = ctx.today_site() + window = config.window(fs, today) + tracked = tracked_keys(conn) + labor_due = _labor_due(conn, fs) + + if ctx.settings.feed_source == "fixture": + fixture = FixtureSource(ctx.settings.fixture_dir) + describe = fixture.describe() + data = _read(fixture, ctx, tracked, window, labor_due) + else: + from visualkit import Visual + + with Visual.from_env() as v: + # The stack guard against this run's own VisualKit settings, before the first read. + result = guard.check_stack(ctx.settings, v.settings) + ctx.visual_target = ctx.visual_target or result.target + vk = VisualKitSource(v) + describe = vk.describe() + data = _read(vk, ctx, tracked, window, labor_due) + for e in data.errors: + ctx.error(e) + + batch = cache.write_batch(conn, ctx.run_id, data, describe, window) + if batch.suspect: + ctx.degrade(f"suspect read: {batch.reason}; nothing marked gone") + + in_play = {r.key for r in data.rows} | set(tracked) + rows, mats, mat_legs = cache.rows_for_apply(conn, in_play) + if data.material_legs: + mat_legs |= data.material_legs + built = candidates.build( + rows, + mats, + mat_legs, + resource_map=resource_map(conn), + window=window, + golive=fs.golive_date, + departments=fs.departments, + tracked_keys=set(tracked), + today=today, + ) + bases = {r.wo for r in rows} + projects = {candidates.project_of(b) for b in bases} | {c.project for c in built.candidates.values()} + orders = cache.orders(conn, bases=bases | projects, projects=projects) + counts = feed_apply.apply( + conn, + ctx=ctx, + batch=batch, + feed_settings=fs, + built=built, + orders=orders, + labor=cache.labor_hours(conn), + today=today, + seen={r.key: r.st for r in data.rows}, + missing=() if batch.suspect else data.tracked_missing, + ) + counts["wos_window"] = data.window_count + counts["wos_tracked"] = len(tracked) + counts["rows"] = len(data.rows) + counts["materials"] = len(data.materials) if data.materials is not None else 0 + counts["orders"] = int(batch.counts.get("orders", 0)) + counts["labor_legs"] = int(batch.counts.get("labor_legs", 0)) + counts["batch_id"] = batch.id + if batch.suspect: + counts["suspect_reason"] = batch.reason + cache.finish_batch(conn, batch.id, counts) + return counts + + +def _cadence(fs: FeedSettings, now_site: datetime) -> int | None: + return config.interval_seconds(fs, now_site) + + +SPEC = JobSpec(name="visual-feed", run=run, cadence=_cadence, pausable=True, chain=("drawing-lookup",)) + +__all__ = ["SPEC", "resource_map", "run", "tracked_keys"] diff --git a/server/packetstation/worker/redaction.py b/server/packetstation/worker/redaction.py new file mode 100644 index 0000000..1975d91 --- /dev/null +++ b/server/packetstation/worker/redaction.py @@ -0,0 +1,62 @@ +"""Redaction for job errors and log lines: no password, key or token ever reaches job_runs or a log.""" + +from __future__ import annotations + +import os +import re + +MAX_ERRORS = 20 +MAX_ERROR_CHARS = 500 + +# Environment variables whose values are secrets when set (MFilesKit reads MFILES_PASSWORD; VisualKit may read +# VISUAL_PASSWORD). They are read only to strip them from text, never logged. +SECRET_ENV = ("VISUAL_PASSWORD", "MFILES_PASSWORD", "PGPASSWORD", "LDAP_PASSWORD") +SECRET_FILE_ENV = ("VISUAL_PASSWORD_FILE", "MFILES_PASSWORD_FILE") + +_PATTERNS = ( + re.compile(r"(?i)\b(password|passwd|pwd|secret|token|api[_-]?key)\s*[=:]\s*(\"[^\"]*\"|'[^']*'|[^\s;,&]+)"), + re.compile(r"(?i)\bauthorization\s*:\s*[^\r\n]*"), + re.compile(r"(?i)\bx-authentication\s*:\s*[^\r\n]*"), + re.compile(r"(?i)(\w+://[^:/\s@]+:)([^@\s]+)(@)"), +) + + +def _secrets() -> list[str]: + out = [] + for name in SECRET_ENV: + v = os.environ.get(name) + if v and len(v) >= 4: + out.append(v) + for name in SECRET_FILE_ENV: + path = os.environ.get(name) + if not path: + continue + try: + with open(path, encoding="utf-8") as f: + v = f.read().rstrip("\r\n") + except OSError: + continue + if v and len(v) >= 4: + out.append(v) + return out + + +def redact(text: str) -> str: + """Strip anything that looks like a credential, plus the VISUAL and M-Files passwords if set.""" + s = str(text) + for secret in _secrets(): + s = s.replace(secret, "[redacted]") + s = _PATTERNS[0].sub(lambda m: f"{m.group(1)}=[redacted]", s) + s = _PATTERNS[1].sub("Authorization: [redacted]", s) + s = _PATTERNS[2].sub("X-Authentication: [redacted]", s) + s = _PATTERNS[3].sub(lambda m: f"{m.group(1)}[redacted]{m.group(3)}", s) + return s + + +def clip(errors: list[str]) -> list[str]: + """At most MAX_ERRORS redacted strings of at most MAX_ERROR_CHARS each.""" + out = [] + for e in errors[:MAX_ERRORS]: + r = redact(e) + out.append(r if len(r) <= MAX_ERROR_CHARS else r[: MAX_ERROR_CHARS - 3] + "...") + return out diff --git a/server/packetstation/worker/runs.py b/server/packetstation/worker/runs.py new file mode 100644 index 0000000..8137959 --- /dev/null +++ b/server/packetstation/worker/runs.py @@ -0,0 +1,173 @@ +"""One run of one job: advisory lock, job_runs row, request claiming, the job itself, the result. + +`run_job` works with a started Scheduler (the worker) or an unstarted one (the CLI, the tests): it only needs +`settings`, `jobs`, `stack`, `host`, `build`, `connect()`, `set_feed()`, `running_add()`, `running_discard()` +and `after_run()`. +""" + +from __future__ import annotations + +import json +import logging +import traceback +from datetime import datetime +from typing import TYPE_CHECKING, Any + +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb + +from .. import guard +from ..feed import config +from .jobs import JobContext, JobSpec +from .redaction import clip, redact + +if TYPE_CHECKING: + from .scheduler import Scheduler + +log = logging.getLogger("packetstation.worker.runs") + +LOCK_SPACE = 7310 +FEED_JOB = "visual-feed" + +__all__ = ["LOCK_SPACE", "redact", "run_job"] + + +def _jsonable(value: Any) -> Any: + return json.loads(json.dumps(value if value is not None else {}, default=str)) + + +def _row(conn: Any, run_id: int) -> dict[str, Any]: + with conn.cursor(row_factory=dict_row) as cur: + row = cur.execute("SELECT * FROM job_runs WHERE id = %s", (run_id,)).fetchone() + return dict(row) if row else {"id": run_id} + + +def _insert(conn: Any, sch: Scheduler, name: str, trigger: str, *, status: str = "running", counts: Any = None) -> int: + rid = conn.execute( + "INSERT INTO job_runs (job, trigger, status, counts, build_version, host, finished_at)" + " VALUES (%s, %s, %s, %s, %s, %s, CASE WHEN %s THEN now() END) RETURNING id", + (name, trigger, status, Jsonb(_jsonable(counts)), sch.build, sch.host, status != "running"), + ).fetchone()[0] + return int(rid) + + +def _finish(conn: Any, run_id: int, *, status: str, counts: Any, errors: list[str], visual_target: str | None) -> None: + conn.execute( + "UPDATE job_runs SET status = %s, counts = %s, errors = %s, visual_target = coalesce(%s, visual_target)," + " finished_at = now() WHERE id = %s", + (status, Jsonb(_jsonable(counts)), Jsonb(clip(errors)), visual_target, run_id), + ) + + +def _target(result: Any) -> str: + """ " / " from a guard result, without the (DEV) or (PRODUCTION) suffix.""" + text = str(getattr(result, "target", "") or "") + return text.rsplit(" (", 1)[0] if text.endswith(")") else text + + +def run_job(sch: Scheduler, name: str, trigger: str) -> dict[str, Any]: + """Run `name` once and return its job_runs row. Never raises for a job failure; raises when the database + cannot be reached at all (the caller logs it).""" + spec: JobSpec = sch.jobs[name] + s = sch.settings + conn = sch.connect() + locked = False + run_id: int | None = None + status, counts, errors = "failed", {}, [] # type: tuple[str, Any, list[str]] + visual_target: str | None = None + pending_after = False + sch.running_add(name) + try: + feed = config.load(conn) + sch.set_feed(feed) + locked = bool(conn.execute("SELECT pg_try_advisory_lock(%s, hashtext(%s))", (LOCK_SPACE, name)).fetchone()[0]) + if not locked: + rid = _insert(conn, sch, name, trigger, status="skipped", counts={"reason": "already running"}) + log.info("job %s skipped: already running", name) + return _row(conn, rid) + + with conn.transaction(): + run_id = _insert(conn, sch, name, trigger) + claimed = conn.execute( + "UPDATE job_requests SET claimed_run_id = %s, claimed_at = now()" + " WHERE job = %s AND claimed_at IS NULL RETURNING requested_by", + (run_id, name), + ).fetchall() + if claimed: + trigger = "request" + who = sorted({r[0] for r in claimed if r[0] is not None}, key=str) + conn.execute("UPDATE job_runs SET trigger = 'request', requested_by = %s WHERE id = %s", (who, run_id)) + + if trigger == "schedule" and spec.pausable and feed.paused: + status, counts = "skipped", {"reason": "paused"} + else: + ctx = JobContext( + conn=conn, + settings=s, + feed_settings=feed, + run_id=run_id, + trigger=trigger, + started_at=datetime.now(sch.tz), + log=logging.getLogger(f"packetstation.worker.jobs.{name}"), + stack=sch.stack, + ) + log.info("job %s started", name, extra={"ps": {"run_id": run_id, "trigger": trigger}}) + try: + if name == FEED_JOB and s.feed_source == "visualkit": + # The stack guard again with fresh VisualKit settings: a test stack never reads production + ctx.visual_target = _target(guard.check_stack(s)) + counts = spec.run(ctx) or {} + status = "degraded" if ctx.degraded else "ok" + except Exception as e: + status = "failed" + ctx.errors.append(f"{type(e).__name__}: {e}") + log.error("job %s failed: %s", name, redact(traceback.format_exc())) + errors, visual_target = ctx.errors, ctx.visual_target + try: + _finish(conn, run_id, status=status, counts=counts, errors=errors, visual_target=visual_target) + except Exception: + # The run's connection is broken: write the result on a fresh one + with sch.connect() as c2: + _finish(c2, run_id, status=status, counts=counts, errors=errors, visual_target=visual_target) + try: + pending_after = bool( + conn.execute( + "SELECT EXISTS (SELECT 1 FROM job_requests WHERE job = %s AND claimed_at IS NULL)", (name,) + ).fetchone()[0] + ) + row = _row(conn, run_id) + except Exception: + with sch.connect() as c2: + row = _row(c2, run_id) + log.info( + "job %s finished: %s", name, status, extra={"ps": {"run_id": run_id, "status": status, "counts": counts}} + ) + return row + except Exception as e: + status = "failed" + if run_id is not None: + try: + with sch.connect() as c2: + _finish( + c2, + run_id, + status="failed", + counts=counts, + errors=[f"{type(e).__name__}: {e}"], + visual_target=visual_target, + ) + except Exception as e2: + log.error("job %s: can't record the failure: %s", name, type(e2).__name__) + raise + finally: + if locked: + try: + conn.execute("SELECT pg_advisory_unlock(%s, hashtext(%s))", (LOCK_SPACE, name)) + except Exception as e: + log.warning("job %s: unlock failed (%s); closing the connection releases it", name, type(e).__name__) + try: + conn.close() + finally: + sch.running_discard(name) + if run_id is not None: + sch.after_run(name, status, pending_after) diff --git a/server/packetstation/worker/scheduler.py b/server/packetstation/worker/scheduler.py new file mode 100644 index 0000000..f7dae0d --- /dev/null +++ b/server/packetstation/worker/scheduler.py @@ -0,0 +1,395 @@ +"""The worker's scheduler: APScheduler 3 in site time, settings read on every fire, "run now" over NOTIFY. + +Each job has a SettingsTrigger whose next fire time comes from the job's cadence and the current feed settings, +so a changed interval applies on the next fire. A save NOTIFYs `ps_settings` and the listener calls +`reschedule()` so it applies at once. A "run now" NOTIFYs `ps_job_request` and the listener moves the job to now. +After a run the job's next fire is measured from the end of the run. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import socket +import threading +from collections.abc import Callable, Iterable +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any +from zoneinfo import ZoneInfo + +from apscheduler.executors.pool import ThreadPoolExecutor +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.base import BaseTrigger + +from .. import db +from ..feed import config +from ..feed.config import FeedSettings +from ..settings import Settings +from . import heartbeat, runs +from .jobs import JobSpec, Stack, load_jobs +from .redaction import redact + +log = logging.getLogger("packetstation.worker") + +CHANNELS = ("ps_settings", "ps_job_request") +IDLE_RECHECK_SECONDS = 3600 # a job with no schedule wakes this often and does nothing +BACKOFF_MAX = 30.0 +THREADS = 4 +MISFIRE_GRACE = 300 + + +class SettingsTrigger(BaseTrigger): + """Fires `seconds(now)` after now. `seconds` reads the current settings, so a change applies on the next fire. + None means "no schedule": the trigger wakes again after an hour and the fire does nothing.""" + + __slots__ = ("seconds",) + + def __init__(self, seconds: Callable[[datetime], int | None]): + self.seconds = seconds + + def get_next_fire_time(self, previous_fire_time: datetime | None, now: datetime) -> datetime: + secs = self.seconds(now) + return now + timedelta(seconds=max(1, IDLE_RECHECK_SECONDS if secs is None else int(secs))) + + def __str__(self) -> str: + return "settings" + + +def next_due( + spec: JobSpec, feed: FeedSettings, now_site: datetime, last_end_site: datetime | None = None +) -> datetime | None: + """When the job is next due: its cadence measured from the end of its last run (or from now when it never ran), + never before now. None when the cadence says "no schedule".""" + base = last_end_site if last_end_site is not None and last_end_site <= now_site else now_site + secs = spec.cadence(feed, base) + if secs is None: + return None + due = base + timedelta(seconds=max(1, int(secs))) + return due if due > now_site else now_site + + +class Scheduler: + def __init__( + self, + settings: Settings, + *, + build: str | None = None, + jobs: dict[str, JobSpec] | None = None, + connect: Callable[[], Any] | None = None, + ): + self.settings = settings + self.build = build or settings.ps_build + self.jobs = dict(jobs) if jobs is not None else dict(load_jobs()) + self.tz = ZoneInfo(settings.site_tz) + self._connect = connect or (lambda: db.connect_as("worker", settings, autocommit=True)) + self.stack = Stack() + self.host = socket.gethostname() + self.pid = os.getpid() + self.feed = FeedSettings() + self._lock = threading.Lock() + self._running: set[str] = set() + self._next_trigger: dict[str, str] = {} + self._again: dict[str, str] = {} + self._last_end: dict[str, datetime] = {} + self._db_ready = False + self._stop = threading.Event() + self._listener: threading.Thread | None = None + self.listening = False + self.aps = BackgroundScheduler( + executors={"default": ThreadPoolExecutor(THREADS)}, + job_defaults={"coalesce": True, "max_instances": 1, "misfire_grace_time": MISFIRE_GRACE}, + timezone=self.tz, + ) + self.started = False + + # ------------------------------------------------ small helpers used by runs.run_job + + def connect(self) -> Any: + return self._connect() + + def now(self) -> datetime: + return datetime.now(self.tz) + + def set_feed(self, feed: FeedSettings) -> None: + self.feed = feed + + def running(self) -> list[str]: + with self._lock: + return sorted(self._running) + + def running_add(self, name: str) -> None: + with self._lock: + self._running.add(name) + + def running_discard(self, name: str) -> None: + with self._lock: + self._running.discard(name) + + # ------------------------------------------------ lifecycle + + def start(self) -> None: + """Set up the database side when it is reachable, add the jobs and start the scheduler and listener.""" + self._setup_db() + now = self.now() + for name, spec in self.jobs.items(): + self.aps.add_job( + self._fire, + trigger=SettingsTrigger(self._seconds_for(spec)), + args=(name,), + id=name, + name=name, + next_run_time=self._next_time(spec, now), + replace_existing=True, + ) + self.aps.start() + self.started = True + self._write_schedule(self.jobs) + self._stop.clear() + self._listener = threading.Thread(target=self._listen, name="ps-worker-listen", daemon=True) + self._listener.start() + log.info("scheduler started", extra={"ps": {"jobs": sorted(self.jobs), "host": self.host}}) + + def shutdown(self, wait: bool = True) -> None: + self._stop.set() + if self.started: + self.aps.shutdown(wait=wait) + self.started = False + if self._listener is not None: + self._listener.join(5.0) + self._listener = None + + def _setup_db(self) -> bool: + """Seed the feed settings, mark this host's leftover runs abandoned, load the settings and the last run + ends. Safe to call again; returns False when the database is not reachable.""" + if self._db_ready: + return True + try: + with self.connect() as conn: + with conn.transaction(): + if config.seed(conn): + log.info("feed settings seeded with the defaults") + self._abandon(conn) + self.feed = config.load(conn) + for job, end in conn.execute( + "SELECT job, max(finished_at) FROM job_runs WHERE finished_at IS NOT NULL" + " AND status <> 'skipped' GROUP BY job" + ).fetchall(): + if end is not None: + self._last_end[job] = end.astimezone(self.tz) + except Exception as e: + log.warning("worker database setup deferred: %s", redact(f"{type(e).__name__}: {e}")) + return False + self._db_ready = True + return True + + def _abandon(self, conn: Any) -> None: + """Runs left `running` by a dead worker become `abandoned`: this host's always, another host's when no + session holds that job's advisory lock (a container recreated with a new host name).""" + rows = conn.execute("SELECT id, job, host FROM job_runs WHERE status = 'running'").fetchall() + for rid, job, host in rows: + if host != self.host: + got = conn.execute("SELECT pg_try_advisory_lock(%s, hashtext(%s))", (runs.LOCK_SPACE, job)).fetchone()[ + 0 + ] + if not got: + continue + conn.execute("SELECT pg_advisory_unlock(%s, hashtext(%s))", (runs.LOCK_SPACE, job)) + conn.execute( + "UPDATE job_runs SET status = 'abandoned', finished_at = now()," + " errors = errors || '[\"The worker stopped during this run.\"]'::jsonb" + " WHERE id = %s AND status = 'running'", + (rid,), + ) + log.warning("job run %s (%s) marked abandoned", rid, job) + + # ------------------------------------------------ timing + + def _seconds_for(self, spec: JobSpec) -> Callable[[datetime], int | None]: + def seconds(now: datetime) -> int | None: + return spec.cadence(self.feed, now.astimezone(self.tz)) + + return seconds + + def _next_time(self, spec: JobSpec, now: datetime) -> datetime: + due = next_due(spec, self.feed, now, self._last_end.get(spec.name)) + return due if due is not None else now + timedelta(seconds=IDLE_RECHECK_SECONDS) + + def reschedule(self, names: Iterable[str] | None = None) -> None: + """Recompute the next run of every job (or of `names`) from the current settings.""" + if not self.started: + return + now = self.now() + todo = list(names) if names is not None else list(self.jobs) + for name in todo: + spec = self.jobs.get(name) + if spec is None: + continue + with contextlib.suppress(Exception): # JobLookupError during shutdown + self.aps.modify_job(name, next_run_time=self._next_time(spec, now)) + self._write_schedule(todo) + + def run_soon(self, name: str, trigger: str = "request") -> None: + """Move a job to now. A job that is running now runs again right after.""" + if name not in self.jobs: + log.warning("ignoring a run request for unknown job %r", name[:60]) + return + with self._lock: + if name in self._running: + self._again[name] = trigger + return + self._next_trigger[name] = trigger + if self.started: + with contextlib.suppress(Exception): + self.aps.modify_job(name, next_run_time=self.now()) + + def after_run(self, name: str, status: str, pending: bool) -> None: + """Called by run_job when a run ends: chain, run again for new requests, else time the next run from now.""" + self._last_end[name] = self.now() + spec = self.jobs[name] + with self._lock: + again = self._again.pop(name, None) + if status in ("ok", "degraded"): + for c in spec.chain: + self.run_soon(c, "chain") + if pending or again: + self.run_soon(name, again or "request") + self._write_schedule([name]) + else: + self.reschedule([name]) + + def _fire(self, name: str) -> None: + with self._lock: + trigger = self._next_trigger.pop(name, "schedule") + spec = self.jobs[name] + if not self._db_ready and not self._setup_db(): + log.warning("job %s not run: database not reachable", name) + return + if trigger == "schedule" and spec.cadence(self.feed, self.now()) is None: + self._write_schedule([name]) + return + try: + runs.run_job(self, name, trigger) + except Exception as e: + log.error("job %s could not run: %s", name, redact(f"{type(e).__name__}: {e}")) + + def _write_schedule(self, names: Iterable[str]) -> None: + """job_schedule rows for the Admin and Clerical screens. Best effort.""" + if not self.started: + return + rows = [] + now = self.now() + for name in names: + spec = self.jobs.get(name) + if spec is None: + continue + job = self.aps.get_job(name) + nxt = job.next_run_time if job is not None else None + secs = spec.cadence(self.feed, now) + if secs is None: + nxt = None + rows.append((name, nxt, secs, bool(spec.pausable and self.feed.paused))) + if not rows: + return + try: + with self.connect() as conn: + for r in rows: + conn.execute( + "INSERT INTO job_schedule (job, next_run_at, interval_seconds, paused, updated_at)" + " VALUES (%s, %s, %s, %s, now()) ON CONFLICT (job) DO UPDATE SET" + " next_run_at = EXCLUDED.next_run_at, interval_seconds = EXCLUDED.interval_seconds," + " paused = EXCLUDED.paused, updated_at = now()", + r, + ) + except Exception as e: + log.warning("job_schedule not written: %s", type(e).__name__) + + # ------------------------------------------------ settings + + def reload_settings(self, conn: Any | None = None) -> None: + try: + if conn is not None: + self.feed = config.load(conn) + else: + with self.connect() as c: + self.feed = config.load(c) + except Exception as e: + log.warning("feed settings not reloaded: %s", type(e).__name__) + return + self.reschedule() + + # ------------------------------------------------ listener + + def _on_connect(self, conn: Any) -> None: + """After every (re)connect: catch up on what was missed while not listening.""" + self._setup_db() + self.reload_settings(conn) + for (job,) in conn.execute("SELECT DISTINCT job FROM job_requests WHERE claimed_at IS NULL").fetchall(): + self.run_soon(job, "request") + + def _notified(self, channel: str, payload: str) -> None: + if channel == "ps_settings": + self.reload_settings() + return + try: + data = json.loads(payload) + except ValueError: + log.warning("ignoring a non-JSON ps_job_request payload") + return + job = data.get("job") if isinstance(data, dict) else None + if isinstance(job, str): + self.run_soon(job, "request") + + def _listen(self) -> None: + delay = 1.0 + while not self._stop.is_set(): + conn = None + try: + conn = self.connect() + for ch in CHANNELS: + conn.execute(f"LISTEN {ch}") + self.listening = True + delay = 1.0 + log.info("worker listening on %s", ", ".join(CHANNELS)) + self._on_connect(conn) + while not self._stop.is_set(): + for n in conn.notifies(timeout=1.0): + self._notified(n.channel, n.payload) + except Exception as e: + if not self._stop.is_set(): + log.warning("worker listener: %s; retrying in %.0f s", type(e).__name__, delay) + finally: + self.listening = False + if conn is not None: + with contextlib.suppress(Exception): + conn.close() + if self._stop.wait(delay): + break + delay = min(delay * 2, BACKOFF_MAX) + + # ------------------------------------------------ heartbeat loop (main thread) + + def serve(self, beat_file: Path, stop: threading.Event, interval: float) -> None: + """Beat until `stop`: the beat file and the worker_heartbeat row.""" + conn = None + while not stop.is_set(): + try: + heartbeat.touch(beat_file) + except OSError as e: + log.error("can't touch beat file %s: %s", beat_file, e.strerror) + try: + if conn is None or conn.closed: + conn = self.connect() + heartbeat.write_row(conn, build=self.build, host=self.host, pid=self.pid, running=self.running()) + except Exception as e: + log.warning("heartbeat row not written: %s", type(e).__name__) + if conn is not None: + with contextlib.suppress(Exception): + conn.close() + conn = None + stop.wait(interval) + if conn is not None: + with contextlib.suppress(Exception): + conn.close() diff --git a/server/requirements.txt b/server/requirements.txt index 9dd1e62..4faa3b0 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -17,6 +17,8 @@ httpx==0.28.1 tzdata==2026.5 pypdf==6.21.0 python-multipart==0.0.32 +pypdfium2==5.14.0 +openpyxl==3.1.5 # needed by visualkit 0.3.0 and mfileskit 0.1.2 pyodbc==5.3.0 @@ -32,6 +34,7 @@ cffi==2.1.1 click==8.5.0 colorama==0.4.6; platform_system == "Windows" cryptography==50.0.2; sys_platform == "linux" +et-xmlfile==2.0.0 h11==0.16.0 httpcore==1.0.9 httptools==0.9.0 diff --git a/tests/e2e/tests/signin.spec.ts b/tests/e2e/tests/signin.spec.ts index 2c722bd..e5ba389 100644 --- a/tests/e2e/tests/signin.spec.ts +++ b/tests/e2e/tests/signin.spec.ts @@ -38,7 +38,7 @@ test("3. a user with no PS group sees the no-role message", async ({ page }) => test("4. clerk.clara lands on Clerical with no role switcher", async ({ page }) => { await signInPassword(page, "clerk.clara"); await expect(page).toHaveURL(/\/clerical/); - await expect(page.getByText(/Arrives in phase/i)).toBeVisible(); + await expect(page.getByRole("heading", { name: /^Changes to approve/ })).toBeVisible(); await expect(page.locator(".roles")).toHaveCount(0); await expect(topBar(page).getByRole("button", { name: /^Supervisor$/ })).toHaveCount(0); }); diff --git a/tests/e2e/tests/visual-feed.spec.ts b/tests/e2e/tests/visual-feed.spec.ts new file mode 100644 index 0000000..815ec9c --- /dev/null +++ b/tests/e2e/tests/visual-feed.spec.ts @@ -0,0 +1,211 @@ +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { expect, test, type APIRequestContext, type Page } from "@playwright/test"; +import { dcUser, signInPassword } from "./helpers"; + +// Phase 3 VISUAL feed scenario on the CI stack. The worker reads the synthetic fixture files +// (tests/fixtures/generator.py) mounted at /fixtures; this container mounts them read-write so the +// scenario can switch the state. clerk.clara presses Run now, the feed makes project L900 and its +// drafts, the chained drawing run attaches the WELD-PART drawing, the packet goes to the warehouse and +// wh.wade releases it to the shop. State 2 moves that drawing to rev 2: the next run opens a change +// order, the drawing run stages the rev 2 PDF, and clara approves it with her password. + +const FIXTURE_DIR = process.env.FIXTURE_DIR ?? "/fixtures"; +const KEY_WELD_PART = "L900001|WELD-PART|L900-0101"; +const POLL_MS = 60_000; + +type Json = Record; + +function writeHeaders(page: Page): Record { + const base = new URL(page.url()).origin; + return { Origin: base, "X-Requested-With": "PacketStation" }; +} + +async function getJson(req: APIRequestContext, url: string): Promise { + const r = await req.get(url); + expect(r.status(), `${url}: ${await r.text()}`).toBe(200); + return (await r.json()) as Json; +} + +async function lastRunId(page: Page): Promise { + const o = await getJson(page.request, "/api/feed/overview"); + return o.last_run ? Number(o.last_run.id) : 0; +} + +/** Wait until a visual-feed run newer than `after` has finished ok (or degraded). Returns it. */ +async function waitForRun(page: Page, after: number): Promise { + let run: Json | null = null; + await expect + .poll( + async () => { + const o = await getJson(page.request, "/api/feed/overview"); + run = (o.runs as Json[]).find((r) => Number(r.id) > after && r.status !== "running") ?? null; + return run ? run.status : "waiting"; + }, + { timeout: POLL_MS, intervals: [1_000, 2_000] }, + ) + .toMatch(/^(ok|degraded)$/); + return run as unknown as Json; +} + +async function pressRunNow(page: Page): Promise { + const button = page.getByRole("button", { name: /run now/i }).first(); + await expect(button).toBeEnabled({ timeout: POLL_MS }); + await button.click(); +} + +/** The live packet for a candidate key "||", found by its card fields. */ +async function packetByKey(page: Page, key: string): Promise { + const [unit, op, drawing] = key.split("|"); + const list = await getJson(page.request, "/api/packets?limit=2000"); + const card = (list.packets as Json[]).find( + (p) => p.unit?.number === unit && p.op === op && (p.drawing ?? "") === drawing && p.source_kind === "visual", + ); + if (!card) throw new Error(`no packet for ${key}`); + return getJson(page.request, `/api/packets/${card.id}`); +} + +async function transition(page: Page, pkt: Json, action: string, extra: Json = {}) { + return page.request.post(`/api/packets/${pkt.id}/transitions`, { + headers: writeHeaders(page), + data: { action, expect: { status: pkt.status, dept: pkt.dept?.code ?? pkt.dept_code }, ...extra }, + }); +} + +let packetId = 0; + +test.describe.serial("VISUAL feed", () => { + test.setTimeout(300_000); + + test("Run now makes project L900, its drafts and attaches the WELD-PART drawing", async ({ page }) => { + writeFileSync(join(FIXTURE_DIR, "state"), "1"); + await signInPassword(page, "clerk.clara"); + await expect(page).toHaveURL(/\/clerical/); + await expect(page.getByText(/from visual/i).first()).toBeVisible(); + + const before = await lastRunId(page); + await pressRunNow(page); + const run = await waitForRun(page, before); + expect(run.counts.new_drafts).toBeGreaterThan(0); + + const projects = await getJson(page.request, "/api/projects"); + const list = projects as unknown as Json[]; + const l900 = list.find((p) => p.number === "L900"); + expect(l900, "project L900").toBeTruthy(); + const proj = await getJson(page.request, `/api/projects/${l900!.id}`); + const unitNumbers = ((proj.units ?? []) as Json[]).map((u) => u.number); + expect(unitNumbers).toEqual(expect.arrayContaining(["L900001", "L900002"])); + + let pkt = await packetByKey(page, KEY_WELD_PART); + packetId = Number(pkt.id); + expect(pkt.status).toBe("draft"); + // Either still waiting for its drawing, or the chained drawing job already attached it + const flags = pkt.flags as string[]; + expect(flags.includes("drawing_needed") || pkt.drawing_state === "attached").toBe(true); + + // The drawing job runs right after the feed (chained) + await expect + .poll(async () => (await getJson(page.request, `/api/packets/${packetId}`)).drawing_state, { + timeout: POLL_MS, + intervals: [1_000, 2_000], + }) + .toBe("attached"); + await page.goto(`/packets/${packetId}`); + // The Drawing tab shows the pages the feed attached from M-Files (the fixture vault) + const full = await getJson(page.request, `/api/packets/${packetId}`); + expect(((full.pages ?? []) as Json[]).length).toBeGreaterThan(0); + await expect(page.getByRole("tab", { name: /^Drawing/ }).or(page.getByRole("button", { name: /^Drawing/ })).first()).toBeVisible(); + + // Send it on, unless the feed auto-released it already + pkt = await getJson(page.request, `/api/packets/${packetId}`); + if (pkt.status === "draft") { + if (pkt.heat_map === null || pkt.heat_map === undefined) { + const r = await page.request.patch(`/api/packets/${packetId}`, { + headers: writeHeaders(page), + data: { set: { heat_map: true }, base: {} }, + }); + expect(r.status(), await r.text()).toBe(200); + pkt = await getJson(page.request, `/api/packets/${packetId}`); + } + const r = await transition(page, pkt, "send_to_warehouse"); + if (r.status() !== 200) { + // the worker may have auto-released it between the two reads + pkt = await getJson(page.request, `/api/packets/${packetId}`); + expect(pkt.status, await r.text()).toBe("warehouse"); + } + } + pkt = await getJson(page.request, `/api/packets/${packetId}`); + expect(pkt.status).toBe("warehouse"); + }); + + test("wh.wade releases it to the shop", async ({ page }) => { + await signInPassword(page, "wh.wade"); + let pkt = await getJson(page.request, `/api/packets/${packetId}`); + // Release needs every material line pulled or short: pull each one, as the warehouse would + for (const line of (pkt.bom ?? []) as Json[]) { + if (line.pulled || line.na) continue; + const pr = await page.request.patch(`/api/packets/${packetId}/bom/${line.item}`, { + headers: { ...writeHeaders(page), "If-Match": String(line.row_version) }, + data: { pulled: true }, + }); + expect(pr.status(), await pr.text()).toBe(200); + } + pkt = await getJson(page.request, `/api/packets/${packetId}`); + const r = await transition(page, pkt, "wh_release"); + expect(r.status(), await r.text()).toBe(200); + expect((await r.json()).status).toBe("ready"); + }); + + test("VISUAL moves the drawing to rev 2: a change order, staged, approved", async ({ page }) => { + await signInPassword(page, "clerk.clara"); + await expect(page).toHaveURL(/\/clerical/); + writeFileSync(join(FIXTURE_DIR, "state"), "2"); + const before = await lastRunId(page); + await pressRunNow(page); + const run = await waitForRun(page, before); + expect(run.counts.cos_opened).toBeGreaterThan(0); + + const cos = await getJson(page.request, `/api/packets/${packetId}/change-orders`); + const list = cos as unknown as Json[]; + const co = list.find((c) => c.review === "pending"); + expect(co, "a pending change order").toBeTruthy(); + + // The Changes tab on the packet and the Clerical home both show it + await page.goto(`/packets/${packetId}`); + await page.getByRole("tab", { name: /changes/i }).or(page.getByRole("button", { name: /^changes/i })).first().click(); + await expect(page.getByText(/waiting on clerical/i).first()).toBeVisible(); + await page.goto("/clerical"); + await expect(page.getByText(/changes to approve/i).first()).toBeVisible(); + await expect(page.getByText("L900001").first()).toBeVisible(); + + // The chained drawing run stages the rev 2 PDF + await expect + .poll( + async () => { + const all = await getJson(page.request, `/api/packets/${packetId}/change-orders`); + const c = (all as unknown as Json[]).find((x) => x.co_no === co!.co_no); + return c?.staged ? "staged" : "waiting"; + }, + { timeout: POLL_MS, intervals: [1_000, 2_000] }, + ) + .toBe("staged"); + + const clara = dcUser("clerk.clara"); + const r = await page.request.post(`/api/packets/${packetId}/change-orders/${co!.co_no}/approve`, { + headers: writeHeaders(page), + data: { sign: { factor: "password", secret: clara.password } }, + }); + expect(r.status(), await r.text()).toBe(200); + + const pkt = await getJson(page.request, `/api/packets/${packetId}`); + expect(pkt.rev).toBe("2"); + // The old rev 1 pages are kept and voided; the rev 2 pages carry the change order + const pages = (pkt.pages ?? []) as Json[]; + expect(pages.some((pg) => pg.void)).toBe(true); + expect(pages.some((pg) => !pg.void && pg.rev === "2")).toBe(true); + await page.goto(`/packets/${packetId}`); + await expect(page.getByText(`${pkt.drawing} R2`, { exact: true }).first()).toBeVisible(); + // The Drawing tab lists the voided rev 1 page beside the new rev 2 page + await expect(page.getByRole("button", { name: /\bvoid\b/i }).first()).toBeVisible(); + }); +}); diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..b930bbc --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,67 @@ +# Synthetic VISUAL and M-Files fixtures + +Everything here is made up. Real job data never goes in git: recordings from DEV stay in +`tests/fixtures/local/` (ignored). + +``` +python -m tests.fixtures.generator --out tests/fixtures/generated +``` + +writes `feed.1.json`, `feed.2.json`, `drawings.1.json`, `drawings.2.json`, the one-page PDFs the drawing files +name, and `state` (`1`). `tests/fixtures/generated/` is ignored by git. CI runs the command before +`compose up` and mounts the folder at `/fixtures` in the worker (read-only) and Playwright (read-write). +With `FEED_SOURCE=fixture` and `DRAWING_SOURCE=fixture` the worker reads `FIXTURE_DIR/state` and the +matching files instead of VISUAL and M-Files. Write `2` to `state` to switch. + +The feed files have the shape of VisualKit's `examples/packet_station_feed.py`: rows carry the status as a +word (Firmed, Released, Unreleased, Closed, Cancelled), materials carry the status code, numbers are JSON +numbers, dates are ISO, lot, split and sub are text, seq and piece are integers. An extra `labor` list holds +hours per operation. + +Dates are offsets from the day the generator runs (`--today` overrides), so the window rows sit inside the +default window. The output is the same for the same day. + +## State 1 + +Project L900, units L900001 and L900002, plus L901001 under project L901. + +| Work order | Lot | What it covers | +| --- | --- | --- | +| L900-01 | 1 | LAYOUT and WELD-PART on drawing L900-0101 rev 1 (two packets), MFG and MAT_PIPE fold, FORKLIFT ignored; split-item header (piece 10 with 11 and 12), MAIN warehouse line, WIP sub-assembly, a line on another drawing (dropped) | +| L900-01 split 1 | 1 | Unreleased (only on a packet when tracked) | +| L900-02 | 1 | two WELD-CODE operations on one drawing: two legs on one packet; VISUAL rev 2, M-Files rev 3 (rev mismatch) | +| L900-03 | 1 | no drawing (`(no drawing)` key) | +| L900-04, L900-08-1 | 2 | one packet, rev `A` (bad rev), piece 1 on both legs (items `1` and `1-1`), a base linked to order C24-0901 | +| L900-05 | 2 | drawing in drafting (not released) | +| L900-06 | 2 | Released, with materials: only a packet when tracked | +| L900-07 | 2 | drawing not in M-Files | +| L900-09, L900-10 | 1 | close and cancel in state 2 | +| L900-11, L900-12 | 1 | Closed and Cancelled, only when tracked | +| L900-13 | 2 | released 10 days back: ignored when the tests set go-live 5 days back | +| L900-14 | 7A | a lettered lot: no unit, no packet | +| L900-15 | 2 | resource NEWRES, not in the map | +| L900-16 | 2 | moves to drawing L900-1602 in state 2 | +| L901-01 | 1 | project L901, only a closed order | + +Orders: L900 has two open orders. C24-0901 (Acme Refining) names top job L900002 and is linked to base +L900-04. C24-0902 (Bayou Chemical) names L900001, so the unit rule picks it for L900001. L901 has one closed +order (Delta Gas). + +## State 2 + +- L900-01 rev 1 to 2 (a change order on an in-shop packet; the LAYOUT draft updates in place). +- L900-02 qty 2 to 3 (a draft updated in place). +- L900-07 target hours only (a minor change order once past draft). +- L900-05 gone from VISUAL (flagged after two runs). +- L900-16 on drawing L900-1602 (the old draft retires, a new one is made). +- L900-09 Closed, L900-10 Cancelled. +- NEWRES still unmapped. + +## Drawings + +`drawings.1.json`: L900-0101 rev 1 (attach), L900-0201 rev 3 (VISUAL calls 2), L900-0401 rev 1 (VISUAL calls +A), L900-0501 in drafting, L900-0701 and L900-1601 missing (not found), the rest at rev 1. +`drawings.2.json`: the same with L900-0101 at rev 2 (a new file, so the change order gets staged) and L900-1602. + +`generate(out_dir)`, `feed_state(n)` and `drawings_state(n)` are importable; `tests/server/feedhelpers.py` +uses them. diff --git a/tests/fixtures/generator.py b/tests/fixtures/generator.py new file mode 100644 index 0000000..b2dc1ab --- /dev/null +++ b/tests/fixtures/generator.py @@ -0,0 +1,672 @@ +"""Synthetic VISUAL feed and M-Files drawing fixtures for tests and CI. No real job data. + + python -m tests.fixtures.generator --out tests/fixtures/generated + +Writes feed.1.json, feed.2.json, drawings.1.json, drawings.2.json, the one-page PDFs the drawing files name, and +`state` = "1". The feed files have the exact shape of VisualKit's examples/packet_station_feed.py: rows carry the +status as a word, materials carry the status code, numbers are JSON numbers, dates are ISO, lot/split/sub are text, +seq and piece are ints. An optional `labor` list follows the FixtureSource contract. + +Dates are offsets from `today` (default: the date the generator runs), so the window rows stay inside the feed's +default window (14 days back, 30 ahead) on the day the files are made. Everything else is fixed: the output is the +same for the same `today`. +""" + +from __future__ import annotations + +import argparse +import io +import json +import sys +from datetime import date, timedelta +from pathlib import Path +from typing import Any + +SITE = "PESCO" +STATUS_WORDS = {"F": "Firmed", "R": "Released", "U": "Unreleased", "C": "Closed", "X": "Cancelled"} + +# The part of VISUAL's resource map the fixtures use (the seed in migration 0004 has these too). NEWRES is unmapped. +RESOURCE_MAP: dict[str, str] = { + "LAYOUT": "01", + "WELD-PART": "03", + "WELD-CODE": "06", + "MFG": "fold", + "FORKLIFT": "ignore", +} + +# Go-live: the tests set feed settings golive_date = today + GOLIVE_OFFSET. Work order L900-13 is released before it. +GOLIVE_OFFSET = -5 + +# Names the tests and the e2e scenario refer to. +PROJECT = "L900" +PROJECT_CLOSED_ORDER = "L901" +UNIT1, UNIT2 = "L900001", "L900002" +KEY_WELD_PART = "L900001|WELD-PART|L900-0101" # rev 1 -> 2 in state 2; the in-shop packet in the scenario +KEY_LAYOUT_A = "L900001|LAYOUT|L900-0101" # same work order, another resource: a second packet +KEY_TWO_OPS = "L900001|WELD-CODE|L900-0201" # two operations, same resource and drawing; qty change in state 2 +KEY_NO_DWG = "L900001|WELD-PART|(no drawing)" +KEY_REV_A = "L900002|WELD-PART|L900-0401" # non-numeric rev; two legs with duplicate pieces +KEY_DRAFTING = "L900002|LAYOUT|L900-0501" # drawing in drafting; this leg is gone from VISUAL in state 2 +KEY_NOT_FOUND = "L900002|WELD-PART|L900-0701" # drawing not in M-Files; tgt-only change in state 2 (minor) +KEY_CLOSES = "L900001|WELD-CODE|L900-0901" # closed in VISUAL in state 2 +KEY_CANCELS = "L900001|WELD-CODE|L900-1001" # cancelled in VISUAL in state 2 +KEY_MOVES = "L900002|WELD-CODE|L900-1601" # moves to drawing L900-1602 in state 2 (retire-moved) +KEY_MOVED_TO = "L900002|WELD-CODE|L900-1602" +KEY_GOLIVE = "L900002|WELD-PART|L900-1301" # released before go-live when the tests set it +KEY_L901 = "L901001|WELD-PART|L901-0101" +KEY_RELEASED = "L900002|WELD-CODE|L900-0601" # Released (not Firmed): only a candidate when tracked +DWG_WELD_PART = "L900-0101" + +# Every candidate state 1 makes on the first run (no tracked keys, no go-live). +STATE1_KEYS = frozenset( + { + KEY_WELD_PART, + KEY_LAYOUT_A, + KEY_TWO_OPS, + KEY_NO_DWG, + KEY_REV_A, + KEY_DRAFTING, + KEY_NOT_FOUND, + KEY_CLOSES, + KEY_CANCELS, + KEY_MOVES, + KEY_GOLIVE, + KEY_L901, + } +) + +ORDER_ACME = "C24-0901" # open, top job L900002, also linked to base L900-04 +ORDER_BAYOU = "C24-0902" # open, top job L900001 (the unit rule must pick it for L900001) +ORDER_L901 = "C23-0450" # closed, the only order of L901 + + +def _d(today: date, days: int | None) -> str | None: + return None if days is None else (today + timedelta(days=days)).isoformat() + + +def _num(x: float | int) -> int | float: + """A JSON number as the feed writes it: integers without '.0'.""" + return int(x) if float(x).is_integer() else float(x) + + +class _Wo: + """One work order leg with its operations, as the generator describes it.""" + + def __init__( + self, + base: str, + lot: str, + ops: list[tuple[int, str, float]], + *, + dwg: str, + rev: str, + st: str, + rel: int, + want: int, + qty: float, + desc: str, + split: str = "0", + sub: str = "0", + part: str = "", + ps: dict[int, str] | None = None, + ) -> None: + self.base, self.lot, self.split, self.sub = base, lot, split, sub + self.ops = ops # (seq, resource, run hours per piece) + self.dwg, self.rev, self.st, self.rel, self.want, self.qty = dwg, rev, st, rel, want, qty + self.desc, self.part, self.ps = desc, part or f"FG-{base}", ps or {} + + def rows(self, today: date) -> list[dict[str, Any]]: + out = [] + for seq, res, run in self.ops: + out.append( + { + "wo": self.base, + "lot": self.lot, + "split": self.split, + "sub": self.sub, + "seq": seq, + "res": res, + "dwg": self.dwg, + "rev": self.rev, + "ps": self.ps.get(seq, ""), + "desc": self.desc, + "qty": _num(self.qty), + "tgt": _num(round(run * self.qty, 4)), + "rel": _d(today, self.rel), + "want": _d(today, self.want), + "st": STATUS_WORDS[self.st], + "barcode": f"*%2{self.base}${self.lot}${self.split}${self.sub}${seq}%*", + "part": self.part, + "site": SITE, + "order": None, + } + ) + return out + + +def _mat( + wo: _Wo, + seq: int, + piece: int, + part: str | None, + qty: float, + *, + issued: float = 0, + whs: str = "SHOP", + loc: str = "R1", + comm: str = "PIPE", + cons: str = "N", + desc: str | None = None, + dwg: str | None = None, + st: str | None = None, + closed: bool = False, +) -> dict[str, Any]: + calc = qty + if issued > calc: + istat = 3 + elif closed: + istat = 0 + elif issued == 0: + istat = 2 + else: + istat = 1 + return { + "wo": wo.base, + "lot": wo.lot, + "split": wo.split, + "sub": wo.sub, + "seq": seq, + "piece": piece, + "part": part, + "qty": _num(qty), + "issued": _num(issued), + "whs": whs if part else "", + "loc": loc if part else "", + "st": st or wo.st, + "istat": istat, + "dwg": wo.dwg if dwg is None else dwg, + "desc": desc if desc is not None else (f"Part {part}" if part else "Split item"), + "comm": comm if part else "", + "cons": cons if part else "", + } + + +def _work_orders() -> dict[str, _Wo]: + """State 1's work orders by a short name.""" + w: dict[str, _Wo] = {} + # A: two operations at different resources on one drawing (two packets, one BOM each), plus fold and ignore rows + w["A"] = _Wo( + "L900-01", + "1", + [(10, "LAYOUT", 0.5), (20, "WELD-PART", 1.5), (30, "MFG", 0.25), (40, "FORKLIFT", 0.1), (50, "MAT_PIPE", 0)], + dwg="L900-0101", + rev="1", + st="F", + rel=-3, + want=20, + qty=2, + desc="Inlet spool 6 in", + ps={10: "Cut to length per sheet 1", 20: "Weld per WPS-11\r\nBack purge", 30: "Weld per WPS-11\r\nBack purge"}, + ) + # A2: an Unreleased split of A's base (only on a packet when tracked) + w["A2"] = _Wo( + "L900-01", + "1", + [(20, "WELD-PART", 1.5)], + dwg="L900-0101", + rev="1", + st="U", + rel=-3, + want=25, + qty=1, + desc="Inlet spool 6 in", + split="1", + ) + # B: two operations at the same resource and drawing (two legs with different seq on one packet) + w["B"] = _Wo( + "L900-02", + "1", + [(10, "WELD-CODE", 2), (20, "WELD-CODE", 1)], + dwg="L900-0201", + rev="2", + st="F", + rel=-2, + want=21, + qty=2, + desc="Header box", + ) + # C: no drawing + w["C"] = _Wo( + "L900-03", "1", [(10, "WELD-PART", 0.75)], dwg="", rev="", st="F", rel=-1, want=18, qty=4, desc="Pipe supports" + ) + # D and H: one candidate (same unit, resource, drawing) with a non-numeric rev; duplicate piece 1 across the legs + w["D"] = _Wo( + "L900-04", + "2", + [(10, "WELD-PART", 1)], + dwg="L900-0401", + rev="A", + st="F", + rel=-4, + want=22, + qty=1, + desc="Outlet nozzle", + ) + w["H"] = _Wo( + "L900-08-1", + "2", + [(10, "WELD-PART", 0.5)], + dwg="L900-0401", + rev="", + st="F", + rel=-2, + want=24, + qty=1, + desc="Outlet nozzle", + ) + # E: drawing in drafting; gone from VISUAL in state 2 + w["E"] = _Wo( + "L900-05", "2", [(10, "LAYOUT", 1)], dwg="L900-0501", rev="1", st="F", rel=-1, want=19, qty=3, desc="Skid frame" + ) + # F: Released, with materials (R legs count as mat_known); not in the window set + w["F"] = _Wo( + "L900-06", + "2", + [(10, "WELD-CODE", 1)], + dwg="L900-0601", + rev="1", + st="R", + rel=-2, + want=20, + qty=1, + desc="Drain leg", + ) + # G: drawing not in M-Files; tgt-only change in state 2 + w["G"] = _Wo( + "L900-07", + "2", + [(10, "WELD-PART", 2)], + dwg="L900-0701", + rev="1", + st="F", + rel=-3, + want=23, + qty=2, + desc="Vent stack", + ) + # J closes in state 2, K is cancelled in state 2 + w["J"] = _Wo( + "L900-09", + "1", + [(10, "WELD-CODE", 1)], + dwg="L900-0901", + rev="1", + st="F", + rel=-5, + want=15, + qty=1, + desc="Gauge board", + ) + w["K"] = _Wo( + "L900-10", + "1", + [(10, "WELD-CODE", 1)], + dwg="L900-1001", + rev="1", + st="F", + rel=-5, + want=15, + qty=1, + desc="Sample cooler", + ) + # A Closed and a Cancelled leg outside the window set (only on a packet when tracked) + w["L"] = _Wo( + "L900-11", + "1", + [(10, "WELD-CODE", 1)], + dwg="L900-1101", + rev="1", + st="C", + rel=-6, + want=10, + qty=1, + desc="Closed spool", + ) + w["X"] = _Wo( + "L900-12", + "1", + [(10, "WELD-CODE", 1)], + dwg="L900-1201", + rev="1", + st="X", + rel=-6, + want=10, + qty=1, + desc="Cancelled spool", + ) + # Go-live: released before golive_date = today + GOLIVE_OFFSET (still inside the window) + w["P"] = _Wo( + "L900-13", + "2", + [(10, "WELD-PART", 1)], + dwg="L900-1301", + rev="1", + st="F", + rel=-10, + want=12, + qty=1, + desc="Old release", + ) + # A non-numeric lot: no unit number, no candidate + w["Q"] = _Wo( + "L900-14", + "7A", + [(10, "WELD-PART", 1)], + dwg="L900-1401", + rev="1", + st="F", + rel=-2, + want=20, + qty=1, + desc="Lettered lot", + ) + # An unmapped resource + w["N"] = _Wo( + "L900-15", + "2", + [(10, "NEWRES", 1), (20, "NEWRES", 1)], + dwg="L900-1501", + rev="1", + st="F", + rel=-2, + want=20, + qty=1, + desc="New resource work", + ) + # M: moves to drawing L900-1602 in state 2 + w["M"] = _Wo( + "L900-16", + "2", + [(10, "WELD-CODE", 1)], + dwg="L900-1601", + rev="1", + st="F", + rel=-2, + want=26, + qty=1, + desc="Bypass loop", + ) + # Project L901: only a closed order + w["Z"] = _Wo( + "L901-01", + "1", + [(10, "WELD-PART", 1)], + dwg="L901-0101", + rev="1", + st="F", + rel=-2, + want=28, + qty=1, + desc="Closed-order spool", + ) + return w + + +def _materials(w: dict[str, _Wo]) -> list[dict[str, Any]]: + a, d, h, f, b = w["A"], w["D"], w["H"], w["F"], w["B"] + lines = [ + # A seq 10 (LAYOUT): a split-item header (piece 10, no part) with lines 11 and 12 + _mat(a, 10, 10, None, 0, desc="Split item: flange set"), + _mat(a, 10, 11, "FLG-6-150", 2, issued=1), + _mat(a, 10, 12, "GSK-6-150", 2, cons="Y", comm="GASKET"), + # A seq 20 (WELD-PART): a MAIN warehouse line, a WIP sub-assembly, a line on another drawing (dropped) + _mat(a, 20, 1, "PIPE-6-SCH40", 12.5, issued=12.5, whs="MAIN", loc="A-01"), + _mat(a, 20, 2, "L900-01-SUB", 1, whs="ASY", loc="", comm="WIP"), + _mat(a, 20, 3, "ELB-6-90", 2, dwg="L900-9999"), + _mat(a, 20, 4, "WOL-1-6", 1, dwg=""), # blank drawing on the line: it belongs + # B (two operations) + _mat(b, 10, 1, "PLT-0.5-A36", 4), + _mat(b, 20, 2, "CPL-2-3000", 2, issued=3), + # D and H: piece 1 on both legs (the second becomes item 1-1) + _mat(d, 10, 1, "NOZ-4-300", 1), + _mat(h, 10, 1, "NOZ-4-300", 1), + _mat(h, 10, 2, "PAD-4", 1, closed=True, issued=1), + # F: a Released leg's line + _mat(f, 10, 1, "PIPE-1-SCH80", 3, st="R"), + # G + _mat(w["G"], 10, 1, "PIPE-8-SCH40", 6), + # Unreleased split of A: not F or R, so not a material leg + _mat(w["A2"], 20, 1, "PIPE-6-SCH40", 6, st="U"), + ] + return lines + + +def _orders(today: date) -> tuple[dict[str, list[dict[str, Any]]], dict[str, list[dict[str, Any]]]]: + acme = { + "cust_order_id": ORDER_ACME, + "lines": [1], + "top_jobs": ["L900002"], + "status": "R", + "customer_id": "ACME01", + "customer_name": "Acme Refining", + "customer_po_ref": "PO-1001", + "order_date": _d(today, -40), + "desired_ship_date": _d(today, 60), + "promise_date": _d(today, 55), + } + bayou = { + "cust_order_id": ORDER_BAYOU, + "lines": [1, 2], + "top_jobs": ["L900001"], + "status": "F", + "customer_id": "BAYOU01", + "customer_name": "Bayou Chemical", + "customer_po_ref": "PO-2002", + "order_date": _d(today, -45), + "desired_ship_date": _d(today, 70), + "promise_date": None, + } + closed = { + "cust_order_id": ORDER_L901, + "lines": [1], + "top_jobs": ["L901001"], + "status": "C", + "customer_id": "DELTA01", + "customer_name": "Delta Gas", + "customer_po_ref": "PO-3003", + "order_date": _d(today, -200), + "desired_ship_date": _d(today, -100), + "promise_date": _d(today, -105), + } + base_entry = {k: v for k, v in acme.items() if k not in ("top_jobs", "order_date")} + by_base = {"L900-04": [base_entry]} + by_project = {PROJECT: [acme, bayou], PROJECT_CLOSED_ORDER: [closed]} + return by_base, by_project + + +def _labor(today: date) -> list[dict[str, Any]]: + return [ + { + "wo": "L900-01", + "lot": "1", + "split": "0", + "sub": "0", + "seq": 20, + "hours": 1.25, + "tickets": 2, + "last_ticket": _d(today, -1), + }, + { + "wo": "L900-02", + "lot": "1", + "split": "0", + "sub": "0", + "seq": 10, + "hours": 0.5, + "tickets": 1, + "last_ticket": _d(today, -1), + }, + ] + + +def _meta(today: date, state: int) -> dict[str, Any]: + return { + "source": "fixture", + "version": "synthetic", + "server": "fixture", + "database": f"state {state}", + "site": SITE, + "since": _d(today, -14), + "until": _d(today, 30), + "statuses": ["F", "R", "U", "C"], + "generated_at": f"{today.isoformat()}T06:00:00", + } + + +def feed_state(n: int, today: date | None = None) -> dict[str, Any]: + """The feed file for state 1 or 2 as a dict (the contract shape plus `labor`).""" + if n not in (1, 2): + raise ValueError("state is 1 or 2") + today = today or date.today() + w = _work_orders() + if n == 2: + w["A"].rev = "2" # rev 1 -> 2 on the WELD-PART drawing (and A's LAYOUT draft) + w["A2"].rev = "2" + w["B"].qty = 3 # qty change (also scales tgt) + w["G"].ops = [(10, "WELD-PART", 2.5)] # tgt-only change: minor + del w["E"] # gone from VISUAL + w["M"].dwg = "L900-1602" # moved to another drawing + w["J"].st = "C" + w["K"].st = "X" + rows = [r for wo in w.values() for r in wo.rows(today)] + mats = _materials(w) if n == 1 else _materials_state2(w) + by_base, by_project = _orders(today) + return { + "meta": _meta(today, n), + "rows": rows, + "materials": mats, + "orders_by_base": by_base, + "orders_by_project": by_project, + "labor": _labor(today), + } + + +def _materials_state2(w: dict[str, _Wo]) -> list[dict[str, Any]]: + # Same lines as state 1 on the state 2 work orders (A's drawing moved with it, E is gone) + full = _work_orders() + full.update(w) + lines = _materials(full) + return [m for m in lines if m["wo"] != "L900-05"] + + +# ------------------------------------------------------------------------------------------------ drawings + + +def _dwg( + rev: int | None, + file: str | None, + *, + n: int, + version: int = 1, + draft_rev: int | None = None, + draft_where: str = "", + in_drafting: bool = False, + base: str | None = None, +) -> dict[str, Any]: + return { + "rev": rev, + "file": file, + "name": file[:-4] if file else None, + "draft_rev": draft_rev, + "draft_where": draft_where, + "in_drafting": in_drafting, + "ok": True, + "note": "", + "base": base, + "obj_id": n, + "obj_version": version, + "file_id": 100 + n, + "file_version": version, + } + + +def drawings_state(n: int) -> dict[str, Any]: + """drawings..json as a dict. A drawing missing from it is not in M-Files.""" + if n not in (1, 2): + raise ValueError("state is 1 or 2") + d: dict[str, Any] = { + "L900-0101": _dwg(1, "L900-0101 r1.pdf", n=1), + "L900-0201": _dwg(3, "L900-0201 r3.pdf", n=2), # VISUAL calls rev 2: rev mismatch + "L900-0401": _dwg(1, "L900-0401 r1.pdf", n=4), # VISUAL calls rev A: bad rev + "L900-0501": _dwg( + None, None, n=5, draft_rev=1, draft_where="in Drafting review", in_drafting=True, base="L900-05" + ), # not released + # L900-0701 is missing: not found. L900-1601 is missing too (so its draft has no pages and can retire). + "L900-0601": _dwg(1, "L900-0601 r1.pdf", n=6), + "L900-0901": _dwg(1, "L900-0901 r1.pdf", n=9), + "L900-1001": _dwg(1, "L900-1001 r1.pdf", n=10), + "L900-1301": _dwg(1, "L900-1301 r1.pdf", n=13), + "L900-1401": _dwg(1, "L900-1401 r1.pdf", n=14), + "L901-0101": _dwg(1, "L901-0101 r1.pdf", n=21), + } + if n == 2: + d["L900-0101"] = _dwg(2, "L900-0101 r2.pdf", n=1, version=2) # the CO gets staged + d["L900-1602"] = _dwg(1, "L900-1602 r1.pdf", n=16) + return d + + +def pdf_bytes(label: str) -> bytes: + """A one-page PDF. The label goes into the document info so each file has its own sha256.""" + from pypdf import PdfWriter + + w = PdfWriter() + w.add_blank_page(width=792, height=612) + w.add_metadata({"/Title": label, "/Producer": "Packet Station fixtures"}) + buf = io.BytesIO() + w.write(buf) + return buf.getvalue() + + +def write_state(out_dir: Path, n: int) -> None: + """Point the fixture sources at state n (what `packetstation fixtures use N` does).""" + (Path(out_dir) / "state").write_text(str(n), encoding="utf-8") + + +def generate(out_dir: Path | str, today: date | None = None) -> Path: + """Write every fixture file into out_dir and set the state to 1. Returns out_dir.""" + out = Path(out_dir) + out.mkdir(parents=True, exist_ok=True) + files: set[str] = set() + for n in (1, 2): + feed = feed_state(n, today) + (out / f"feed.{n}.json").write_text(json.dumps(feed, indent=1, ensure_ascii=False), encoding="utf-8") + dwgs = drawings_state(n) + (out / f"drawings.{n}.json").write_text(json.dumps(dwgs, indent=1, ensure_ascii=False), encoding="utf-8") + files.update(v["file"] for v in dwgs.values() if v.get("file")) + for name in sorted(files): + (out / name).write_bytes(pdf_bytes(name)) + write_state(out, 1) + return out + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Write Packet Station's synthetic VISUAL and M-Files fixtures") + ap.add_argument("--out", type=Path, required=True, help="directory to write into") + ap.add_argument("--today", type=date.fromisoformat, default=None, help="date the offsets count from") + args = ap.parse_args(argv) + out = generate(args.out, args.today) + print(f"Wrote fixtures to {out} (state 1)", file=sys.stderr) + return 0 + + +__all__ = [ + "RESOURCE_MAP", + "drawings_state", + "feed_state", + "generate", + "pdf_bytes", + "write_state", +] + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/server/feedhelpers.py b/tests/server/feedhelpers.py new file mode 100644 index 0000000..e85863e --- /dev/null +++ b/tests/server/feedhelpers.py @@ -0,0 +1,241 @@ +"""Helpers for the feed and drawing tests: the generator, fixture dirs, feed settings and one job run. + +`run_feed(settings, state)` and `run_drawings(settings)` run the real job functions (visual-feed, drawing-lookup) +once, the way the worker would, over a connection as the worker role, against the fixture files in +`settings.fixture_dir`. Never VISUAL or M-Files. +""" + +from __future__ import annotations + +import importlib +import sys +from datetime import date, timedelta +from pathlib import Path +from typing import Any + +from pshelpers import make_settings +from psycopg.types.json import Jsonb + +from packetstation.settings import Settings + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +gen = importlib.import_module("tests.fixtures.generator") + +FEED_JOB = "visual-feed" +DRAWING_JOB = "drawing-lookup" +FEED_COUNT_KEYS = ( + "wos_window", + "wos_tracked", + "rows", + "materials", + "orders", + "labor_legs", + "new_drafts", + "drafts_updated", + "cos_opened", + "cos_reaimed", + "cos_withdrawn", + "retired", + "wo_gone_flagged", + "closed_observed", + "ignored_golive", + "projects_created", + "units_created", + "auto_released", + "unmapped_resources", + "skipped_removed", + "skipped_closed", + "suspect", +) +# The counts that are zero when VISUAL did not change between two runs. +CHANGE_KEYS = ( + "new_drafts", + "drafts_updated", + "cos_opened", + "cos_reaimed", + "cos_withdrawn", + "retired", + "wo_gone_flagged", + "closed_observed", + "projects_created", + "units_created", + "auto_released", +) + + +def fixture_dir(tmp: Path, today: date | None = None) -> Path: + """Generate the fixtures into tmp/fixtures (state 1).""" + out: Path = gen.generate(tmp / "fixtures", today) + return out + + +def switch_state(directory: Path, n: int | str) -> None: + (Path(directory) / "state").write_text(str(n), encoding="utf-8") + + +def feed_settings_over(base: Settings, tmp: Path) -> Settings: + """`base` (a pg_settings) with the fixture sources pointed at freshly generated files under tmp.""" + d = fixture_dir(tmp) + return base.model_copy(update={"feed_source": "fixture", "drawing_source": "fixture", "fixture_dir": d}) + + +def fixture_settings(tmp: Path, **over: Any) -> Settings: + """Plain test settings (no database) with fixture sources.""" + return make_settings(tmp, feed_source="fixture", drawing_source="fixture", fixture_dir=tmp / "fixtures", **over) + + +def set_feed(settings: Settings, **values: Any) -> None: + """Merge values into the `feed` settings row by SQL (no audit: test setup).""" + from pshelpers import app_conn + + from packetstation.feed import config + + with app_conn(settings) as conn: + cur = config.load(conn).model_dump(mode="json") + cur.update({k: (v.isoformat() if isinstance(v, date) else v) for k, v in values.items()}) + conn.execute( + "INSERT INTO settings (key, value, version, updated_at) VALUES ('feed', %s, 1, now())" + " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1", + (Jsonb(cur),), + ) + + +def golive_today_offset() -> date: + return date.today() + timedelta(days=gen.GOLIVE_OFFSET) + + +# ---------------------------------------------------------------- one job run + + +def scheduler(settings: Settings) -> Any: + """An unstarted worker Scheduler: what runs.run_job needs, without APScheduler or the listener.""" + from packetstation.worker.scheduler import Scheduler + + return Scheduler(settings, build="pytest") + + +def run_job(settings: Settings, job: str, *, requested: bool = False, allow_fail: bool = False) -> dict[str, Any]: + """Run one registered job once through runs.run_job, as the worker does. Returns the job_runs row. + + `requested` inserts a job_requests row first, so the run is a "run now". A failed run raises AssertionError + with its errors unless allow_fail. + """ + from packetstation import db + from packetstation.worker import runs + + if requested: + with db.connect_as("worker", settings, autocommit=True) as conn: + conn.execute("INSERT INTO job_requests (job, note) VALUES (%s, 'pytest')", (job,)) + row = runs.run_job(scheduler(settings), job, "schedule") + if not allow_fail: + assert row["status"] != "failed", row.get("errors") + return dict(row) + + +def _counts(row: dict[str, Any]) -> dict[str, Any]: + counts = dict(row.get("counts") or {}) + counts.update( + {"_status": row["status"], "_errors": row.get("errors") or [], "_run_id": row["id"], "_trigger": row["trigger"]} + ) + return counts + + +def run_feed( + settings: Settings, state: int | None = None, *, requested: bool = False, allow_fail: bool = False +) -> dict[str, Any]: + """Point the fixtures at `state` (when given) and run the visual-feed job once. Returns the run's counts with + `_status`, `_errors`, `_run_id` and `_trigger` added.""" + if state is not None: + switch_state(settings.fixture_dir, state) + return _counts(run_job(settings, FEED_JOB, requested=requested, allow_fail=allow_fail)) + + +def run_drawings(settings: Settings, state: int | None = None, *, requested: bool = False) -> dict[str, Any]: + if state is not None: + switch_state(settings.fixture_dir, state) + return _counts(run_job(settings, DRAWING_JOB, requested=requested)) + + +# ---------------------------------------------------------------- reading back + + +def packet_by_key(conn: Any, key: str, *, live: bool = True) -> dict[str, Any] | None: + from psycopg.rows import dict_row + + with conn.cursor(row_factory=dict_row) as cur: + live_sql = "SELECT * FROM packets WHERE visual_key = %s AND removed_at IS NULL ORDER BY id DESC LIMIT 1" + any_sql = "SELECT * FROM packets WHERE visual_key = %s ORDER BY id DESC LIMIT 1" + row = cur.execute(live_sql if live else any_sql, (key,)).fetchone() + return dict(row) if row else None + + +def packet_id(conn: Any, key: str) -> int: + p = packet_by_key(conn, key) + assert p is not None, key + return int(p["id"]) + + +def legs_of(conn: Any, pid: int) -> list[tuple[Any, ...]]: + return [ + tuple(r) + for r in conn.execute( + "SELECT wo_base, lot, split, sub, seq, qty, target_hrs, visual_status, missing_runs, gone_at IS NOT NULL" + " FROM packet_work_orders WHERE packet_id = %s ORDER BY wo_base, lot, split, sub, seq", + (pid,), + ).fetchall() + ] + + +def cos_of(conn: Any, pid: int) -> list[dict[str, Any]]: + from psycopg.rows import dict_row + + with conn.cursor(row_factory=dict_row) as cur: + return [ + dict(r) + for r in cur.execute("SELECT * FROM change_orders WHERE packet_id = %s ORDER BY co_no", (pid,)).fetchall() + ] + + +def move_status(conn: Any, pid: int, status: str) -> None: + """Move a packet past draft by SQL (test setup: the transitions are tested elsewhere).""" + conn.execute( + "UPDATE packets SET status = %s::packet_status, status_at = now(), heat_map = coalesce(heat_map, true)," + " version = version + 1 WHERE id = %s", + (status, pid), + ) + + +def system_audit(conn: Any, action: str) -> list[tuple[Any, ...]]: + return [ + tuple(r) + for r in conn.execute( + "SELECT role_snapshot, user_id, subject_type, subject_id, packet_id FROM audit_events WHERE action = %s" + " ORDER BY id", + (action,), + ).fetchall() + ] + + +__all__ = [ + "CHANGE_KEYS", + "FEED_COUNT_KEYS", + "cos_of", + "feed_settings_over", + "fixture_dir", + "fixture_settings", + "gen", + "golive_today_offset", + "legs_of", + "move_status", + "packet_by_key", + "packet_id", + "run_drawings", + "run_feed", + "run_job", + "set_feed", + "switch_state", + "system_audit", +] diff --git a/tests/server/test_candidates.py b/tests/server/test_candidates.py new file mode 100644 index 0000000..c37e263 --- /dev/null +++ b/tests/server/test_candidates.py @@ -0,0 +1,343 @@ +"""Candidates from the generator's state 1: packet keys, routing, go-live, tracked keys, revs and the BOM rules.""" + +from __future__ import annotations + +from datetime import date, timedelta +from decimal import Decimal +from typing import Any + +import pytest +from feedhelpers import gen + +from packetstation.feed import candidates, orders +from packetstation.feed.materials import index +from packetstation.feed.types import Material, WoKey, parse_material, parse_orders, parse_row + +TODAY = date.today() +WINDOW = (TODAY - timedelta(days=14), TODAY + timedelta(days=30)) + + +def k(base: str, lot: str, split: str = "0", sub: str = "0") -> WoKey: + return WoKey(base, lot, split, sub) + + +def state(n: int = 1) -> dict[str, Any]: + return gen.feed_state(n, TODAY) + + +def material_legs(feed: dict[str, Any]) -> set[WoKey]: + return {k(m["wo"], m["lot"], m["split"], m["sub"]) for m in feed["materials"] if m["st"] in ("F", "R")} + + +def build( + n: int = 1, + *, + tracked: set[WoKey] | None = None, + golive: date | None = None, + departments: tuple[str, ...] = (), + mats: bool = True, +) -> candidates.Built: + feed = state(n) + rows = [parse_row(r) for r in feed["rows"]] + materials = [parse_material(m) for m in feed["materials"]] if mats else None + return candidates.build( + rows, + materials, + material_legs(feed) if mats else None, + resource_map=gen.RESOURCE_MAP, + window=WINDOW, + golive=golive, + departments=departments, + tracked_keys=tracked or set(), + today=TODAY, + ) + + +def test_feed_file_shape() -> None: + feed = state(1) + assert set(feed) >= {"meta", "rows", "materials", "orders_by_base", "orders_by_project"} + row = feed["rows"][0] + assert set(row) >= { + "wo", + "lot", + "split", + "sub", + "seq", + "res", + "dwg", + "rev", + "ps", + "desc", + "qty", + "tgt", + "rel", + "want", + "st", + "barcode", + "part", + } + words = {r["st"] for r in feed["rows"]} + assert words <= {"Firmed", "Released", "Unreleased", "Closed", "Cancelled"} + assert {m["st"] for m in feed["materials"]} <= {"F", "R", "U", "C", "X"} + for r in feed["rows"]: + assert isinstance(r["seq"], int) and isinstance(r["lot"], str) and isinstance(r["split"], str) + assert isinstance(r["qty"], int | float) and not isinstance(r["qty"], bool) + assert r["rel"] is None or date.fromisoformat(r["rel"]) + for m in feed["materials"]: + assert isinstance(m["piece"], int) and (m["part"] is None or isinstance(m["part"], str)) + assert any(r["lot"] == "7A" for r in feed["rows"]) + assert any(m["part"] is None for m in feed["materials"]) + + +def test_generator_is_deterministic(tmp_path: Any) -> None: + a, b = tmp_path / "a", tmp_path / "b" + gen.generate(a, TODAY) + gen.generate(b, TODAY) + for name in ("feed.1.json", "feed.2.json", "drawings.1.json", "drawings.2.json", "state"): + assert (a / name).read_bytes() == (b / name).read_bytes(), name + assert (a / "state").read_text() == "1" + assert (a / "L900-0101 r1.pdf").read_bytes().startswith(b"%PDF-") + assert (a / "L900-0101 r1.pdf").read_bytes() != (a / "L900-0101 r2.pdf").read_bytes() + + +def test_packet_keys() -> None: + built = build() + assert set(built.candidates) == set(gen.STATE1_KEYS) + c = built.candidates[gen.KEY_WELD_PART] + assert (c.project, c.unit, c.lot, c.dept, c.op, c.drawing) == ( + "L900", + "L900001", + "001", + "03", + "WELD-PART", + "L900-0101", + ) + assert c.base == "L900-01" and c.in_window and c.title == "Inlet spool 6 in" + assert built.candidates[gen.KEY_LAYOUT_A].dept == "01" + nd = built.candidates[gen.KEY_NO_DWG] + assert nd.drawing == "" and nd.key.endswith("|(no drawing)") + + +def test_two_operations_at_one_resource_are_two_legs() -> None: + c = build().candidates[gen.KEY_TWO_OPS] + assert [(lg.key, lg.seq) for lg in c.legs] == [(k("L900-02", "1"), 10), (k("L900-02", "1"), 20)] + assert [lg.tgt for lg in c.legs] == [Decimal(4), Decimal(2)] + + +def test_fold_and_ignore() -> None: + c = build().candidates[gen.KEY_WELD_PART] + folded = {(f.seq, f.op) for f in c.folded} + assert folded == {(30, "MFG"), (50, "MAT_PIPE")} + assert all(lg.op not in ("MFG", "FORKLIFT", "MAT_PIPE") for cand in build().candidates.values() for lg in cand.legs) + # the LAYOUT packet of the same work order and lot folds the same operations + assert {f.seq for f in build().candidates[gen.KEY_LAYOUT_A].folded} == {30, 50} + assert candidates.route("MAT_STEEL", {}) == "fold" and candidates.route("FORKLIFT", gen.RESOURCE_MAP) == "ignore" + assert candidates.route("NEWRES", gen.RESOURCE_MAP) is None + + +def test_partspecs_distinct_in_first_seen_order() -> None: + c = build().candidates[gen.KEY_WELD_PART] + assert c.partspecs == ["Cut to length per sheet 1", "Weld per WPS-11\nBack purge"] + + +def test_unmapped_counts() -> None: + assert build().unmapped == {"NEWRES": 2} + + +def test_lettered_lot_makes_no_candidate() -> None: + built = build() + assert built.skipped_lots == {"L900-14/7A": 1} + assert candidates.unit_number("L900", "7A") is None + assert candidates.unit_number("L900", "1") == "L900001" + assert candidates.unit_number("L900", "012") == "L900012" + + +def test_golive_ignores_old_releases() -> None: + golive = TODAY + timedelta(days=gen.GOLIVE_OFFSET) + built = build(golive=golive) + assert gen.KEY_GOLIVE not in built.candidates + assert [(r.wo, r.lot, r.seq) for r in built.ignored] == [("L900-13", "2", 10)] + # tracked keys come through whatever their release date + built = build(golive=golive, tracked={k("L900-13", "2")}) + assert gen.KEY_GOLIVE in built.candidates and built.ignored == [] + + +def test_departments_filter() -> None: + built = build(departments=("06",)) + assert built.candidates and {c.dept for c in built.candidates.values()} == {"06"} + + +def test_tracked_keys_outside_the_window_and_status() -> None: + assert gen.KEY_RELEASED not in build().candidates # Released: not in the window set + tracked = {k("L900-06", "2"), k("L900-01", "1", "1"), k("L900-11", "1"), k("L900-12", "1")} + built = build(tracked=tracked) + rel = built.candidates[gen.KEY_RELEASED] + assert not rel.in_window and rel.statuses == {k("L900-06", "2"): "R"} + wp = built.candidates[gen.KEY_WELD_PART] + assert wp.statuses == {k("L900-01", "1"): "F", k("L900-01", "1", "1"): "U"} + assert {(lg.key.split, lg.seq) for lg in wp.legs} == {("0", 20), ("1", 20)} + assert built.candidates["L900001|WELD-CODE|L900-1101"].statuses == {k("L900-11", "1"): "C"} + assert built.candidates["L900001|WELD-CODE|L900-1201"].statuses == {k("L900-12", "1"): "X"} + + +def test_rel_min_want_max() -> None: + c = build().candidates[gen.KEY_REV_A] # legs L900-04 (rel -4, want 22) and L900-08-1 (rel -2, want 24) + assert c.rel == TODAY - timedelta(days=4) and c.want == TODAY + timedelta(days=24) + + +def test_rev_highest_numeric() -> None: + c = build().candidates[gen.KEY_REV_A] + assert c.rev == "A" # the non-numeric rev holds against an empty rev + built = build(2, tracked={k("L900-01", "1", "1")}) + assert built.candidates[gen.KEY_WELD_PART].rev == "2" + + +def _row(seq: int, rev: str) -> Any: + return parse_row( + { + "wo": "L900-20", + "lot": "1", + "split": "0", + "sub": "0", + "seq": seq, + "res": "WELD-PART", + "dwg": "D-1", + "rev": rev, + "qty": 1, + "tgt": 1, + "rel": TODAY.isoformat(), + "st": "Firmed", + } + ) + + +@pytest.mark.parametrize( + "revs,expect", + [(["1", "3", "2"], "3"), (["", "2"], "2"), (["2", "A"], "2"), (["A", "2"], "A"), (["10", "9"], "10")], +) +def test_rev_rule(revs: list[str], expect: str) -> None: + rows = [_row(10 * (i + 1), r) for i, r in enumerate(revs)] + built = candidates.build( + rows, + None, + None, + resource_map=gen.RESOURCE_MAP, + window=WINDOW, + golive=None, + departments=(), + tracked_keys=set(), + today=TODAY, + ) + assert built.candidates["L900001|WELD-PART|D-1"].rev == expect + + +def _bom(key: str, n: int = 1, tracked: set[WoKey] | None = None) -> dict[str, Any]: + c = build(n, tracked=tracked).candidates[key] + assert c.bom is not None + return {b.item: b for b in c.bom} + + +def test_bom_split_items() -> None: + bom = _bom(gen.KEY_LAYOUT_A) + assert set(bom) == {"11", "12"} # the piece 10 header has no part: not a line + assert bom["11"].grp == "10" and bom["12"].grp == "10" + assert bom["12"].cons is True and bom["11"].cons is False + assert bom["11"].vkey == "L900-01|1|0|10|11" + + +def test_bom_main_loc_wip_sub_and_dwg_filter() -> None: + bom = _bom(gen.KEY_WELD_PART) + assert set(bom) == {"1", "2", "4"} # piece 3 is on another drawing: dropped; piece 4 has a blank drawing + assert bom["1"].loc == "A-01" and bom["1"].whs == "MAIN" # MAIN is left out of loc + assert bom["4"].loc == "SHOP R1" + assert bom["2"].sub is True and bom["2"].na is True + assert bom["1"].sub is False and bom["1"].issued == Decimal("12.5") and bom["1"].qty == Decimal("12.5") + assert all(b.grp is None for b in bom.values()) + + +def test_bom_duplicate_items() -> None: + bom = _bom(gen.KEY_REV_A) + assert set(bom) == {"1", "1-1", "2"} + assert bom["1"].vkey.startswith("L900-04|") and bom["1-1"].vkey.startswith("L900-08-1|") + assert bom["2"].istat == 0 + + +def test_bom_two_operations() -> None: + bom = _bom(gen.KEY_TWO_OPS) + assert set(bom) == {"1", "2"} and bom["2"].istat == 3 + + +def test_released_leg_counts_as_known() -> None: + bom = _bom(gen.KEY_RELEASED, tracked={k("L900-06", "2")}) + assert set(bom) == {"1"} + + +def test_unreleased_leg_alone_is_unknown() -> None: + # A2 (Unreleased) is not a material leg; on the WELD-PART packet the firmed leg still makes it known + built = build(tracked={k("L900-01", "1", "1")}) + assert built.candidates[gen.KEY_WELD_PART].bom is not None + + +def test_mat_known_none() -> None: + built = build() + assert built.candidates[gen.KEY_NO_DWG].bom is None # C has no material lines: not a material leg + assert all(c.bom is None for c in build(mats=False).candidates.values()) + + +def test_best_status_per_piece() -> None: + key = k("L900-30", "1") + + def m(st: str, qty: int) -> Material: + return parse_material( + { + "wo": "L900-30", + "lot": "1", + "split": "0", + "sub": "0", + "seq": 10, + "piece": 1, + "part": "P-1", + "qty": qty, + "issued": 0, + "st": st, + "dwg": "D-9", + } + ) + + leg = candidates.Leg( + key=key, seq=10, op="WELD-PART", qty=Decimal(1), tgt=Decimal(1), dwg="D-9", rev="1", partspecs="", st="F" + ) + from packetstation.feed.materials import bom_for + + for lines, qty in ( + ([m("U", 3), m("F", 2), m("R", 1)], 1), + ([m("U", 3), m("F", 2)], 2), + ([m("C", 4), m("U", 3)], 3), + ): + bom = bom_for([leg], index(lines)) + assert [(b.item, b.qty) for b in bom] == [("1", Decimal(qty))] + + +def test_orders_unit_rule_on_fixture_orders() -> None: + feed = state(1) + by_base, by_project = parse_orders(feed["orders_by_base"]), parse_orders(feed["orders_by_project"]) + assert by_base is not None and by_project is not None + + def pick(unit: str, base: str, project: str = "L900", pin: str | None = None) -> tuple[str, str] | None: + ch = orders.choose( + unit_number=unit, base=base, by_base=by_base, by_project=by_project, project=project, pin=pin + ) + return None if ch is None else (ch.order.cust_order_id, ch.via) + + assert pick("L900001", "L900-01") == (gen.ORDER_BAYOU, "unit_top_job") + assert pick("L900002", "L900-04") == (gen.ORDER_ACME, "base") + assert pick("L900002", "L900-07") == (gen.ORDER_ACME, "unit_top_job") + assert pick("L900003", "L900-99") == (gen.ORDER_ACME, "project_open") + assert pick("L901001", "L901-01", "L901") == (gen.ORDER_L901, "unit_top_job") + assert pick("L901002", "L901-02", "L901") == (gen.ORDER_L901, "project_newest") + assert pick("L900001", "L900-01", pin=gen.ORDER_ACME) == (gen.ORDER_ACME, "pin") + assert pick("L900001", "L900-01", pin="C99-0000") == (gen.ORDER_BAYOU, "unit_top_job") # not the project's + assert pick("L902001", "L902-01", "L902") is None + d = orders.derive(by_project["L900"][1]) + assert d["customer"] == "Bayou Chemical" and d["po"] == "PO-2002" + assert d["ship_date"] == TODAY + timedelta(days=70) # no promise date: the desired ship date diff --git a/tests/server/test_drawing_decide.py b/tests/server/test_drawing_decide.py new file mode 100644 index 0000000..cd79390 --- /dev/null +++ b/tests/server/test_drawing_decide.py @@ -0,0 +1,351 @@ +"""The drawing decision table (pure), then the drawing job against the fixture lookup (PostgreSQL).""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +from feedhelpers import ( + cos_of, + feed_settings_over, + gen, + move_status, + packet_by_key, + packet_id, + run_drawings, + run_feed, + system_audit, +) +from pshelpers import app_conn, needs_pg + +from packetstation.drawings.decide import Decision, Target, decide +from packetstation.drawings.fake import FixtureLookup +from packetstation.drawings.lookup import Found +from packetstation.settings import Settings + +DRAWING_FLAGS = {"drawing_needed", "rev_mismatch", "not_released", "not_found", "lookup_failed", "bad_rev"} + + +def target(**over: Any) -> Target: + values: dict[str, Any] = { + "dwg": "L900-0101", + "visual_rev": "1", + "attached_rev": None, + "attached_sha": None, + "status": "draft", + "drawing_state": "needed", + "has_hand_drawing": False, + "rev_ok": None, + } + values.update(over) + return Target(**values) + + +def found(**over: Any) -> Found: + values: dict[str, Any] = { + "dwg": "L900-0101", + "ok": True, + "rev": 1, + "name": "L900-0101 r1", + "obj_id": 1, + "obj_version": 1, + "file_id": 101, + "file_version": 1, + "file_size": 900, + "base": None, + "draft_rev": None, + "draft_where": "", + "in_drafting": False, + "note": "", + } + values.update(over) + return Found(**values) + + +def d(t: Target, f: Found, same_file: bool = False) -> Decision: + return decide(t, f, same_file=same_file) + + +# ---------------------------------------------------------------- the table + + +def test_lookup_failed_on_a_needed_drawing() -> None: + r = d(target(), found(ok=False, rev=None, note="M-Files lookup failed: timeout")) + assert (r.kind, r.flag, r.drawing_state) == ("lookup_failed", "lookup_failed", "lookup_failed") + assert "timeout" in r.note + + +def test_lookup_failed_keeps_what_is_attached() -> None: + r = d(target(drawing_state="attached", attached_rev="1"), found(ok=False, rev=None, note="503")) + assert (r.kind, r.flag, r.drawing_state) == ("lookup_failed", "lookup_failed", "attached") + + +def test_not_released() -> None: + r = d(target(), found(rev=None, draft_rev=1, draft_where="in Drafting review", in_drafting=True, base="L900-05")) + assert (r.kind, r.flag, r.drawing_state) == ("not_released", "not_released", "not_released") + assert r.note == "Not released in M-Files yet (sheet of L900-05) - drafting status: rev 1 in Drafting review" + + +def test_not_released_in_drafting_without_a_rev() -> None: + r = d(target(), found(rev=None, in_drafting=True)) + assert r.kind == "not_released" and r.note.endswith("in drafting, no released PDF yet") + + +def test_not_found() -> None: + r = d(target(), found(rev=None, name=None)) + assert (r.kind, r.flag, r.drawing_state) == ("not_found", "not_found", "not_found") + assert r.note == "Drawing not in M-Files - check the drawing number in VISUAL" + + +def test_bad_rev() -> None: + r = d(target(visual_rev="A"), found(rev=1)) + assert (r.kind, r.flag) == ("bad_rev", "bad_rev") + assert "A" in r.note and "M-Files has rev 1" in r.note and "not attached" in r.note + + +def test_rev_mismatch() -> None: + r = d(target(visual_rev="2"), found(rev=3)) + assert (r.kind, r.flag, r.drawing_state) == ("rev_mismatch", "rev_mismatch", "rev_mismatch") + assert r.note == "Rev mismatch: VISUAL calls rev 2, M-Files has rev 3" + + +def test_rev_mismatch_with_a_newer_draft() -> None: + r = d(target(visual_rev="2"), found(rev=3, draft_rev=4, draft_where="in Drafting")) + assert r.note == "Rev mismatch: VISUAL calls rev 2, M-Files has rev 3; drafting status: rev 4 in Drafting" + + +def test_rev_mismatch_on_an_in_shop_packet_opens_nothing() -> None: + r = d(target(visual_rev="2", status="ready", drawing_state="attached", attached_rev="2"), found(rev=3)) + assert r.kind == "rev_mismatch" and r.drawing_state == "attached" + + +def test_approved_rev_attaches() -> None: + r = d(target(visual_rev="2", rev_ok="3"), found(rev=3)) + assert (r.kind, r.flag, r.drawing_state) == ("attach", None, "attached") + + +def test_approved_other_rev_still_mismatches() -> None: + assert d(target(visual_rev="2", rev_ok="4"), found(rev=3)).kind == "rev_mismatch" + + +def test_match_on_a_draft_attaches() -> None: + r = d(target(), found()) + assert (r.kind, r.flag, r.drawing_state) == ("attach", None, "attached") + + +def test_attach_names_the_sheet() -> None: + r = d(target(dwg="L900-0101H2"), found(dwg="L900-0101H2", base="L900-0101")) + assert r.kind == "attach" and r.note == "sheet of drawing L900-0101" + + +def test_hand_drawing_at_the_same_rev_stays() -> None: + r = d(target(has_hand_drawing=True, drawing_state="hand"), found()) + assert r.kind == "unchanged" and r.drawing_state == "hand" + + +def test_past_draft_same_file_unchanged() -> None: + r = d(target(status="ready", drawing_state="attached", attached_rev="1", attached_sha=b"\x01"), found(), True) + assert r.kind == "unchanged" and r.flag is None + + +def test_past_draft_new_file_same_rev_opens_a_co() -> None: + r = d(target(status="in_progress", drawing_state="attached", attached_rev="1", attached_sha=b"\x01"), found()) + assert r.kind == "open_co" + assert r.note == "M-Files re-released rev 1 of L900-0101 (new file)" + + +def test_past_draft_higher_rev_opens_a_co() -> None: + t = target(status="ready", visual_rev="2", drawing_state="attached", attached_rev="1", attached_sha=b"\x01") + r = d(t, found(rev=2)) + assert r.kind == "open_co" + assert r.note == "M-Files released rev 2 of L900-0101; this packet has rev 1" + + +def test_past_draft_waiting_co_gets_staged() -> None: + t = target( + status="ready", + visual_rev="2", + drawing_state="attached", + attached_rev="1", + attached_sha=b"\x01", + co_waiting=("L900-0101", "2"), + co_pending=True, + ) + assert d(t, found(rev=2)).kind == "stage_for_co" + + +def test_complete_packet_still_gets_a_co() -> None: + t = target(status="complete", drawing_state="attached", attached_rev="1", attached_sha=b"\x01") + assert d(t, found()).kind == "open_co" + + +# ---------------------------------------------------------------- the fixture lookup + + +def test_fixture_lookup_states(tmp_path: Path) -> None: + gen.generate(tmp_path) + one = FixtureLookup(tmp_path) + assert one.status("L900-0101").rev == 1 + assert one.status("L900-0201").rev == 3 + nr = one.status("L900-0501") + assert nr.rev is None and nr.in_drafting and nr.draft_rev == 1 + assert one.status("L900-0701").rev is None and one.status("L900-0701").ok + dest = tmp_path / "out" / "x.pdf" + assert one.download(one.status("L900-0101"), dest) > 0 and dest.read_bytes().startswith(b"%PDF-") + (tmp_path / "state").write_text("2") + two = FixtureLookup(tmp_path) + f2 = two.status("L900-0101") + assert f2.rev == 2 and f2.identity != one.status("L900-0101").identity + + +# ---------------------------------------------------------------- the job (PostgreSQL) + + +@pytest.fixture +def fx(pg_settings: Settings, tmp_path: Path) -> Settings: + return feed_settings_over(pg_settings, tmp_path) + + +def _drawing_pages(conn: Any, pid: int) -> list[tuple[Any, ...]]: + return [ + tuple(r) + for r in conn.execute( + "SELECT n, pn, dwg, rev, asset_id IS NOT NULL, void_at IS NOT NULL FROM packet_pages" + " WHERE packet_id = %s AND type = 'drawing' ORDER BY n", + (pid,), + ).fetchall() + ] + + +@needs_pg +def test_job_attaches_and_flags(fx: Settings) -> None: + run_feed(fx, 1) + counts = run_drawings(fx) + assert counts["_status"] in ("ok", "degraded"), counts + assert counts["checked"] >= 8 and counts["attached"] >= 1 + with app_conn(fx) as conn: + wp = packet_by_key(conn, gen.KEY_WELD_PART) + assert wp is not None + assert wp["drawing_state"] == "attached" + assert not set(wp["flags"]) & DRAWING_FLAGS + assert wp["visual"].get("rev_confirmed") is True + pages = _drawing_pages(conn, wp["id"]) + assert pages and all(p[2] == "L900-0101" and p[3] == "1" and p[4] for p in pages) + lk = conn.execute( + "SELECT ok, rev, asset_id IS NOT NULL, sha256 IS NOT NULL, checked_at IS NOT NULL FROM drawing_lookups" + " WHERE dwg = 'L900-0101'" + ).fetchone() + assert lk == (True, 1, True, True, True) + # the LAYOUT packet shares the drawing: attached from the same asset + lay = packet_by_key(conn, gen.KEY_LAYOUT_A) + assert lay is not None and lay["drawing_state"] == "attached" + + mm = packet_by_key(conn, gen.KEY_TWO_OPS) # VISUAL rev 2, M-Files rev 3 + assert mm is not None and "rev_mismatch" in mm["flags"] and mm["drawing_state"] == "rev_mismatch" + assert mm["visual"]["appr"]["rev"] == 3 + staged = conn.execute( + "SELECT asset_id IS NOT NULL, rev FROM drawing_lookups WHERE dwg = 'L900-0201'" + ).fetchone() + assert staged == (True, 3) # staged for Clerical Supervisor approval + assert _drawing_pages(conn, mm["id"]) == [] + + nr = packet_by_key(conn, gen.KEY_DRAFTING) + assert nr is not None and "not_released" in nr["flags"] and nr["drawing_state"] == "not_released" + nf = packet_by_key(conn, gen.KEY_NOT_FOUND) + assert nf is not None and "not_found" in nf["flags"] and nf["drawing_state"] == "not_found" + br = packet_by_key(conn, gen.KEY_REV_A) + assert br is not None and "bad_rev" in br["flags"] + nd = packet_by_key(conn, gen.KEY_NO_DWG) + assert nd is not None and nd["drawing_state"] == "none" + log = conn.execute( + "SELECT by, by_name, role FROM packet_log WHERE packet_id = %s" + " AND action = 'Drawing attached from M-Files'", + (wp["id"],), + ).fetchall() + assert len(log) == 1 and log[0][0] is None and log[0][2] == "system" + assert all(r[0] == "system" and r[1] is None for r in system_audit(conn, "packet.vis_update_draft")) + assert all(r[0] == "system" for r in system_audit(conn, "drawing.fetched")) + assert system_audit(conn, "drawing.fetched") + + +@needs_pg +def test_job_second_run_changes_nothing(fx: Settings) -> None: + run_feed(fx, 1) + run_drawings(fx) + with app_conn(fx) as conn: + conn.execute( + "UPDATE drawing_lookups SET next_check_at = now() - interval '1 minute'," + " checked_at = now() - interval '1 day'" + ) + before = conn.execute("SELECT count(*) FROM packet_pages").fetchone()[0] + counts = run_drawings(fx) + assert counts.get("attached", 0) == 0 and counts.get("fetched", 0) == 0, counts + with app_conn(fx) as conn: + assert conn.execute("SELECT count(*) FROM packet_pages").fetchone()[0] == before + + +@needs_pg +def test_stage_for_co_after_state_2(fx: Settings) -> None: + run_feed(fx, 1) + run_drawings(fx) + with app_conn(fx) as conn: + pid = packet_id(conn, gen.KEY_WELD_PART) + move_status(conn, pid, "ready") + counts = run_feed(fx, 2) + assert counts["cos_opened"] >= 1, counts + with app_conn(fx) as conn: + co = [c for c in cos_of(conn, pid) if c["review"] == "pending"] + assert len(co) == 1 and co[0]["origin"] == "visual" + assert co[0]["waiting"] is not None and co[0]["waiting"]["rev"] == "2" + assert co[0]["staged"] is None + counts = run_drawings(fx) + assert counts.get("staged", 0) >= 1, counts + with app_conn(fx) as conn: + co = [c for c in cos_of(conn, pid) if c["review"] == "pending"] + assert len(co) == 1 + assert co[0]["waiting"] is None + staged = co[0]["staged"] + assert staged["rev"] == "2" and staged["dwg"] == "L900-0101" and staged["asset_id"] + # the in-shop packet keeps its rev 1 pages until the CO is approved + assert all(p[3] == "1" for p in _drawing_pages(conn, pid)) + assert ( + conn.execute( + "SELECT count(*) FROM packet_log WHERE packet_id = %s AND action LIKE 'Drawing for change order%%'", + (pid,), + ).fetchone()[0] + == 1 + ) + + +@needs_pg +def test_new_file_on_an_in_shop_packet_opens_a_drawing_co(fx: Settings) -> None: + run_feed(fx, 1) + run_drawings(fx) + with app_conn(fx) as conn: + pid = packet_id(conn, gen.KEY_CLOSES) # L900-0901 rev 1, attached + assert _drawing_pages(conn, pid) + move_status(conn, pid, "in_progress") + conn.execute( + "UPDATE drawing_lookups SET checked_at = now() - interval '2 days', next_check_at = now()" + " WHERE dwg = 'L900-0901'" + ) + # M-Files re-releases rev 1 of L900-0901 as a new file (same rev, new version and bytes) + d1 = fx.fixture_dir / "drawings.1.json" + import json + + data = json.loads(d1.read_text()) + data["L900-0901"].update({"file": "L900-0901 r1b.pdf", "obj_version": 2, "file_version": 2}) + d1.write_text(json.dumps(data)) + (fx.fixture_dir / "L900-0901 r1b.pdf").write_bytes(gen.pdf_bytes("L900-0901 r1b")) + counts = run_drawings(fx) + assert counts.get("cos_opened", 0) == 1, counts + with app_conn(fx) as conn: + cos = cos_of(conn, pid) + assert len(cos) == 1 + co = cos[0] + assert (co["origin"], co["review"], co["ecn"], co["issued_by"]) == ("drawing", "pending", "M-FILES", None) + assert co["reason"] == "M-Files re-released rev 1 of L900-0901 (new file)" + assert co["staged"]["asset_id"] and co["vis"]["dwg_change"] is True + assert all(r[0] == "system" for r in system_audit(conn, "packet.vis_open_co")) + assert system_audit(conn, "packet.vis_open_co") diff --git a/tests/server/test_feed_api.py b/tests/server/test_feed_api.py new file mode 100644 index 0000000..8027969 --- /dev/null +++ b/tests/server/test_feed_api.py @@ -0,0 +1,438 @@ +"""The Clerical feed API (/api/feed), rev approval, hand uploads and feed change order approval, on the generator's +fixture files run through the real feed and drawing jobs.""" + +from __future__ import annotations + +import io +import json +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest +from fakedir import FAKE_PASSWORD, FakeDirectory +from fastapi.testclient import TestClient +from feedhelpers import ( + cos_of, + feed_settings_over, + gen, + golive_today_offset, + move_status, + packet_id, + run_drawings, + run_feed, + set_feed, +) +from pkthelpers import SIGN, seed_users, tiny_pdf, upload +from pshelpers import ORIGIN_HEADERS, app_conn, needs_pg, sign_in + +from packetstation import db +from packetstation.settings import Settings + +pytestmark = needs_pg + + +@pytest.fixture +def fx(pg_settings: Settings, tmp_path: Path) -> Settings: + s = feed_settings_over(pg_settings, tmp_path) + set_feed(s, auto_release=False) + return s + + +@pytest.fixture +def client(fx: Settings, fake_dir: FakeDirectory, dev_visual: None) -> Iterator[TestClient]: + from packetstation.main import create_app + + with TestClient(create_app(fx, directory=fake_dir), headers=ORIGIN_HEADERS) as c: + seed_users(c, ("admin.ann", "clerk.clara", "clerk.leo", "wh.wade")) + yield c + + +def as_user(c: TestClient, sam: str) -> None: + assert sign_in(c, sam).status_code == 200, sam + + +def drawing_pages(conn: Any, pid: int) -> list[tuple[Any, ...]]: + return [ + tuple(r) + for r in conn.execute( + "SELECT n, rev, co_id IS NOT NULL, void_at IS NOT NULL, void_co_id IS NOT NULL FROM packet_pages" + " WHERE packet_id = %s AND type = 'drawing' ORDER BY n", + (pid,), + ).fetchall() + ] + + +def packet_row(conn: Any, pid: int) -> dict[str, Any]: + from psycopg.rows import dict_row + + with conn.cursor(row_factory=dict_row) as cur: + r = cur.execute("SELECT * FROM packets WHERE id = %s", (pid,)).fetchone() + assert r is not None + return dict(r) + + +# ---------------------------------------------------------------- the From VISUAL panel + + +def test_overview_shape(client: TestClient, fx: Settings) -> None: + set_feed(fx, golive_date=golive_today_offset()) + run_feed(fx, 1) + run_drawings(fx) + as_user(client, "clerk.clara") + r = client.get("/api/feed/overview") + assert r.status_code == 200, r.text + o = r.json() + assert set(o) >= { + "status", + "last_run", + "runs", + "drawing_runs", + "ignored", + "unmapped", + "projects_without_names", + "golive_date", + "departments", + } + assert o["status"]["paused"] is False + assert o["status"]["worker"] is None or o["status"]["worker"]["stale"] is True + assert o["last_run"]["status"] == "ok" and o["last_run"]["counts"]["new_drafts"] == len(gen.STATE1_KEYS) - 1 + assert len(o["runs"]) == 1 and len(o["drawing_runs"]) == 1 + assert o["unmapped"] == [{"resource": "NEWRES", "rows": 2}] + assert [(i["wo_base"], i["lot"]) for i in o["ignored"]] == [("L900-13", "2")] + assert {p["number"] for p in o["projects_without_names"]} == {"L900", "L901"} + l900 = next(p for p in o["projects_without_names"] if p["number"] == "L900") + assert l900["units"] == 2 and l900["packets"] == len(gen.STATE1_KEYS) - 2 + assert o["golive_date"] == golive_today_offset().isoformat() and o["departments"] == [] + + +def test_feed_api_is_clerical_only(client: TestClient) -> None: + as_user(client, "wh.wade") + assert client.get("/api/feed/overview").status_code == 403 + assert client.post("/api/feed/run", json={}).status_code == 403 + assert client.get("/api/feed/batches").status_code == 403 + + +def test_run_now_inserts_a_request_and_notifies(client: TestClient, fx: Settings) -> None: + listener = db.connect_as("worker", fx, autocommit=True) + listener.execute("LISTEN ps_job_request") + try: + as_user(client, "clerk.clara") + r = client.post("/api/feed/run", json={}) + assert r.status_code == 200 and r.json()["requested"] is True and r.json()["pending"] == 1, r.text + got = list(listener.notifies(timeout=3, stop_after=1)) + assert got and json.loads(got[0].payload) == {"job": "visual-feed"} + r = client.post("/api/feed/run", json={"job": "drawing-lookup"}) + assert r.status_code == 200 and r.json()["job"] == "drawing-lookup" + assert client.post("/api/feed/run", json={"job": "housekeeping"}).status_code == 422 + finally: + listener.close() + with app_conn(fx) as conn: + rows = conn.execute("SELECT job FROM job_requests WHERE claimed_at IS NULL ORDER BY id").fetchall() + assert [r[0] for r in rows] == ["visual-feed", "drawing-lookup"] + assert conn.execute("SELECT count(*) FROM audit_events WHERE action = 'feed.run_requested'").fetchone()[0] == 2 + # the worker's next run claims the request and records the trigger + c = run_feed(fx, 1) + assert c["_trigger"] == "request" + assert client.get("/api/feed/overview").json()["last_run"]["trigger"] == "request" + + +def test_pull_in(client: TestClient, fx: Settings) -> None: + set_feed(fx, golive_date=golive_today_offset()) + run_feed(fx, 1) + as_user(client, "clerk.clara") + ig = client.get("/api/feed/overview").json()["ignored"][0] + r = client.post(f"/api/feed/ignored/{ig['id']}/pull-in", json={"reason": ""}) + assert r.status_code == 400 and r.json()["code"] == "reason_required" + r = client.post(f"/api/feed/ignored/{ig['id']}/pull-in", json={"reason": "Customer still wants it"}) + assert r.status_code == 200 and r.json()["pulled_in_at"], r.text + r = client.post(f"/api/feed/ignored/{ig['id']}/pull-in", json={"reason": "again please"}) + assert r.status_code == 409 + assert client.post("/api/feed/ignored/999999/pull-in", json={"reason": "nothing"}).status_code == 404 + with app_conn(fx) as conn: + assert conn.execute("SELECT count(*) FROM job_requests WHERE claimed_at IS NULL").fetchone()[0] == 1 + assert conn.execute("SELECT count(*) FROM audit_events WHERE action = 'feed.pulled_in'").fetchone()[0] == 1 + c = run_feed(fx, 1) + assert c["new_drafts"] == 1 and c["_trigger"] == "request" + assert client.get("/api/feed/overview").json()["ignored"] == [] + + +def test_batches_and_changes(client: TestClient, fx: Settings) -> None: + run_feed(fx, 1) + run_feed(fx, 2) + as_user(client, "clerk.clara") + batches = client.get("/api/feed/batches").json() + assert len(batches) == 2 and batches[0]["id"] > batches[1]["id"] + ch = client.get("/api/feed/changes").json() + assert ch["batch"]["id"] == batches[0]["id"] + kinds = {(r["wo_base"], r["kind"]) for r in ch["rows"]} + assert ("L900-01", "changed") in kinds and ("L900-05", "gone") in kinds + first = client.get(f"/api/feed/changes?batch_id={batches[1]['id']}").json() + assert all(r["kind"] == "added" for r in first["rows"]) and first["rows"] + assert client.get("/api/feed/changes?batch_id=999999").status_code == 404 + + +# ---------------------------------------------------------------- rev approval + + +def test_rev_approve(client: TestClient, fx: Settings) -> None: + run_feed(fx, 1) + run_drawings(fx) + with app_conn(fx) as conn: + pid = packet_id(conn, gen.KEY_TWO_OPS) # VISUAL rev 2, M-Files rev 3: staged + nf = packet_id(conn, gen.KEY_NOT_FOUND) + body = {"reason": "Engineering confirmed rev 3 is right", "sign": SIGN} + as_user(client, "clerk.clara") + r = client.post(f"/api/packets/{pid}/rev-approve", json=body) + assert r.status_code == 403, r.text + as_user(client, "clerk.leo") + r = client.post(f"/api/packets/{pid}/rev-approve", json={"reason": "no signature"}) + assert r.status_code == 400 and r.json()["code"] == "signature_required", r.text + r = client.post(f"/api/packets/{nf}/rev-approve", json=body) + assert r.status_code == 409 and r.json()["code"] == "nothing_staged", r.text + r = client.post(f"/api/packets/{pid}/rev-approve", json=body) + assert r.status_code == 200, r.text + with app_conn(fx) as conn: + p = packet_row(conn, pid) + assert p["drawing_state"] == "attached" and "rev_mismatch" not in p["flags"] + assert p["visual"]["rev_ok"]["rev"] == "3" and p["visual"]["rev_ok"]["vis_rev"] == "2" + assert p["visual"]["rev_confirmed"] is True + pages = drawing_pages(conn, pid) + assert pages and all(pg[1] == "3" for pg in pages) + assert ( + conn.execute( + "SELECT count(*) FROM packet_log WHERE packet_id = %s AND action = 'Different rev approved by hand'", + (pid,), + ).fetchone()[0] + == 1 + ) + assert ( + conn.execute( + "SELECT count(*) FROM audit_events WHERE action = 'packet.rev_approve' AND packet_id = %s", (pid,) + ).fetchone()[0] + == 1 + ) + # the drawing job leaves the approved rev alone + with app_conn(fx) as conn: + conn.execute("UPDATE drawing_lookups SET checked_at = now() - interval '2 days', next_check_at = now()") + run_drawings(fx) + with app_conn(fx) as conn: + p = packet_row(conn, pid) + assert p["drawing_state"] == "attached" and "rev_mismatch" not in p["flags"] + + +# ---------------------------------------------------------------- hand uploads + + +def test_hand_drawing(client: TestClient, fx: Settings) -> None: + run_feed(fx, 1) + run_drawings(fx) + with app_conn(fx) as conn: + nf = packet_id(conn, gen.KEY_NOT_FOUND) + wp = packet_id(conn, gen.KEY_WELD_PART) + as_user(client, "clerk.clara") + a = upload(client, tiny_pdf(2), "L900-0701 hand.pdf") + assert a.status_code == 201, a.text + asset = a.json()["id"] + r = client.post(f"/api/packets/{wp}/hand/drawing", json={"asset_id": asset, "reason": "Have it", "sign": SIGN}) + assert r.status_code == 409 and r.json()["code"] == "not_flagged", r.text + r = client.post(f"/api/packets/{nf}/hand/drawing", json={"asset_id": asset, "reason": "From drafting"}) + assert r.status_code == 400 and r.json()["code"] == "signature_required", r.text + r = client.post( + f"/api/packets/{nf}/hand/drawing", json={"asset_id": asset, "reason": "From drafting by email", "sign": SIGN} + ) + assert r.status_code == 200, r.text + with app_conn(fx) as conn: + p = packet_row(conn, nf) + assert p["drawing_state"] == "hand" and "hand_drawing" in p["flags"] and "not_found" not in p["flags"] + assert p["visual"]["manual_dwg"]["reason"] == "From drafting by email" + assert len(drawing_pages(conn, nf)) == 2 + assert ( + conn.execute( + "SELECT count(*) FROM packet_log WHERE packet_id = %s AND action = 'Drawing uploaded by hand'", (nf,) + ).fetchone()[0] + == 1 + ) + # a hand flag keeps it off auto-release, and the drawing job does not take it away + run_drawings(fx) + with app_conn(fx) as conn: + assert packet_row(conn, nf)["drawing_state"] == "hand" + + +def _xlsx(rows: list[list[Any]]) -> bytes: + from openpyxl import Workbook + + wb = Workbook() + ws = wb.active + assert ws is not None + for r in rows: + ws.append(r) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def _hand_bom(client: TestClient, pid: int, data: bytes, name: str, **form: str) -> Any: + fields = {"reason": "Engineering sent the BOM", "sign_factor": "password", "sign_secret": FAKE_PASSWORD} + fields.update(form) + return client.post( + f"/api/packets/{pid}/hand/bom", files={"file": (name, data, "application/octet-stream")}, data=fields + ) + + +@pytest.mark.parametrize("kind", ["xlsx", "csv"]) +def test_hand_bom(client: TestClient, fx: Settings, kind: str) -> None: + run_feed(fx, 1) + run_drawings(fx) + with app_conn(fx) as conn: + mm = packet_id(conn, gen.KEY_TWO_OPS) # rev mismatch: flagged + wp = packet_id(conn, gen.KEY_WELD_PART) # attached, no flags + visual_lines = conn.execute("SELECT count(*) FROM bom_lines WHERE packet_id = %s", (mm,)).fetchone()[0] + if kind == "xlsx": + data, name = ( + _xlsx( + [ + ["Item", "Qty", "Part ID", "Description"], + [1, 4, "PLT-0.5-A36", "Plate"], + [2, 2, "CPL-2-3000", "Coupling"], + [3, 1, None, "Gusset"], + ] + ), + "bom.xlsx", + ) + else: + data, name = ( + b"Item,Qty,Part ID,Description\r\n1,4,PLT-0.5-A36,Plate\r\n2,2,CPL-2-3000,Coupling\r\n3,1,,Gusset\r\n", + "bom.csv", + ) + as_user(client, "clerk.clara") + r = _hand_bom(client, wp, data, name) + assert r.status_code == 409 and r.json()["code"] == "not_flagged", r.text + r = _hand_bom(client, mm, data, name, sign_secret="wrong password") + assert r.status_code in (400, 401, 403), r.text + r = _hand_bom(client, mm, b"Name,Colour\nPipe,Red\n", "bad.csv") + assert r.status_code == 422 and r.json()["code"] == "bad_file", r.text + r = _hand_bom(client, mm, data, name) + assert r.status_code == 200, r.text + with app_conn(fx) as conn: + lines = conn.execute( + "SELECT item, qty, part_id, description, src FROM bom_lines WHERE packet_id = %s ORDER BY item", (mm,) + ).fetchall() + assert [(ln[0], ln[2], ln[4]) for ln in lines] == [ + (1, "PLT-0.5-A36", "hand"), + (2, "CPL-2-3000", "hand"), + (3, None, "hand"), + ] + p = packet_row(conn, mm) + assert "hand_bom" in p["flags"] and p["visual"]["manual_bom"]["lines"] == 3 + assert ( + conn.execute( + "SELECT count(*) FROM packet_log WHERE packet_id = %s AND action = 'BOM uploaded by hand'", (mm,) + ).fetchone()[0] + == 1 + ) + # the feed keeps a hand BOM on a draft + run_feed(fx, 2) + with app_conn(fx) as conn: + assert ( + conn.execute("SELECT count(*) FROM bom_lines WHERE packet_id = %s AND src = 'hand'", (mm,)).fetchone()[0] + == 3 + ) + # back to VISUAL's BOM + r = client.request("DELETE", f"/api/packets/{mm}/hand/bom", json={"sign": SIGN}) + assert r.status_code == 200, r.text + with app_conn(fx) as conn: + src = conn.execute("SELECT DISTINCT src FROM bom_lines WHERE packet_id = %s", (mm,)).fetchall() + assert src == [("visual",)] + assert conn.execute("SELECT count(*) FROM bom_lines WHERE packet_id = %s", (mm,)).fetchone()[0] == visual_lines + p = packet_row(conn, mm) + assert "hand_bom" not in p["flags"] and "manual_bom" not in p["visual"] + + +# ---------------------------------------------------------------- feed change orders + + +def test_co_approve_applies_vis_and_voids_pages(client: TestClient, fx: Settings) -> None: + run_feed(fx, 1) + run_drawings(fx) + with app_conn(fx) as conn: + wp = packet_id(conn, gen.KEY_WELD_PART) + assert packet_row(conn, wp)["drawing_state"] == "attached" + move_status(conn, wp, "ready") + old_pages = drawing_pages(conn, wp) + assert old_pages and all(pg[1] == "1" for pg in old_pages) + run_feed(fx, 2) + with app_conn(fx) as conn: + co = cos_of(conn, wp)[0] + assert co["waiting"] is not None and co["staged"] is None + as_user(client, "clerk.clara") + r = client.post(f"/api/packets/{wp}/change-orders/{co['co_no']}/approve", json={"sign": SIGN}) + assert r.status_code == 409 and r.json()["code"] == "co_not_staged", r.text + assert "not been fetched" in r.json()["detail"] + r = client.get(f"/api/packets/{wp}") + assert r.status_code == 200 + assert r.json()["status"] == "ready" + run_drawings(fx, 2) + with app_conn(fx) as conn: + co = cos_of(conn, wp)[0] + assert co["staged"] is not None and co["staged"]["rev"] == "2" and co["waiting"] is None + r = client.post(f"/api/packets/{wp}/change-orders/{co['co_no']}/approve", json={}) + assert r.status_code == 400 and r.json()["code"] == "signature_required", r.text + r = client.post(f"/api/packets/{wp}/change-orders/{co['co_no']}/approve", json={"sign": SIGN}) + assert r.status_code == 200, r.text + with app_conn(fx) as conn: + p = packet_row(conn, wp) + co = cos_of(conn, wp)[0] + assert co["review"] == "approved" and co["approved_by"] is not None + assert p["rev"] == "2" and p["status"] == "ready" + assert p["visual_fp_applied"] == co["target_fp"] + assert p["visual"]["fp_parts"] == co["vis"]["fp_parts"] + pages = drawing_pages(conn, wp) + void = [pg for pg in pages if pg[3]] + live = [pg for pg in pages if not pg[3]] + assert len(void) == len(old_pages) and all(pg[4] for pg in void) # VOID by this CO + assert live and all(pg[1] == "2" and pg[2] for pg in live) + items = conn.execute("SELECT old_rev, new_rev FROM co_items WHERE co_id = %s", (co["id"],)).fetchall() + assert ("1", "2") in [tuple(i) for i in items] + required = { + r[0] + for r in conn.execute("SELECT dept_code FROM co_required_depts WHERE co_id = %s", (co["id"],)).fetchall() + } + assert "03" in required + # the next feed run sees the packet matches VISUAL: no new change order + c = run_feed(fx, 2) + assert c["cos_opened"] == 0, c + with app_conn(fx) as conn: + assert len(cos_of(conn, wp)) == 1 + + +def test_minor_co_approve_needs_no_departments(client: TestClient, fx: Settings) -> None: + run_feed(fx, 1) + with app_conn(fx) as conn: + g = packet_id(conn, gen.KEY_NOT_FOUND) + move_status(conn, g, "warehouse") + run_feed(fx, 2) + with app_conn(fx) as conn: + co = cos_of(conn, g)[0] + assert co["minor"] is True + as_user(client, "clerk.clara") + r = client.post(f"/api/packets/{g}/change-orders/{co['co_no']}/approve", json={"sign": SIGN}) + assert r.status_code == 200, r.text + with app_conn(fx) as conn: + assert conn.execute("SELECT count(*) FROM co_required_depts WHERE co_id = %s", (co["id"],)).fetchone()[0] == 0 + assert conn.execute("SELECT target_hrs FROM packet_work_orders WHERE packet_id = %s", (g,)).fetchone()[0] == 5 + + +def test_changes_to_approve_filter(client: TestClient, fx: Settings) -> None: + run_feed(fx, 1) + with app_conn(fx) as conn: + wp = packet_id(conn, gen.KEY_WELD_PART) + move_status(conn, wp, "ready") + run_feed(fx, 2) + as_user(client, "clerk.clara") + r = client.get("/api/packets?co=pending") + assert r.status_code == 200, r.text + body = r.json() + items = body["packets"] + assert [i["id"] for i in items] == [wp] + assert items[0]["co_pending"] is True diff --git a/tests/server/test_feed_apply.py b/tests/server/test_feed_apply.py new file mode 100644 index 0000000..d81af31 --- /dev/null +++ b/tests/server/test_feed_apply.py @@ -0,0 +1,507 @@ +"""The VISUAL feed end to end on PostgreSQL: the generator's fixture files, the FixtureSource, the cache, the +candidates and the apply, through the visual-feed job as the worker runs it.""" + +from __future__ import annotations + +import json +from datetime import date, timedelta +from decimal import Decimal +from pathlib import Path +from typing import Any + +import pytest +from feedhelpers import ( + CHANGE_KEYS, + cos_of, + feed_settings_over, + gen, + legs_of, + move_status, + packet_by_key, + packet_id, + run_drawings, + run_feed, + set_feed, + switch_state, + system_audit, +) +from pshelpers import app_conn, needs_pg +from psycopg.rows import dict_row + +from packetstation.feed import fingerprint +from packetstation.settings import Settings + +pytestmark = needs_pg + +TODAY = date.today() + + +@pytest.fixture +def fx(pg_settings: Settings, tmp_path: Path) -> Settings: + s = feed_settings_over(pg_settings, tmp_path) + set_feed(s, auto_release=False) # most tests look at drafts; auto-release has its own test + return s + + +def row(conn: Any, sql: str, params: Any = None) -> dict[str, Any]: + with conn.cursor(row_factory=dict_row) as cur: + r = cur.execute(sql, params).fetchone() + assert r is not None, sql + return dict(r) + + +def no_human_writes(conn: Any) -> None: + """Every audit row the feed wrote has role_snapshot 'system' and no user.""" + bad = conn.execute( + "SELECT action, role_snapshot FROM audit_events WHERE user_id IS NULL" + " AND (action LIKE 'packet.%' OR action LIKE 'feed.%' OR action LIKE 'project.%' OR action LIKE 'unit.%'" + " OR action LIKE 'drawing.%') AND role_snapshot IS DISTINCT FROM 'system'" + ).fetchall() + assert bad == [] + logs = conn.execute("SELECT DISTINCT by_name, role FROM packet_log WHERE by IS NULL ORDER BY 1").fetchall() + assert all(r[1] == "system" for r in logs), logs + + +def window_firmed_wos(state: int = 1) -> int: + """Firmed work order legs released inside the default window (what visual_batches.window_rows counts).""" + since, until = TODAY - timedelta(days=14), TODAY + timedelta(days=30) + return len( + { + (r["wo"], r["lot"], r["split"], r["sub"]) + for r in gen.feed_state(state, TODAY)["rows"] + if r["st"] == "Firmed" and r["rel"] and since <= date.fromisoformat(r["rel"]) <= until + } + ) + + +def window_firmed_rows(state: int = 1) -> int: + since, until = TODAY - timedelta(days=14), TODAY + timedelta(days=30) + return sum( + 1 + for r in gen.feed_state(state, TODAY)["rows"] + if r["st"] == "Firmed" and r["rel"] and since <= date.fromisoformat(r["rel"]) <= until + ) + + +# ---------------------------------------------------------------- state 1 + + +def test_state_1_creates_projects_units_and_drafts(fx: Settings) -> None: + c = run_feed(fx, 1) + assert c["_status"] == "ok", c + assert c["new_drafts"] == len(gen.STATE1_KEYS) + assert c["projects_created"] == 2 and c["units_created"] == 3 + assert c["unmapped_resources"] >= 1 and c["unmapped"] == {"NEWRES": 2} + assert c["suspect"] == 0 and c["rows"] >= window_firmed_rows() + with app_conn(fx) as conn: + keys = {r[0] for r in conn.execute("SELECT visual_key FROM packets WHERE removed_at IS NULL").fetchall()} + assert keys == set(gen.STATE1_KEYS) + + p = row(conn, "SELECT * FROM projects WHERE number = 'L900'") + assert p["source"] == "visual" and p["needs_details"] is True and p["name"] is None + assert p["created_by"] is None + assert (p["customer"], p["customer_src"], p["customer_visual"]) == ("Acme Refining", "visual", "Acme Refining") + assert p["po"] == "PO-1001" and p["order_ref"]["cust_order_id"] == gen.ORDER_ACME + + u1 = row( + conn, + "SELECT u.* FROM units u JOIN projects p ON p.id = u.project_id WHERE p.number = 'L900' AND u.lot = '001'", + ) + assert u1["customer"] == "Bayou Chemical" and u1["po"] == "PO-2002" and u1["customer_src"] == "visual" + assert u1["order_ref"]["cust_order_id"] == gen.ORDER_BAYOU and u1["order_ref"]["via"] == "unit_top_job" + assert u1["ship_date"] == TODAY + timedelta(days=70) and u1["ship_date_visual"] == TODAY + timedelta(days=70) + u2 = row( + conn, + "SELECT u.* FROM units u JOIN projects p ON p.id = u.project_id WHERE p.number = 'L900' AND u.lot = '002'", + ) + assert u2["customer"] == "Acme Refining" and u2["order_ref"]["cust_order_id"] == gen.ORDER_ACME + z = row(conn, "SELECT u.* FROM units u JOIN projects p ON p.id = u.project_id WHERE p.number = 'L901'") + assert z["customer"] == "Delta Gas" and z["order_ref"]["status"] == "C" + assert row(conn, "SELECT order_ref FROM projects WHERE number = 'L901'")["order_ref"]["via"] in ( + "project_newest", + "unit_top_job", + "base", + ) + aliases = {r[0] for r in conn.execute("SELECT alias FROM project_aliases WHERE kind = 'import'").fetchall()} + assert {"L900", "L900001", "L900002", "L901", "L901001"} <= aliases + assert conn.execute("SELECT count(*) FROM units WHERE lot = '7A'").fetchone()[0] == 0 + + wp = packet_by_key(conn, gen.KEY_WELD_PART) + assert wp is not None + assert (wp["status"], wp["source_kind"], wp["dept_code"], wp["op"]) == ("draft", "visual", "03", "WELD-PART") + assert (wp["drawing"], wp["rev"], wp["drawing_state"]) == ("L900-0101", "1", "needed") + assert wp["flags"] == ["drawing_needed"] and wp["created_by"] is None + assert wp["title"] == "Inlet spool 6 in" and wp["customer"] == "Acme Refining" + assert wp["visual_rel"] == TODAY - timedelta(days=3) and wp["visual_want"] == TODAY + timedelta(days=20) + assert wp["visual_fp_applied"] and set(wp["visual"]["fp_parts"]) >= {"legs", "qty", "rev", "drawing", "bom"} + assert wp["visual_fp_applied"] == fingerprint._full(wp["visual"]["fp_parts"]) + assert {f["seq"] for f in wp["visual"]["folded"]} == {30, 50} + assert wp["visual"]["partspecs"] == ["Cut to length per sheet 1", "Weld per WPS-11\nBack purge"] + assert wp["source"]["kind"] == "visual" + legs = legs_of(conn, wp["id"]) + assert len(legs) == 1 and legs[0][:5] == ("L900-01", "1", "0", "0", 20) + assert (legs[0][5], legs[0][6], legs[0][7]) == (Decimal(2), Decimal(3), "F") + wo = row(conn, "SELECT * FROM packet_work_orders WHERE packet_id = %s", (wp["id"],)) + assert (wo["dwg"], wo["rev"], wo["op"]) == ("L900-0101", "1", "WELD-PART") + bom = conn.execute( + "SELECT coalesce(cl_item, item::text), part_id, qty, loc, sub, na, src, vkey FROM bom_lines" + " WHERE packet_id = %s ORDER BY sort", + (wp["id"],), + ).fetchall() + assert {b[1] for b in bom} == {"PIPE-6-SCH40", "L900-01-SUB", "WOL-1-6"} + assert all(b[6] == "visual" for b in bom) + sub = next(b for b in bom if b[1] == "L900-01-SUB") + assert sub[5] is True + main = next(b for b in bom if b[1] == "PIPE-6-SCH40") + assert main[3] == "A-01" and main[7] == "L900-01|1|0|20|1" + vs = row(conn, "SELECT * FROM packet_visual_state WHERE packet_id = %s", (wp["id"],)) + assert vs["ticket_hours"] == Decimal("1.25") and vs["observed_at"] is not None + assert {(lg["seq"], lg["st"]) for lg in vs["legs"]} == {(20, "F")} + + two = packet_by_key(conn, gen.KEY_TWO_OPS) + assert two is not None + assert [lg[4] for lg in legs_of(conn, two["id"])] == [10, 20] # two operations, one packet + + nd = packet_by_key(conn, gen.KEY_NO_DWG) + assert nd is not None and nd["drawing_state"] == "none" and "no_drawing" in nd["flags"] + assert nd["drawing"] in (None, "") + + rev_a = packet_by_key(conn, gen.KEY_REV_A) + assert rev_a is not None and rev_a["rev"] == "A" + items = [ + r[0] + for r in conn.execute( + "SELECT coalesce(cl_item, item::text) FROM bom_lines WHERE packet_id = %s ORDER BY sort", (rev_a["id"],) + ).fetchall() + ] + assert sorted(items) == ["1", "1-1", "2"] + + log = conn.execute( + "SELECT by, by_name, role FROM packet_log WHERE packet_id = %s AND action = 'Packet created from VISUAL'", + (wp["id"],), + ).fetchall() + assert log == [(None, "VISUAL feed", "system")] + created = system_audit(conn, "packet.created") + assert len(created) == len(gen.STATE1_KEYS) and all(r[0] == "system" and r[1] is None for r in created) + assert all(r[0] == "system" for r in system_audit(conn, "project.created")) + assert len(system_audit(conn, "project.created")) == 2 + assert len(system_audit(conn, "unit.created")) == 3 + assert len(system_audit(conn, "feed.run")) == 1 + no_human_writes(conn) + + batch = row(conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT 1") + assert batch["suspect"] is False and batch["window_rows"] == window_firmed_wos() + assert conn.execute("SELECT count(*) FROM visual_rows WHERE gone_batch IS NULL").fetchone()[0] >= ( + window_firmed_rows() + ) + assert conn.execute("SELECT count(*) FROM visual_orders").fetchone()[0] == 3 + assert conn.execute("SELECT count(*) FROM visual_rows WHERE lot = '7A'").fetchone()[0] >= 1 + assert conn.execute("SELECT count(*) FROM visual_labor").fetchone()[0] == 2 + + +def test_second_run_changes_nothing(fx: Settings) -> None: + run_feed(fx, 1) + with app_conn(fx) as conn: + before = conn.execute("SELECT id, version FROM packets ORDER BY id").fetchall() + logs = conn.execute("SELECT count(*) FROM packet_log").fetchone()[0] + c = run_feed(fx, 1) + assert c["_status"] == "ok" + assert {k: c.get(k, 0) for k in CHANGE_KEYS} == dict.fromkeys(CHANGE_KEYS, 0) + with app_conn(fx) as conn: + assert conn.execute("SELECT id, version FROM packets ORDER BY id").fetchall() == before + assert conn.execute("SELECT count(*) FROM packet_log").fetchone()[0] == logs + assert conn.execute("SELECT count(*) FROM visual_row_changes WHERE kind <> 'added'").fetchone()[0] == 0 + + +def test_hand_customer_is_never_overwritten(fx: Settings) -> None: + run_feed(fx, 1) + with app_conn(fx) as conn: + conn.execute( + "UPDATE units SET customer = 'Hand Co', customer_src = 'hand' WHERE lot = '001'" + " AND project_id = (SELECT id FROM projects WHERE number = 'L900')" + ) + run_feed(fx, 1) + with app_conn(fx) as conn: + u = row( + conn, + "SELECT u.* FROM units u JOIN projects p ON p.id = u.project_id WHERE p.number = 'L900' AND u.lot = '001'", + ) + assert (u["customer"], u["customer_src"], u["customer_visual"]) == ("Hand Co", "hand", "Bayou Chemical") + + +def test_order_pin_wins(fx: Settings) -> None: + run_feed(fx, 1) + with app_conn(fx) as conn: + conn.execute( + "UPDATE units SET order_pin = %s, order_pin_reason = 'Spool is for Acme' WHERE lot = '001'" + " AND project_id = (SELECT id FROM projects WHERE number = 'L900')", + (gen.ORDER_ACME,), + ) + run_feed(fx, 1) + with app_conn(fx) as conn: + u = row( + conn, + "SELECT u.* FROM units u JOIN projects p ON p.id = u.project_id WHERE p.number = 'L900' AND u.lot = '001'", + ) + assert u["order_ref"]["cust_order_id"] == gen.ORDER_ACME and u["order_ref"]["via"] == "pin" + assert u["customer"] == "Acme Refining" + + +def test_golive_ignores_and_pull_in(fx: Settings) -> None: + set_feed(fx, golive_date=TODAY + timedelta(days=gen.GOLIVE_OFFSET)) + c = run_feed(fx, 1) + assert c["ignored_golive"] == 1 and c["new_drafts"] == len(gen.STATE1_KEYS) - 1 + with app_conn(fx) as conn: + assert packet_by_key(conn, gen.KEY_GOLIVE, live=False) is None + ig = row(conn, "SELECT * FROM visual_ignored_wos") + assert (ig["wo_base"], ig["lot"], ig["split"], ig["sub"]) == ("L900-13", "2", "0", "0") + assert ig["pulled_in_at"] is None and ig["rel"] == TODAY - timedelta(days=10) + conn.execute( + "UPDATE visual_ignored_wos SET pulled_in_at = now(), pulled_in_reason = 'needed' WHERE id = %s", (ig["id"],) + ) + c = run_feed(fx, 1) + assert c["new_drafts"] == 1 + with app_conn(fx) as conn: + assert packet_by_key(conn, gen.KEY_GOLIVE) is not None + + +# ---------------------------------------------------------------- state 2 + + +def _shop(fx: Settings) -> tuple[int, int]: + """Run state 1, then put the WELD-PART packet in the shop (ready) and the not-found one at the warehouse.""" + run_feed(fx, 1) + with app_conn(fx) as conn: + wp, g = packet_id(conn, gen.KEY_WELD_PART), packet_id(conn, gen.KEY_NOT_FOUND) + move_status(conn, wp, "ready") + move_status(conn, g, "warehouse") + return wp, g + + +def test_state_2_change_orders_updates_gone_and_moved(fx: Settings) -> None: + wp, g = _shop(fx) + with app_conn(fx) as conn: + fp1 = row(conn, "SELECT visual_fp_applied FROM packets WHERE id = %s", (wp,))["visual_fp_applied"] + two_id = packet_id(conn, gen.KEY_TWO_OPS) + moves_id = packet_id(conn, gen.KEY_MOVES) + c = run_feed(fx, 2) + assert c["_status"] == "ok", c + assert c["cos_opened"] == 2 and c["drafts_updated"] >= 2 and c["retired"] == 1, c + assert c["wo_gone_flagged"] == 0 # the first miss only counts + with app_conn(fx) as conn: + # rev 1 -> 2 on an in-shop packet: a change order waiting for the rev 2 drawing; the packet stays as it was + p = row(conn, "SELECT * FROM packets WHERE id = %s", (wp,)) + assert (p["status"], p["rev"], p["visual_fp_applied"]) == ("ready", "1", fp1) + cos = cos_of(conn, wp) + assert len(cos) == 1 + co = cos[0] + assert (co["origin"], co["review"], co["minor"], co["ecn"], co["issued_by"]) == ( + "visual", + "pending", + False, + "VISUAL", + None, + ) + assert co["target_fp"] and co["target_fp"] != fp1 + assert co["vis"]["to_rev"] == "2" and co["vis"]["from_rev"] == "1" and co["vis"]["fp_parts"] + assert co["waiting"]["dwg"] == "L900-0101" and co["waiting"]["rev"] == "2" + assert co["staged"] is None + + # tgt only: a minor change order with nothing to wait for + gco = cos_of(conn, g) + assert len(gco) == 1 and gco[0]["minor"] is True and gco[0]["waiting"] is None + + # qty change on a draft: updated in place, both operations + two = row(conn, "SELECT * FROM packets WHERE id = %s", (two_id,)) + assert two["status"] == "draft" and cos_of(conn, two_id) == [] + assert [lg[5] for lg in legs_of(conn, two_id)] == [Decimal(3), Decimal(3)] + assert ( + conn.execute( + "SELECT count(*) FROM packet_log WHERE packet_id = %s AND action = 'Updated from VISUAL'", (two_id,) + ).fetchone()[0] + == 1 + ) + + # the LAYOUT draft shares the drawing: rev moved in place + lay = packet_by_key(conn, gen.KEY_LAYOUT_A) + assert lay is not None and lay["rev"] == "2" and lay["drawing_state"] == "needed" + + # retire-moved + old = row(conn, "SELECT * FROM packets WHERE id = %s", (moves_id,)) + assert old["removed_at"] is not None and old["removed_by"] is None and old["removed_reason"] == "moved" + assert old["visual"]["moved"]["to"] + assert packet_by_key(conn, gen.KEY_MOVED_TO) is not None + + # gone: one miss counted, no flag yet + e = packet_id(conn, gen.KEY_DRAFTING) + assert [lg[8] for lg in legs_of(conn, e)] == [1] + assert "wo_gone" not in row(conn, "SELECT flags FROM packets WHERE id = %s", (e,))["flags"] + + # closed and cancelled observed + j = row(conn, "SELECT * FROM packets WHERE visual_key = %s", (gen.KEY_CLOSES,)) + assert j["visual_closed_at"] is not None and legs_of(conn, j["id"])[0][7] == "C" + k = row(conn, "SELECT * FROM packets WHERE visual_key = %s", (gen.KEY_CANCELS,)) + assert "cancelled" in k["flags"] and k["visual_cancelled_at"] is not None and k["removed_at"] is None + assert all(r[0] == "system" for r in system_audit(conn, "packet.vis_open_co")) + assert all(r[0] == "system" for r in system_audit(conn, "packet.vis_update_draft")) + assert len(system_audit(conn, "packet.vis_retire_draft")) == 1 + no_human_writes(conn) + + # the second miss flags the work order gone; nothing else moves + c = run_feed(fx, 2) + assert c["wo_gone_flagged"] == 1 and c["cos_opened"] == 0 and c["cos_reaimed"] == 0, c + assert c["drafts_updated"] == 0 and c["retired"] == 0 and c["new_drafts"] == 0 + with app_conn(fx) as conn: + e = row(conn, "SELECT * FROM packets WHERE visual_key = %s AND removed_at IS NULL", (gen.KEY_DRAFTING,)) + assert "wo_gone" in e["flags"] and e["wo_gone_at"] is not None + assert [lg[8:] for lg in legs_of(conn, e["id"])] == [(2, True)] + assert len(cos_of(conn, wp)) == 1 # the same CO, not a second one + assert ( + conn.execute( + "SELECT count(*) FROM packet_log WHERE packet_id = %s" + " AND action LIKE 'Work order%%no longer in VISUAL'", + (e["id"],), + ).fetchone()[0] + == 1 + ) + + # VISUAL reverts: the pending feed change orders are withdrawn and the gone work order is back + c = run_feed(fx, 1) + assert c["cos_withdrawn"] == 2, c + with app_conn(fx) as conn: + co = cos_of(conn, wp)[0] + assert co["review"] == "rejected" and co["rejected_by"] is None and co["rejected_reason"] + e = row(conn, "SELECT * FROM packets WHERE visual_key = %s AND removed_at IS NULL", (gen.KEY_DRAFTING,)) + assert "wo_gone" not in e["flags"] and e["wo_gone_at"] is None + assert [lg[8:] for lg in legs_of(conn, e["id"])] == [(0, False)] + k = row(conn, "SELECT * FROM packets WHERE visual_key = %s", (gen.KEY_CANCELS,)) + assert "cancelled" not in k["flags"] + # the retired key is never made again + assert ( + conn.execute( + "SELECT count(*) FROM packets WHERE visual_key = %s AND removed_at IS NULL", (gen.KEY_MOVES,) + ).fetchone()[0] + == 0 + ) + assert c["skipped_removed"] >= 1 + + +def test_rejected_target_is_not_reopened(fx: Settings) -> None: + wp, _ = _shop(fx) + run_feed(fx, 2) + with app_conn(fx) as conn: + conn.execute( + "UPDATE change_orders SET review = 'rejected', rejected_at = now(), rejected_reason = 'not this one'," + " row_version = row_version + 1 WHERE packet_id = %s", + (wp,), + ) + c = run_feed(fx, 2) + assert c["cos_opened"] == 0, c + with app_conn(fx) as conn: + assert len(cos_of(conn, wp)) == 1 + + +def test_complete_packet_gets_a_change_order(fx: Settings) -> None: + run_feed(fx, 1) + with app_conn(fx) as conn: + wp = packet_id(conn, gen.KEY_WELD_PART) + move_status(conn, wp, "complete") + fl = packet_id(conn, gen.KEY_TWO_OPS) + move_status(conn, fl, "filed") + c = run_feed(fx, 2) + with app_conn(fx) as conn: + assert len(cos_of(conn, wp)) == 1 + assert cos_of(conn, fl) == [] # filed: nothing + assert [lg[5] for lg in legs_of(conn, fl)] == [Decimal(2), Decimal(2)] + assert c["cos_opened"] >= 1 + + +def test_removed_packet_key_is_never_recreated(fx: Settings) -> None: + run_feed(fx, 1) + with app_conn(fx) as conn: + pid = packet_id(conn, gen.KEY_NO_DWG) + conn.execute("UPDATE packets SET removed_at = now(), removed_reason = 'not needed' WHERE id = %s", (pid,)) + c = run_feed(fx, 1) + assert c["new_drafts"] == 0 and c["skipped_removed"] >= 1 + with app_conn(fx) as conn: + assert packet_by_key(conn, gen.KEY_NO_DWG) is None + + +def test_closed_unit_is_skipped(fx: Settings) -> None: + run_feed(fx, 1) + with app_conn(fx) as conn: + conn.execute( + "UPDATE units SET status = 'closed' WHERE project_id = (SELECT id FROM projects WHERE number = 'L901')" + ) + c = run_feed(fx, 2) + assert c["skipped_closed"] >= 1 + + +# ---------------------------------------------------------------- pause, suspect, auto-release + + +def test_paused_skips_a_scheduled_run_but_a_request_runs(fx: Settings) -> None: + set_feed(fx, paused=True) + c = run_feed(fx, 1) + assert c["_status"] == "skipped" and c["reason"] == "paused" + with app_conn(fx) as conn: + assert conn.execute("SELECT count(*) FROM packets").fetchone()[0] == 0 + c = run_feed(fx, 1, requested=True) + assert c["_status"] == "ok" and c["_trigger"] == "request" and c["new_drafts"] == len(gen.STATE1_KEYS) + + +def test_suspect_batch_marks_nothing_gone(fx: Settings) -> None: + run_feed(fx, 1) + empty = gen.feed_state(1, TODAY) + empty.update({"rows": [], "materials": [], "labor": []}) + (fx.fixture_dir / "feed.3.json").write_text(json.dumps(empty), encoding="utf-8") + c = run_feed(fx, 3) + assert c["_status"] == "degraded" and c["suspect"] == 1, c + assert c["_errors"] + with app_conn(fx) as conn: + assert conn.execute("SELECT count(*) FROM visual_rows WHERE gone_batch IS NOT NULL").fetchone()[0] == 0 + assert conn.execute("SELECT max(missing_runs) FROM packet_work_orders").fetchone()[0] == 0 + assert row(conn, "SELECT suspect FROM visual_batches ORDER BY id DESC LIMIT 1")["suspect"] is True + assert conn.execute("SELECT count(*) FROM packets WHERE removed_at IS NOT NULL").fetchone()[0] == 0 + # back to normal: nothing was lost + c = run_feed(fx, 1) + assert c["suspect"] == 0 and c["new_drafts"] == 0 + + +def test_auto_release(fx: Settings) -> None: + run_feed(fx, 1) + run_drawings(fx) + set_feed(fx, auto_release=True, auto_release_max=1) + c = run_feed(fx, 1) + assert c["auto_released"] == 1, c + set_feed(fx, auto_release_max=25) + c = run_feed(fx, 1) + assert c["auto_released"] >= 1 + with app_conn(fx) as conn: + wp = packet_by_key(conn, gen.KEY_WELD_PART) + assert wp is not None and wp["status"] == "warehouse" and wp["heat_map"] is True + wh = row(conn, "SELECT * FROM packet_wh WHERE packet_id = %s", (wp["id"],)) + assert wh["stage"] == "incoming" and wh["by"] is None + tr = row(conn, "SELECT * FROM packet_transitions WHERE packet_id = %s", (wp["id"],)) + assert tr["action"] == "vis_auto_release" and tr["by"] is None + # flagged drafts stay with Clerical + for key in (gen.KEY_TWO_OPS, gen.KEY_NOT_FOUND, gen.KEY_DRAFTING, gen.KEY_NO_DWG): + p = packet_by_key(conn, key) + assert p is not None and p["status"] == "draft", key + assert all(r[0] == "system" for r in system_audit(conn, "packet.vis_auto_release")) + no_human_writes(conn) + + +def test_paused_stops_auto_release(fx: Settings) -> None: + run_feed(fx, 1) + run_drawings(fx) + set_feed(fx, auto_release=True, paused=True) + c = run_feed(fx, 1, requested=True) + assert c.get("auto_released", 0) == 0 + with app_conn(fx) as conn: + assert conn.execute("SELECT count(*) FROM packets WHERE status = 'warehouse'").fetchone()[0] == 0 + + +def test_switch_state_file(fx: Settings) -> None: + switch_state(fx.fixture_dir, 2) + assert (fx.fixture_dir / "state").read_text() == "2" diff --git a/tests/server/test_fingerprint.py b/tests/server/test_fingerprint.py new file mode 100644 index 0000000..f55b8c6 --- /dev/null +++ b/tests/server/test_fingerprint.py @@ -0,0 +1,215 @@ +"""The candidate fingerprint: which changes are major, minor or nothing, and what never counts.""" + +from __future__ import annotations + +import dataclasses +from collections.abc import Callable +from datetime import date +from decimal import Decimal +from typing import Any + +import pytest + +from packetstation.feed import fingerprint as fp +from packetstation.feed.candidates import BomLine, Candidate, Folded, Leg +from packetstation.feed.types import WoKey + +K1 = WoKey("L900-01", "1", "0", "0") +K2 = WoKey("L900-02", "1", "0", "0") + + +def leg(key: WoKey = K1, seq: int = 20, qty: Any = "2", tgt: Any = "3", **over: Any) -> Leg: + values: dict[str, Any] = { + "key": key, + "seq": seq, + "op": "WELD-PART", + "qty": Decimal(str(qty)), + "tgt": Decimal(str(tgt)), + "dwg": "L900-0101", + "rev": "1", + "partspecs": "", + "st": "F", + } + values.update(over) + return Leg(**values) + + +def line(item: str = "1", part: str | None = "PIPE-6", qty: Any = "2", **over: Any) -> BomLine: + values: dict[str, Any] = { + "item": item, + "qty": Decimal(str(qty)), + "part_id": part or "", + "desc": "Pipe", + "loc": "R1", + "whs": "SHOP", + "comm": "PIPE", + "cons": False, + "sub": False, + "na": False, + "grp": None, + "vkey": f"L900-01|1|0|20|{item}", + "issued": Decimal(0), + "istat": 2, + } + values.update(over) + return BomLine(**values) + + +def cand(**over: Any) -> Candidate: + values: dict[str, Any] = { + "key": "L900001|WELD-PART|L900-0101", + "project": "L900", + "unit": "L900001", + "lot": "1", + "dept": "03", + "op": "WELD-PART", + "drawing": "L900-0101", + "rev": "1", + "title": "Inlet spool", + "legs": [leg()], + "folded": [], + "partspecs": [], + "rel": date(2026, 10, 10), + "want": date(2026, 11, 5), + "bom": [line("1"), line("2", "ELB-6", "1")], + "statuses": {K1: "F"}, + "base": "L900-01", + "in_window": True, + } + values.update(over) + return Candidate(**values) + + +def classify(before: Candidate, after: Candidate) -> fp.Change: + return fp.classify(fp.compute(before).parts, fp.compute(after).parts) + + +MAJOR_EDITS: dict[str, tuple[str, Callable[[], dict[str, Any]]]] = { + "leg added": ("legs", lambda: {"legs": [leg(), leg(K2, 10)]}), + "leg removed": ("legs", lambda: {"legs": []}), + "leg seq": ("legs", lambda: {"legs": [leg(seq=30)]}), + "qty": ("qty", lambda: {"legs": [leg(qty="3")]}), + "rev": ("rev", lambda: {"rev": "2"}), + "drawing": ("drawing", lambda: {"drawing": "L900-0102"}), + "bom item": ("bom", lambda: {"bom": [line("1"), line("3", "ELB-6", "1")]}), + "bom part": ("bom", lambda: {"bom": [line("1"), line("2", "ELB-6-SR", "1")]}), + "bom qty": ("bom", lambda: {"bom": [line("1"), line("2", "ELB-6", "2")]}), + "bom line added": ("bom", lambda: {"bom": [line("1"), line("2", "ELB-6", "1"), line("3", "TEE-6", "1")]}), +} + + +@pytest.mark.parametrize("name", list(MAJOR_EDITS)) +def test_major_change(name: str) -> None: + part, edit = MAJOR_EDITS[name] + ch = classify(cand(), cand(**edit())) + assert ch.changed and not ch.is_minor + assert part in ch.major + + +MINOR_EDITS: dict[str, tuple[str, Callable[[], dict[str, Any]]]] = { + "tgt": ("tgt", lambda: {"legs": [leg(tgt="3.5")]}), + "want": ("want", lambda: {"want": date(2026, 11, 12)}), + "want cleared": ("want", lambda: {"want": None}), +} + + +@pytest.mark.parametrize("name", list(MINOR_EDITS)) +def test_minor_change(name: str) -> None: + part, edit = MINOR_EDITS[name] + ch = classify(cand(), cand(**edit())) + assert ch.changed and ch.is_minor + assert ch.major == frozenset() and ch.minor == frozenset({part}) + + +def test_minor_and_major_together_is_major() -> None: + ch = classify(cand(), cand(rev="2", want=date(2026, 12, 1))) + assert ch.major == {"rev"} and ch.minor == {"want"} and not ch.is_minor + + +NEVER_EDITS: dict[str, Callable[[], dict[str, Any]]] = { + "rel": lambda: {"rel": date(2026, 10, 1)}, + "folded": lambda: {"folded": [Folded(K1, 30, "MFG", Decimal("0.5"))]}, + "partspecs": lambda: {"partspecs": ["Weld per WPS-11"]}, + "title": lambda: {"title": "Another title"}, + "statuses": lambda: {"statuses": {K1: "R"}}, + "issued": lambda: {"bom": [line("1", issued=Decimal(2)), line("2", "ELB-6", "1")]}, + "whs": lambda: {"bom": [line("1", whs="MAIN"), line("2", "ELB-6", "1")]}, + "loc": lambda: {"bom": [line("1", loc="A-01"), line("2", "ELB-6", "1")]}, + "dept and op": lambda: {"dept": "06", "op": "WELD-CODE"}, + "leg status": lambda: {"legs": [leg(st="C")]}, + "leg partspecs": lambda: {"legs": [leg(partspecs="Back purge")]}, + "in_window": lambda: {"in_window": False}, +} + + +@pytest.mark.parametrize("name", list(NEVER_EDITS)) +def test_never_in_the_fingerprint(name: str) -> None: + before, after = cand(), cand(**NEVER_EDITS[name]()) + assert not classify(before, after).changed + assert fp.compute(before).full == fp.compute(after).full + + +INVARIANT: dict[str, tuple[Callable[[], dict[str, Any]], Callable[[], dict[str, Any]]]] = { + "2 vs 2.0": (lambda: {"legs": [leg(qty="2")]}, lambda: {"legs": [leg(qty="2.0")]}), + "trailing zeros": (lambda: {"legs": [leg(tgt="1.5")]}, lambda: {"legs": [leg(tgt="1.500")]}), + "empty vs None rev": (lambda: {"rev": ""}, lambda: {"rev": None}), + "list order": ( + lambda: {"legs": [leg(), leg(K2, 10)], "bom": [line("1"), line("2", "ELB-6", "1")]}, + lambda: {"legs": [leg(K2, 10), leg()], "bom": [line("2", "ELB-6", "1"), line("1")]}, + ), + "drawing case and spaces": (lambda: {"drawing": "m00543"}, lambda: {"drawing": " M00 543 "}), + "part id case": ( + lambda: {"bom": [line("1", "m00543")]}, + lambda: {"bom": [line("1", "M00543")]}, + ), + "CRLF": (lambda: {"rev": "1\r\n"}, lambda: {"rev": "1\n"}), +} + + +@pytest.mark.parametrize("name", list(INVARIANT)) +def test_invariance(name: str) -> None: + a, b = INVARIANT[name] + assert fp.compute(cand(**a())).full == fp.compute(cand(**b())).full + + +def test_lot_7a_stays_text() -> None: + k = WoKey("L900-14", "7A", "0", "0") + c = cand(legs=[leg(k, 10)], statuses={k: "F"}) + values = fp.part_values(c) + assert values["legs"] == [["L900-14", "7A", "0", "0", 10]] + # A lot with leading zeros is not a number either + k0 = WoKey("L900-14", "007", "0", "0") + other = fp.part_values(cand(legs=[leg(k0, 10)])) + assert other["legs"][0][1] == "007" + assert fp.compute(c).full != fp.compute(cand(legs=[leg(k0, 10)])).full + + +def test_canonical_is_stable_json() -> None: + assert fp.canonical({"b": Decimal("2.0"), "a": ["x\r\n", None, ""]}) == b'{"a":["x",null,null],"b":"2"}' + assert fp.canonical({"d": date(2026, 10, 16)}) == b'{"d":"2026-10-16"}' + assert fp.canonical("é") == fp.canonical("é") # NFC + + +def test_parts_and_full() -> None: + f = fp.compute(cand()) + assert set(f.parts) == set(fp.MAJOR) | set(fp.MINOR) + assert all(len(v) == 64 for v in f.parts.values()) and len(f.full) == 64 + assert fp.compute(cand()) == f # deterministic + + +def test_no_before_is_all_changed() -> None: + ch = fp.classify(None, fp.compute(cand()).parts) + assert ch.major == frozenset(fp.MAJOR) and ch.minor == frozenset(fp.MINOR) + + +def test_unknown_materials_are_not_a_bom_change() -> None: + before = fp.compute(cand()) + after = fp.compute(cand(bom=None)) + assert "bom" not in classify(cand(), cand(bom=None)).major + merged = fp.merge(before.parts, after) + assert merged.parts["bom"] == before.parts["bom"] and merged.full == before.full + + +def test_dataclass_copy_is_unchanged() -> None: + c = cand() + assert fp.compute(dataclasses.replace(c)).full == fp.compute(c).full diff --git a/tests/server/test_handbom.py b/tests/server/test_handbom.py new file mode 100644 index 0000000..f8c1171 --- /dev/null +++ b/tests/server/test_handbom.py @@ -0,0 +1,112 @@ +"""Hand-uploaded BOM files: header detection, the column patterns, skipped rows and bad files.""" + +from __future__ import annotations + +import io +from decimal import Decimal +from typing import Any + +import pytest + +from packetstation.domain import handbom + + +def xlsx(rows: list[list[Any]]) -> bytes: + from openpyxl import Workbook + + wb = Workbook() + ws = wb.active + assert ws is not None + for r in rows: + ws.append(r) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def csv_bytes(text: str, bom: bool = True) -> bytes: + return (b"\xef\xbb\xbf" if bom else b"") + text.encode("utf-8") + + +def test_csv_header_after_title_rows() -> None: + data = csv_bytes( + "Bill of material,,,\n" + "Job L900-01,,,\n" + "\n" + "Item,Qty,Part No.,Description,Length,Material,Dwg Ref\n" + "1,2,PIPE-6-SCH40,Pipe 6 in sch 40,12 ft,A106-B,L900-0101\n" + "2,4,,Gusset 1/4 plate,,A36,\n" + ) + lines = handbom.parse(data, "bom.csv") + assert [ln["item"] for ln in lines] == [1, 2] + first = lines[0] + assert first["qty"] == Decimal(2) and first["part_id"] == "PIPE-6-SCH40" + assert first["description"] == "Pipe 6 in sch 40" and first["length"] == "12 ft" + assert first["material"] == "A106-B" and first["dwg_ref"] == "L900-0101" + assert lines[1]["part_id"] is None and lines[1]["description"] == "Gusset 1/4 plate" + + +def test_xlsx_with_quantity_and_part_number() -> None: + data = xlsx( + [ + ["PESCO", None, None], + ["Find", "Quantity", "PESCO Part Number", "Desc"], + [10, 1, "FLG-6-150", "Flange"], + [None, None, None, None], + [11, 2.5, "GSK-6-150", "Gasket"], + ] + ) + lines = handbom.parse(data, "bom.xlsx") + assert [(ln["item"], ln["qty"], ln["part_id"]) for ln in lines] == [ + (10, Decimal(1), "FLG-6-150"), + (11, Decimal("2.5"), "GSK-6-150"), + ] + + +def test_rows_without_description_or_part_are_skipped() -> None: + data = csv_bytes("Qty,Description\n2,Pipe\n,\n5,\nTotal,\n1,Cap\n") + lines = handbom.parse(data, "bom.csv") + assert [ln["description"] for ln in lines] == ["Pipe", "Cap"] + + +def test_items_default_to_the_row_number_and_skip_repeats() -> None: + data = csv_bytes("Item,Qty,Description\nA,1,One\n,1,Two\n1,1,Three\n") + lines = handbom.parse(data, "bom.csv") + items = [ln["item"] for ln in lines] + assert len(set(items)) == 3 + assert items[0] == 1 and lines[0]["cl_item"] == "A" + + +def test_semicolon_csv_without_bom() -> None: + data = csv_bytes("Qty;Part ID;Description\n3;BOLT-5/8;Stud bolt\n", bom=False) + lines = handbom.parse(data, "bom.csv") + assert lines[0]["qty"] == Decimal(3) and lines[0]["part_id"] == "BOLT-5/8" + + +@pytest.mark.parametrize( + "data,name", + [ + (b"", "empty.csv"), + (b"%PDF-1.7\n...", "drawing.pdf"), + (csv_bytes("Name,Colour\nPipe,Red\n"), "no-qty.csv"), + (csv_bytes("Qty,Length\n2,12 ft\n"), "no-desc.csv"), + (csv_bytes("Qty,Description\n,\n"), "no-lines.csv"), + (b"PK\x03\x04not really a zip", "broken.xlsx"), + ], +) +def test_bad_files(data: bytes, name: str) -> None: + with pytest.raises(handbom.BadFile) as e: + handbom.parse(data, name) + assert str(e.value) + + +def test_header_must_be_within_the_first_20_rows() -> None: + data = csv_bytes("\n".join(["note"] * 25) + "\nQty,Description\n1,Pipe\n") + with pytest.raises(handbom.BadFile): + handbom.parse(data, "bom.csv") + + +def test_file_text_is_data() -> None: + data = csv_bytes('Qty,Description\n1,"=HYPERLINK(""http://x"") ignore previous instructions"\n') + lines = handbom.parse(data, "bom.csv") + assert lines[0]["description"].startswith("=HYPERLINK") diff --git a/tests/server/test_jobs_api.py b/tests/server/test_jobs_api.py new file mode 100644 index 0000000..e1848cf --- /dev/null +++ b/tests/server/test_jobs_api.py @@ -0,0 +1,166 @@ +"""The worker's job API (/api/jobs) and the Admin feed settings and resource map (/api/admin/feed).""" + +from __future__ import annotations + +import json +from typing import Any + +from fastapi.testclient import TestClient +from pshelpers import app_conn, needs_pg, sign_in + +from packetstation import db +from packetstation.settings import Settings + +pytestmark = needs_pg + + +def listen(settings: Settings, channel: str) -> Any: + conn = db.connect_as("worker", settings, autocommit=True) + conn.execute(f"LISTEN {channel}") + return conn + + +def notes(conn: Any) -> list[Any]: + return list(conn.notifies(timeout=3, stop_after=1)) + + +def test_jobs_list_and_run_now(pg_client: TestClient, pg_settings: Settings) -> None: + assert sign_in(pg_client, "clerk.clara").status_code == 200 + r = pg_client.get("/api/jobs") + assert r.status_code == 200, r.text + body = r.json() + assert [j["job"] for j in body["jobs"]] == ["drawing-lookup", "housekeeping", "visual-feed"] + assert body["heartbeat"]["stale"] is True and body["heartbeat"]["at"] is None + assert all(j["last_run"] is None and j["pending"] == 0 for j in body["jobs"]) + + listener = listen(pg_settings, "ps_job_request") + try: + r = pg_client.post("/api/jobs/visual-feed/run", json={"note": "check"}) + assert r.status_code == 200 and r.json() == {"requested": True, "pending": 1}, r.text + got = notes(listener) + assert got and json.loads(got[0].payload) == {"job": "visual-feed"} + finally: + listener.close() + r = pg_client.post("/api/jobs/visual-feed/run") + assert r.json()["pending"] == 2 + with app_conn(pg_settings) as conn: + req = conn.execute( + "SELECT r.note, u.sam_account FROM job_requests r JOIN users u ON u.id = r.requested_by ORDER BY r.id" + ).fetchall() + assert req == [("check", "clerk.clara"), (None, "clerk.clara")] + audit = conn.execute( + "SELECT role_snapshot, subject_id FROM audit_events WHERE action = 'job.run_requested'" + ).fetchall() + assert len(audit) == 2 and audit[0][1] == "visual-feed" + assert [j["pending"] for j in pg_client.get("/api/jobs").json()["jobs"] if j["job"] == "visual-feed"] == [2] + + +def test_jobs_unknown_and_refused(pg_client: TestClient) -> None: + assert sign_in(pg_client, "clerk.clara").status_code == 200 + assert pg_client.post("/api/jobs/no-such/run").status_code == 404 + assert pg_client.get("/api/jobs/no-such/runs").status_code == 404 + assert pg_client.post("/api/jobs/DROP TABLE/run").status_code == 404 + assert sign_in(pg_client, "wh.wade").status_code == 200 + assert pg_client.get("/api/jobs").status_code == 403 + assert pg_client.post("/api/jobs/visual-feed/run").status_code == 403 + + +def test_job_runs_listing(pg_client: TestClient, pg_settings: Settings) -> None: + with app_conn(pg_settings) as conn: + for i in range(12): + conn.execute( + "INSERT INTO job_runs (job, trigger, status, counts, finished_at) VALUES" + " ('visual-feed', 'schedule', 'ok', %s, now())", + (json.dumps({"new_drafts": i}),), + ) + assert sign_in(pg_client, "admin.ann").status_code == 200 + runs = pg_client.get("/api/jobs/visual-feed/runs").json() + assert len(runs) == 10 and runs[0]["counts"] == {"new_drafts": 11} + assert len(pg_client.get("/api/jobs/visual-feed/runs?limit=3").json()) == 3 + last = next(j for j in pg_client.get("/api/jobs").json()["jobs"] if j["job"] == "visual-feed")["last_run"] + assert last["status"] == "ok" and last["counts"] == {"new_drafts": 11} + + +def test_feed_settings_patch_is_audited_and_notifies(pg_client: TestClient, pg_settings: Settings) -> None: + assert sign_in(pg_client, "clerk.clara").status_code == 200 + assert pg_client.get("/api/admin/feed/settings").status_code == 403 + assert pg_client.patch("/api/admin/feed/settings", json={"paused": True}).status_code == 403 + + assert sign_in(pg_client, "admin.ann").status_code == 200 + r = pg_client.get("/api/admin/feed/settings") + assert r.status_code == 200, r.text + s = r.json() + assert s["interval_shop_minutes"] == 5 and s["paused"] is False and s["auto_release_max"] == 25 + assert "runs" in s and "next_run_at" in s + + listener = listen(pg_settings, "ps_settings") + try: + r = pg_client.patch( + "/api/admin/feed/settings", + json={"interval_shop_minutes": 7, "golive_date": "2026-11-02", "departments": ["03", "06"]}, + ) + assert r.status_code == 200, r.text + assert r.json()["interval_shop_minutes"] == 7 and r.json()["golive_date"] == "2026-11-02" + assert notes(listener) + finally: + listener.close() + with app_conn(pg_settings) as conn: + detail, role = conn.execute( + "SELECT detail, role_snapshot FROM audit_events WHERE action = 'feed.settings_changed' ORDER BY id DESC" + ).fetchone() + assert detail["changed"]["interval_shop_minutes"] == {"from": 5, "to": 7} + assert detail["changed"]["departments"] == {"from": [], "to": ["03", "06"]} + assert "paused" not in detail["changed"] and "admin" in role + assert conn.execute("SELECT value->>'interval_shop_minutes' FROM settings WHERE key = 'feed'").fetchone() == ( + "7", + ) + + for bad in ( + {"interval_shop_minutes": 0}, + {"shop_start": "6am"}, + {"nope": 1}, + {"departments": ["99"]}, + {"window_back_days": 500}, + ): + r = pg_client.patch("/api/admin/feed/settings", json=bad) + assert r.status_code == 422 and r.json()["code"] == "bad_settings", (bad, r.text) + + +def test_resource_map(pg_client: TestClient, pg_settings: Settings) -> None: + assert sign_in(pg_client, "clerk.clara").status_code == 200 + assert pg_client.get("/api/admin/feed/resource-map").status_code == 403 + assert sign_in(pg_client, "admin.ann").status_code == 200 + rows = {r["resource"]: r for r in pg_client.get("/api/admin/feed/resource-map").json()} + assert rows["WELD-PART"]["target"] == "03" and rows["WELD-PART"]["confirmed"] is False + assert rows["MFG"]["target"] == "fold" and rows["FORKLIFT"]["target"] == "ignore" + assert rows["Electical"]["target"] == "14" + + r = pg_client.put("/api/admin/feed/resource-map/NEWRES", json={"target": "03"}) + assert r.status_code == 200 and r.json()["target"] == "03" and r.json()["confirmed"] is True, r.text + r = pg_client.put("/api/admin/feed/resource-map/NEWRES", json={"target": "Fold"}) + assert r.status_code == 200 and r.json()["target"] == "fold" + for bad in ("77", "WH", "", "fold; drop"): + r = pg_client.put("/api/admin/feed/resource-map/NEWRES", json={"target": bad}) + assert r.status_code == 422, (bad, r.text) + assert pg_client.delete("/api/admin/feed/resource-map/NEWRES").status_code == 204 + assert pg_client.delete("/api/admin/feed/resource-map/NEWRES").status_code == 404 + with app_conn(pg_settings) as conn: + acts = conn.execute( + "SELECT action, detail FROM audit_events WHERE subject_type = 'visual_resource' ORDER BY id" + ).fetchall() + assert [a[0] for a in acts] == ["feed.resource_mapped", "feed.resource_mapped", "feed.resource_unmapped"] + assert acts[1][1]["before"]["target"] == "03" and acts[1][1]["after"]["target"] == "fold" + assert acts[2][1]["after"] is None + + +def test_resource_map_lists_unmapped_from_the_last_batch( + pg_client: TestClient, pg_settings: Settings, tmp_path: Any +) -> None: + from feedhelpers import feed_settings_over, run_feed + + fx = feed_settings_over(pg_settings, tmp_path) + run_feed(fx, 1) + assert sign_in(pg_client, "admin.ann").status_code == 200 + rows = {r["resource"]: r for r in pg_client.get("/api/admin/feed/resource-map").json()} + assert rows["NEWRES"]["target"] is None and rows["NEWRES"]["rows_last_run"] == 2 + assert rows["WELD-PART"]["rows_last_run"] >= 1 diff --git a/tests/server/test_scheduler.py b/tests/server/test_scheduler.py new file mode 100644 index 0000000..5f768ba --- /dev/null +++ b/tests/server/test_scheduler.py @@ -0,0 +1,315 @@ +"""The worker's timing, pause, redaction and run bookkeeping (run_job against PostgreSQL with a fake job).""" + +from __future__ import annotations + +import threading +from datetime import date, datetime, timedelta +from typing import Any +from zoneinfo import ZoneInfo + +import pytest +from pshelpers import app_conn, needs_pg +from psycopg.types.json import Jsonb + +from packetstation.feed import config +from packetstation.feed.config import FeedSettings +from packetstation.settings import Settings +from packetstation.worker import runs +from packetstation.worker.jobs import JobContext, JobSpec, load_jobs, seconds_until +from packetstation.worker.redaction import clip, redact +from packetstation.worker.scheduler import Scheduler, next_due + +TZ = ZoneInfo("America/Chicago") +FRI = date(2026, 10, 16) # a Friday +SAT = date(2026, 10, 17) + + +def at(d: date, hh: int, mm: int = 0) -> datetime: + return datetime(d.year, d.month, d.day, hh, mm, tzinfo=TZ) + + +# ---------------------------------------------------------------- cadence (pure) + + +@pytest.mark.parametrize( + "when,inside", + [ + (at(FRI, 6, 0), True), + (at(FRI, 12, 30), True), + (at(FRI, 17, 59), True), + (at(FRI, 18, 0), False), + (at(FRI, 5, 59), False), + (at(SAT, 12, 0), False), + (at(date(2026, 10, 18), 12, 0), False), # Sunday + (at(date(2026, 10, 19), 7, 0), True), # Monday + ], +) +def test_in_shop_hours(when: datetime, inside: bool) -> None: + fs = FeedSettings() + assert config.in_shop_hours(fs, when) is inside + assert config.interval_seconds(fs, when) == (300 if inside else 1800) + + +def test_interval_follows_settings() -> None: + fs = FeedSettings( + interval_shop_minutes=2, interval_off_minutes=45, shop_days=[5], shop_start="08:00", shop_end="12:00" + ) + assert config.interval_seconds(fs, at(SAT, 9)) == 120 + assert config.interval_seconds(fs, at(FRI, 9)) == 45 * 60 + assert config.interval_seconds(fs, at(SAT, 12)) == 45 * 60 + + +def test_overnight_shift_belongs_to_the_day_it_started() -> None: + fs = FeedSettings(shop_start="22:00", shop_end="06:00", shop_days=[4]) + assert config.in_shop_hours(fs, at(FRI, 23)) + assert config.in_shop_hours(fs, at(SAT, 3)) # Friday's shift + assert not config.in_shop_hours(fs, at(SAT, 23)) + + +def test_window() -> None: + fs = FeedSettings(window_back_days=14, window_ahead_days=30) + assert config.window(fs, FRI) == (FRI - timedelta(days=14), FRI + timedelta(days=30)) + + +def test_settings_validation() -> None: + with pytest.raises(config.SettingsError): + config.validate_patch(FeedSettings(), {"interval_shop_minutes": 0}) + with pytest.raises(config.SettingsError): + config.validate_patch(FeedSettings(), {"shop_start": "25:00"}) + with pytest.raises(config.SettingsError): + config.validate_patch(FeedSettings(), {"no_such_key": 1}) + assert config.from_value({"interval_shop_minutes": 7, "unknown": 1}).interval_shop_minutes == 7 + assert config.from_value({"interval_shop_minutes": -3, "paused": True}).paused is True # good fields kept + + +def _spec(name: str = "fake", run: Any = None, pausable: bool = True, cadence: Any = None) -> JobSpec: + return JobSpec( + name=name, + run=run or (lambda ctx: {"done": 1}), + cadence=cadence or (lambda fs, now: config.interval_seconds(fs, now)), + pausable=pausable, + ) + + +def test_registered_jobs_and_cadences() -> None: + jobs = load_jobs() + assert {"housekeeping", "visual-feed", "drawing-lookup"} <= set(jobs) + fs = FeedSettings(drawing_interval_minutes=20) + assert jobs["visual-feed"].cadence(fs, at(FRI, 10)) == 300 + assert jobs["visual-feed"].cadence(fs, at(SAT, 10)) == 1800 + assert jobs["drawing-lookup"].cadence(fs, at(FRI, 10)) == 1200 + assert "drawing-lookup" in jobs["visual-feed"].chain + assert jobs["visual-feed"].pausable and jobs["drawing-lookup"].pausable + hk = jobs["housekeeping"].cadence(fs, at(FRI, 3, 0)) + assert hk == 30 * 60 # 03:30 site time + + +def test_next_due() -> None: + spec = _spec() + fs = FeedSettings() + now = at(FRI, 10) + assert next_due(spec, fs, now) == now + timedelta(minutes=5) + assert next_due(spec, fs, now, now - timedelta(minutes=2)) == now + timedelta(minutes=3) + assert next_due(spec, fs, now, now - timedelta(hours=1)) == now # overdue: now, never in the past + assert next_due(_spec(cadence=lambda fs, n: None), fs, now) is None + + +def test_seconds_until() -> None: + assert seconds_until(at(FRI, 3, 0), datetime.min.time().replace(hour=3, minute=30)) == 1800 + assert seconds_until(at(FRI, 4, 0), datetime.min.time().replace(hour=3, minute=30)) == 23 * 3600 + 1800 + + +# ---------------------------------------------------------------- redaction (pure) + + +@pytest.mark.parametrize( + "text,secret", + [ + ("connect failed: password=Hunter2xyz host=vsql", "Hunter2xyz"), + ("odbc PWD=S3cret!!;UID=visualkit_ro", "S3cret!!"), + ("Authorization: Bearer abc.def.ghi", "abc.def.ghi"), + ("X-Authentication: tok123456", "tok123456"), + ("postgresql://ps_app:pa55word@db/ps", "pa55word"), + ], +) +def test_redact_patterns(text: str, secret: str) -> None: + out = redact(text) + assert secret not in out and "[redacted]" in out + + +def test_redact_environment_passwords(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None: + monkeypatch.setenv("MFILES_PASSWORD", "vault-pass-0001") + f = tmp_path / "visual_pw" + f.write_text("visual-pass-0002\n") + monkeypatch.setenv("VISUAL_PASSWORD_FILE", str(f)) + out = redact("login vault-pass-0001 failed; then visual-pass-0002 failed") + assert "vault-pass-0001" not in out and "visual-pass-0002" not in out + assert runs.redact is redact or "vault-pass-0001" not in runs.redact("x vault-pass-0001") + + +def test_clip() -> None: + out = clip([f"error {i} " + "x" * 600 for i in range(30)]) + assert len(out) == 20 and all(len(e) <= 500 for e in out) + + +# ---------------------------------------------------------------- run_job (PostgreSQL) + + +def _sched(settings: Settings, spec: JobSpec) -> Scheduler: + return Scheduler(settings, build="pytest", jobs={spec.name: spec}) + + +def _runs(settings: Settings, job: str = "fake") -> list[tuple[Any, ...]]: + with app_conn(settings) as conn: + return [ + tuple(r) + for r in conn.execute( + "SELECT status, trigger, counts, errors, finished_at IS NOT NULL, build_version FROM job_runs" + " WHERE job = %s ORDER BY id", + (job,), + ).fetchall() + ] + + +@needs_pg +def test_run_job_records_a_run(pg_settings: Settings) -> None: + seen: list[JobContext] = [] + + def run(ctx: JobContext) -> dict[str, Any]: + seen.append(ctx) + assert ctx.conn.execute("SELECT current_setting('application_name')").fetchone()[0] == "ps-worker" + return {"done": 3} + + row = runs.run_job(_sched(pg_settings, _spec(run=run)), "fake", "schedule") + assert row["status"] == "ok" and row["counts"] == {"done": 3} and row["trigger"] == "schedule" + assert seen and seen[0].run_id == row["id"] and seen[0].trigger == "schedule" + assert _runs(pg_settings) == [("ok", "schedule", {"done": 3}, [], True, "pytest")] + + +@needs_pg +def test_run_job_degraded_and_failed(pg_settings: Settings) -> None: + def degraded(ctx: JobContext) -> dict[str, Any]: + ctx.error("orders read failed: password=hunter22") + return {"n": 1} + + def boom(ctx: JobContext) -> dict[str, Any]: + raise RuntimeError("PWD=topsecret9 broke") + + row = runs.run_job(_sched(pg_settings, _spec(run=degraded)), "fake", "schedule") + assert row["status"] == "degraded" and row["errors"] and "hunter22" not in row["errors"][0] + row = runs.run_job(_sched(pg_settings, _spec(run=boom)), "fake", "schedule") + assert row["status"] == "failed" and len(row["errors"]) == 1 and "topsecret9" not in row["errors"][0] + assert row["finished_at"] is not None + + +@needs_pg +def test_ten_requests_one_run(pg_settings: Settings) -> None: + with app_conn(pg_settings) as conn: + for _ in range(10): + conn.execute("INSERT INTO job_requests (job) VALUES ('fake')") + calls: list[int] = [] + row = runs.run_job(_sched(pg_settings, _spec(run=lambda ctx: calls.append(1) or {})), "fake", "schedule") + assert calls == [1] and row["trigger"] == "request" + with app_conn(pg_settings) as conn: + claimed = conn.execute( + "SELECT count(*), count(DISTINCT claimed_run_id) FROM job_requests WHERE claimed_at IS NOT NULL" + ).fetchone() + assert claimed == (10, 1) + assert conn.execute("SELECT count(*) FROM job_requests WHERE claimed_at IS NULL").fetchone()[0] == 0 + assert len(_runs(pg_settings)) == 1 + + +@needs_pg +def test_skips_when_the_lock_is_held(pg_settings: Settings) -> None: + from packetstation import db + + calls: list[int] = [] + with db.connect_as("worker", pg_settings, autocommit=True) as holder: + assert holder.execute("SELECT pg_try_advisory_lock(7310, hashtext('fake'))").fetchone()[0] + row = runs.run_job(_sched(pg_settings, _spec(run=lambda ctx: calls.append(1) or {})), "fake", "schedule") + assert calls == [] and row["status"] == "skipped" and row["counts"] == {"reason": "already running"} + row = runs.run_job(_sched(pg_settings, _spec(run=lambda ctx: calls.append(1) or {})), "fake", "schedule") + assert calls == [1] and row["status"] == "ok" + + +@needs_pg +def test_concurrent_runs_one_skips(pg_settings: Settings) -> None: + started, release = threading.Event(), threading.Event() + + def slow(ctx: JobContext) -> dict[str, Any]: + started.set() + release.wait(10) + return {} + + sch = _sched(pg_settings, _spec(run=slow)) + t = threading.Thread(target=runs.run_job, args=(sch, "fake", "schedule")) + t.start() + try: + assert started.wait(10) + assert sch.running() == ["fake"] + row = runs.run_job(_sched(pg_settings, _spec(run=lambda ctx: {})), "fake", "schedule") + assert row["status"] == "skipped" + finally: + release.set() + t.join(10) + assert sorted(r[0] for r in _runs(pg_settings)) == ["ok", "skipped"] + + +def _pause(settings: Settings, paused: bool) -> None: + with app_conn(settings) as conn: + conn.execute( + "INSERT INTO settings (key, value) VALUES ('feed', %s)" + " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + (Jsonb({"paused": paused}),), + ) + + +@needs_pg +def test_pause_skips_scheduled_fires_but_requests_run(pg_settings: Settings) -> None: + _pause(pg_settings, True) + calls: list[str] = [] + spec = _spec(run=lambda ctx: calls.append(ctx.trigger) or {}) + row = runs.run_job(_sched(pg_settings, spec), "fake", "schedule") + assert row["status"] == "skipped" and row["counts"] == {"reason": "paused"} and calls == [] + with app_conn(pg_settings) as conn: + conn.execute("INSERT INTO job_requests (job) VALUES ('fake')") + row = runs.run_job(_sched(pg_settings, spec), "fake", "schedule") + assert row["status"] == "ok" and calls == ["request"] + # a job that is not pausable runs on schedule while paused + row = runs.run_job( + _sched(pg_settings, _spec(run=lambda ctx: calls.append("np") or {}, pausable=False)), "fake", "schedule" + ) + assert row["status"] == "ok" and calls[-1] == "np" + _pause(pg_settings, False) + row = runs.run_job(_sched(pg_settings, spec), "fake", "schedule") + assert row["status"] == "ok" and calls[-1] == "schedule" + + +@needs_pg +def test_chain_after_a_run(pg_settings: Settings) -> None: + feed = JobSpec(name="a", run=lambda ctx: {}, cadence=lambda fs, n: 60, pausable=True, chain=("b",)) + other = JobSpec(name="b", run=lambda ctx: {}, cadence=lambda fs, n: 60, pausable=True) + sch = Scheduler(pg_settings, build="pytest", jobs={"a": feed, "b": other}) + runs.run_job(sch, "a", "schedule") + assert sch._next_trigger.get("b") == "chain" # moved to now with trigger chain + + +@needs_pg +def test_settings_save_audits_and_notifies(pg_settings: Settings) -> None: + from packetstation import db + from packetstation.domain.projects import Who + + with db.connect_as("worker", pg_settings, autocommit=True) as listener: + listener.execute("LISTEN ps_settings") + with app_conn(pg_settings) as conn: + who = Who(user_id=None, session_id=None, role_snapshot="admin", ip=None) + new = config.save(conn, who=who, patch={"interval_shop_minutes": 7}) + assert new.interval_shop_minutes == 7 + row = conn.execute("SELECT value, version FROM settings WHERE key = 'feed'").fetchone() + assert row[0]["interval_shop_minutes"] == 7 + audit = conn.execute( + "SELECT detail FROM audit_events WHERE action = 'feed.settings_changed' ORDER BY id DESC LIMIT 1" + ).fetchone() + assert audit[0]["changed"]["interval_shop_minutes"] == {"from": 5, "to": 7} + notes = list(listener.notifies(timeout=5, stop_after=1)) + assert notes and notes[0].channel == "ps_settings" diff --git a/tests/server/test_schema.py b/tests/server/test_schema.py index cdaecb1..f4e4273 100644 --- a/tests/server/test_schema.py +++ b/tests/server/test_schema.py @@ -247,7 +247,7 @@ def test_v_open_cos(pg_settings: Settings) -> None: conn.execute("INSERT INTO co_required_depts (co_id, dept_code) VALUES (%s, 'WH'), (%s, '03')", (co, co)) conn.execute(verify, (co, "WH", ids["user"], ids["sig"])) held = new_co(conn, pk, 2, "visual", "pending", hold=True) - new_co(conn, pk, 3, "drawing", "pending", hold=False) + new_co(conn, pk, 3, "manual", "pending", hold=False) # 0004: one pending feed CO, visual or drawing rejected = new_co(conn, pk, 4, "manual", "rejected", hold=True) conn.execute("INSERT INTO co_required_depts (co_id, dept_code) VALUES (%s, '03')", (rejected,)) rows = conn.execute( @@ -276,6 +276,8 @@ def test_one_pending_visual_co_per_packet(pg_settings: Settings) -> None: new_co(conn, ids["packet"], 3, "visual", "approved") with pytest.raises(psycopg.errors.UniqueViolation): new_co(conn, ids["packet"], 4, "visual", "pending") + with pytest.raises(psycopg.errors.UniqueViolation): + new_co(conn, ids["packet"], 4, "drawing", "pending") # 0004: drawing-origin counts too with pytest.raises(psycopg.errors.UniqueViolation): new_co(conn, ids["packet"], 2, "manual", "approved") @@ -367,3 +369,99 @@ def test_app_can_write_every_table_and_read_every_view(pg_settings: Settings) -> for priv in ("SELECT", "INSERT", "UPDATE", "DELETE"): allowed = one(conn, "SELECT has_table_privilege('ps_app', %s, %s)", (view, priv)) assert allowed is (priv == "SELECT"), (view, priv) + + +FEED_TABLES = [ + "visual_batches", + "visual_rows", + "visual_materials", + "visual_orders", + "visual_order_links", + "visual_labor", + "visual_row_changes", + "visual_material_changes", + "visual_ignored_wos", + "visual_resource_map", + "drawing_lookups", + "job_requests", + "job_runs", + "job_schedule", + "worker_heartbeat", +] + + +def test_feed_tables_and_app_grants(pg_settings: Settings) -> None: + with db.connect_as("migrate", pg_settings, autocommit=True) as conn: + names = [r[0] for r in conn.execute("SELECT name FROM schema_migrations ORDER BY name").fetchall()] + assert "0004_feed.sql" in names + with app_conn(pg_settings) as conn: + for table in FEED_TABLES: + for priv in ("SELECT", "INSERT", "UPDATE", "DELETE"): + assert one(conn, "SELECT has_table_privilege('ps_app', %s, %s)", (table, priv)) is True, (table, priv) + jobs = [r[0] for r in conn.execute("SELECT job FROM job_schedule ORDER BY job").fetchall()] + assert jobs == ["drawing-lookup", "housekeeping", "visual-feed"] + assert one(conn, "SELECT target FROM visual_resource_map WHERE resource = 'WELD-PART'") == "03" + assert one(conn, "SELECT target FROM visual_resource_map WHERE resource = 'Electical'") == "14" + assert one(conn, "SELECT count(*) FROM visual_resource_map WHERE confirmed") == 0 + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute("INSERT INTO visual_resource_map (resource, target) VALUES ('X', 'nowhere')") + + +def test_feed_columns_are_text_ids(pg_settings: Settings) -> None: + with app_conn(pg_settings) as conn: + cols = dict( + conn.execute( + "SELECT table_name || '.' || column_name, data_type FROM information_schema.columns" + " WHERE table_schema = 'public' AND column_name IN ('wo_base', 'lot', 'split', 'sub', 'seq', 'piece'," + " 'part', 'cust_order_id', 'dwg')" + ).fetchall() + ) + for name, kind in cols.items(): + col = name.split(".")[1] + if col in ("seq", "piece"): + assert kind == "integer", name + else: + assert kind == "text", name + for t, c in ( + ("packets", "visual_cancelled_at"), + ("packets", "wo_gone_at"), + ("packet_work_orders", "rev"), + ("projects", "order_ref"), + ("units", "customer_visual"), + ("units", "order_pin"), + ): + assert ( + one( + conn, + "SELECT count(*) FROM information_schema.columns WHERE table_name = %s AND column_name = %s", + (t, c), + ) + == 1 + ), (t, c) + + +def test_two_operations_of_one_work_order_on_one_packet(pg_settings: Settings) -> None: + """Amendment 1: the key includes seq; hand rows (seq NULL) stay one per key.""" + with app_conn(pg_settings) as conn: + ids = seed(conn) + for seq in (10, 20): + conn.execute( + "INSERT INTO packet_work_orders (packet_id, wo_base, lot, split, sub, seq) " + "VALUES (%s, 'L900-02', '1', '0', '0', %s)", + (ids["packet"], seq), + ) + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute( + "INSERT INTO packet_work_orders (packet_id, wo_base, lot, split, sub, seq) " + "VALUES (%s, 'L900-02', '1', '0', '0', 20)", + (ids["packet"],), + ) + conn.execute( + "INSERT INTO packet_work_orders (packet_id, wo_base, lot, split, sub) VALUES (%s, 'H1', '1', '0', '0')", + (ids["packet"],), + ) + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute( + "INSERT INTO packet_work_orders (packet_id, wo_base, lot, split, sub) VALUES (%s, 'H1', '1', '0', '0')", + (ids["packet"],), + )