Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

frontend/src/api/feed.ts

@@ -0,0 +1,15 @@
+// The VISUAL feed endpoints for Clerical (phase 3 contract, api/feed.py).
+import { api } from "./client";
+import { asList, qs } from "./query";
+import type { FeedOverview, IgnoredWo } from "./types";
+
+export type FeedJob = "visual-feed" | "drawing-lookup";
+
+export const feedApi = {
+  overview: (signal?: AbortSignal) => api.get<FeedOverview>("/api/feed/overview", signal),
+  run: (job?: FeedJob) => api.post<{ requested: boolean; pending: number }>("/api/feed/run", job ? { job } : {}),
+  pullIn: (id: number, reason: string) =>
+    api.post<IgnoredWo>(`/api/feed/ignored/${encodeURIComponent(String(id))}/pull-in`, { reason }),
+  changes: async (opts: { batch_id?: number; limit?: number } = {}, signal?: AbortSignal): Promise<unknown[]> =>
+    asList<unknown>(await api.get<unknown>(`/api/feed/changes${qs({ batch_id: opts.batch_id, limit: opts.limit ?? 200 })}`, signal), "changes"),
+};

frontend/src/api/jobs.ts

@@ -0,0 +1,30 @@
+// The worker's jobs (phase 3 contract, api/jobs.py) and the Admin feed settings (api/feed_settings.py).
+import { api } from "./client";
+import { asList, qs } from "./query";
+import type { FeedSettings, FeedSettingsAnswer, JobInfo, JobRun, JobsAnswer, ResourceMapRow, WorkerStatus } from "./types";
+
+const enc = encodeURIComponent;
+
+/** GET /api/jobs may come as {jobs, heartbeat} or as a bare list. Both give the same answer. */
+function asJobs(raw: unknown): JobsAnswer {
+  const jobs = asList<JobInfo>(raw, "jobs");
+  const hb = raw && typeof raw === "object" && !Array.isArray(raw) ? ((raw as { heartbeat?: WorkerStatus | null }).heartbeat ?? null) : null;
+  return { jobs, heartbeat: hb };
+}
+
+export const jobsApi = {
+  list: async (signal?: AbortSignal): Promise<JobsAnswer> => asJobs(await api.get<unknown>("/api/jobs", signal)),
+  runs: async (job: string, limit = 10, signal?: AbortSignal): Promise<JobRun[]> =>
+    asList<JobRun>(await api.get<unknown>(`/api/jobs/${enc(job)}/runs${qs({ limit })}`, signal), "runs"),
+  run: (job: string) => api.post<{ requested: boolean; pending: number }>(`/api/jobs/${enc(job)}/run`, {}),
+};
+
+export const feedAdminApi = {
+  settings: (signal?: AbortSignal) => api.get<FeedSettingsAnswer>("/api/admin/feed/settings", signal),
+  saveSettings: (patch: Partial<FeedSettings>) => api.patch<FeedSettingsAnswer>("/api/admin/feed/settings", patch),
+  resourceMap: async (signal?: AbortSignal): Promise<ResourceMapRow[]> =>
+    asList<ResourceMapRow>(await api.get<unknown>("/api/admin/feed/resource-map", signal), "resources"),
+  mapResource: (resource: string, target: string) =>
+    api.put<ResourceMapRow>(`/api/admin/feed/resource-map/${enc(resource)}`, { target }),
+  unmapResource: (resource: string) => api.del<null>(`/api/admin/feed/resource-map/${enc(resource)}`),
+};

frontend/src/api/packets.ts

@@ -43,4 +43,29 @@ export const packetsApi = {
   setVclosed: (id: number, closed: boolean, extra: Guarded = {}) =>
     api.post<PacketRead>(`${base(id)}/vclosed`, { closed, ...extra }),
   databook: (id: number) => api.post<PacketRead>(`${base(id)}/databook`, {}),
+
+  /* Phase 3: VISUAL packets. */
+  revApprove: (id: number, body: { reason: string } & Guarded) => api.post<PacketRead>(`${base(id)}/rev-approve`, body),
+  handDrawing: (id: number, body: { asset_id: string; reason: string } & Guarded) =>
+    api.post<PacketRead>(`${base(id)}/hand/drawing`, body),
+  /** Multipart cannot carry JSON, so the signature rides as sign_factor and sign_secret. */
+  handBom: (id: number, file: File, reason: string, extra: Guarded = {}) => {
+    const fd = new FormData();
+    fd.append("file", file, file.name);
+    fd.append("reason", reason);
+    if (extra.sign) {
+      fd.append("sign_factor", extra.sign.factor);
+      fd.append("sign_secret", extra.sign.secret);
+    }
+    if (extra.it_reason) fd.append("it_reason", extra.it_reason);
+    return request<PacketRead>(`${base(id)}/hand/bom`, { method: "POST", body: fd });
+  },
+  removeHandBom: (id: number, extra: Guarded = {}) => api.delWithBody<PacketRead>(`${base(id)}/hand/bom`, extra),
+
+  /* Change orders. */
+  coApprove: (id: number, no: number, extra: Guarded = {}) => api.post<PacketRead>(`${base(id)}/change-orders/${no}/approve`, extra),
+  coReject: (id: number, no: number, body: { reason: string } & Guarded) =>
+    api.post<PacketRead>(`${base(id)}/change-orders/${no}/reject`, body),
+  coHold: (id: number, no: number, hold: boolean, extra: Guarded = {}) =>
+    api.post<PacketRead>(`${base(id)}/change-orders/${no}/hold`, { hold, ...extra }),
 };

frontend/src/api/projects.ts

@@ -12,6 +12,8 @@ export interface ProjectFields {
   due: string | null;
   status: Project["status"];
   notes: string | null;
+  order_pin?: string | null;
+  order_pin_reason?: string | null;
 }
 
 export interface UnitFields {
@@ -19,6 +21,8 @@ export interface UnitFields {
   due: string | null;
   status: Unit["status"];
   notes: string | null;
+  order_pin?: string | null;
+  order_pin_reason?: string | null;
 }
 
 const pid = (id: number) => `/api/projects/${encodeURIComponent(String(id))}`;

frontend/src/api/types.ts

@@ -72,6 +72,9 @@ export interface Unit {
   packets?: StageCounts;
   version: number;
   project?: ProjectRef;
+  order_ref?: OrderRef | null;
+  order_pin?: string | null;
+  order_pin_reason?: string | null;
 }
 
 /** One row of GET /api/projects. GET /api/projects/{id} adds units and aliases. */
@@ -91,6 +94,11 @@ export interface Project {
   packets?: StageCounts;
   aliases?: Array<string | { alias: string; kind?: string }>;
   version: number;
+  /** Phase 3: cached VISUAL orders for the project and its bases. */
+  orders?: VisualOrder[];
+  order_ref?: OrderRef | null;
+  order_pin?: string | null;
+  order_pin_reason?: string | null;
 }
 
 export interface ProjectCreate {
@@ -161,6 +169,12 @@ export interface PacketCard {
   blocked: Record<string, string[]>;
   /** Not in the contract's card shape. Shown when the server sends it. */
   wo_count?: number;
+  /** Phase 3: the raw flag kinds (drawing_needed, rev_mismatch, ...). */
+  flag_kinds?: string[];
+  /** Phase 3: a feed change order waits for Clerical. */
+  co_pending?: boolean;
+  /** Phase 3: every leg is closed in VISUAL (observed). */
+  visual_closed?: boolean;
 }
 
 export interface PacketList {
@@ -174,6 +188,8 @@ export interface PacketListQuery {
   project_id?: number;
   q?: string;
   flag?: string;
+  /** "pending": packets with a pending feed change order. */
+  co?: string;
   limit?: number;
 }
 
@@ -282,7 +298,9 @@ export interface PacketRead {
   source: Record<string, unknown>;
   drawing_state: string;
   flags: string[];
-  visual: Record<string, unknown>;
+  visual: VisualInfo;
+  /** Phase 3: what the feed last observed in VISUAL. */
+  visual_state?: VisualState | null;
   removed: { at: string; by: WhoRef | null; reason: string | null } | null;
   work_orders: WorkOrder[];
   pages: PacketPage[];
@@ -295,7 +313,7 @@ export interface PacketRead {
   wh: Record<string, unknown> | null;
   qc: Record<string, unknown> | null;
   qc_history: unknown[];
-  change_orders: unknown[];
+  change_orders: ChangeOrder[];
   open_cos: number[];
   open_shorts: number;
   docs: PacketDoc[];
@@ -394,3 +412,245 @@ export interface Guarded {
   it_reason?: string;
   sign?: SignBody;
 }
+
+/* ---------- Phase 3: VISUAL feed on the packet ---------- */
+
+/** visual.appr: the last M-Files answer for the drawing. */
+export interface VisualAppr {
+  ok?: boolean;
+  rev?: number | string | null;
+  vrev?: string | null;
+  name?: string | null;
+  note?: string | null;
+  base?: string | null;
+  draft_rev?: number | null;
+  draft_where?: string | null;
+  in_drafting?: boolean;
+  at?: string | null;
+}
+
+/** visual.manual_dwg and visual.manual_bom. `by` may be a who reference or a name. */
+export interface HandUpload {
+  asset_id?: string;
+  name?: string | null;
+  reason?: string | null;
+  by?: WhoRef | string | null;
+  at?: string | null;
+  vis_rev?: string | null;
+  lines?: number;
+}
+
+/** One folded operation (MFG, inspection and the like) kept on the packet for reference. */
+export interface FoldedOp {
+  key?: string | { base?: string; lot?: string; split?: string; sub?: string };
+  wo?: string;
+  wo_base?: string;
+  lot?: string;
+  seq?: number;
+  op?: string;
+  tgt?: number | string;
+}
+
+/** The packet's `visual` jsonb. Every key is optional; hand drafts have {}. */
+export interface VisualInfo {
+  appr?: VisualAppr;
+  /** Set by the read model from drawing_lookups when M-Files has a staged copy. */
+  staged_rev?: { rev: number | string; name: string | null };
+  partspecs?: string[];
+  folded?: FoldedOp[];
+  manual_dwg?: HandUpload;
+  manual_bom?: HandUpload;
+  rev_ok?: { rev: string | number; vis_rev?: string | null; by?: WhoRef | string | null; at?: string | null; reason?: string | null };
+  rev_confirmed?: boolean;
+  [key: string]: unknown;
+}
+
+export interface VisualLeg {
+  key?: string;
+  seq?: number;
+  st?: string;
+}
+
+export interface VisualState {
+  legs: VisualLeg[];
+  ticket_hours: number | string | null;
+  order_ref: OrderRef | null;
+  observed_at: string | null;
+  closed_at: string | null;
+  cancelled_at: string | null;
+}
+
+/** change_orders[].vis for feed change orders. */
+export interface CoVis {
+  from_dwg?: string | null;
+  from_rev?: string | null;
+  to_dwg?: string | null;
+  to_rev?: string | null;
+  dwg_change?: boolean;
+  changes?: string[];
+  [key: string]: unknown;
+}
+
+export interface ChangeOrderItem {
+  old_page: number | null;
+  old_dwg: string | null;
+  old_rev: string | null;
+  new_page: number | null;
+  new_dwg: string | null;
+  new_rev: string | null;
+  added?: boolean;
+}
+
+export interface ChangeOrder {
+  id: number;
+  co_no: number;
+  origin: string;
+  review: string;
+  minor: boolean;
+  hold: boolean;
+  ecn: string | null;
+  reason: string | null;
+  issued_at: string | null;
+  issued_by: WhoRef | null;
+  approved_at?: string | null;
+  approved_by?: WhoRef | null;
+  rejected_at?: string | null;
+  rejected_by?: WhoRef | null;
+  rejected_reason?: string | null;
+  items: ChangeOrderItem[];
+  required: string[];
+  verified: Record<string, { by: WhoRef | null; at: string; note: string | null }>;
+  vis?: CoVis | null;
+  waiting?: { dwg: string; rev: string; since?: string | null; err?: string | null } | null;
+  staged?: { asset_id: string; name: string | null; rev: string; dwg: string; sha256?: string; pages?: number | null } | null;
+  target_fp?: string | null;
+  row_version?: number;
+}
+
+/* ---------- Phase 3: orders ---------- */
+
+/** One cached VISUAL customer order. */
+export interface VisualOrder {
+  cust_order_id: string;
+  status: string | null;
+  customer_id?: string | null;
+  customer_name: string | null;
+  customer_po_ref: string | null;
+  order_date: string | null;
+  desired_ship_date: string | null;
+  promise_date: string | null;
+  /** "project" or "base" link, when the server says. */
+  kind?: string;
+  key?: string;
+  top_jobs?: string[];
+}
+
+/** The chosen order stored on projects and units. */
+export interface OrderRef {
+  cust_order_id: string;
+  via: string;
+  status: string | null;
+  customer_name: string | null;
+  customer_po_ref: string | null;
+  order_date: string | null;
+  desired_ship_date: string | null;
+  promise_date: string | null;
+  chosen_at?: string | null;
+}
+
+/* ---------- Phase 3: jobs and the feed ---------- */
+
+export interface JobRun {
+  id: number;
+  job: string;
+  trigger: string;
+  requested_by?: string[];
+  started_at: string;
+  finished_at: string | null;
+  status: string;
+  counts: Record<string, unknown>;
+  errors: string[];
+  build_version?: string | null;
+  visual_target?: string | null;
+  host?: string | null;
+}
+
+export interface WorkerStatus {
+  at: string | null;
+  build: string | null;
+  running: string[];
+  stale: boolean;
+}
+
+export interface IgnoredWo {
+  id: number;
+  wo_base: string;
+  lot: string;
+  split: string;
+  sub: string;
+  rel: string | null;
+  project: string | null;
+  unit_number: string | null;
+  res: string | null;
+  dwg: string | null;
+  pulled_in_at: string | null;
+}
+
+export interface FeedOverview {
+  status: { paused: boolean; next_run_at: string | null; worker: WorkerStatus | null };
+  last_run: JobRun | null;
+  runs: JobRun[];
+  drawing_runs: JobRun[];
+  ignored: IgnoredWo[];
+  unmapped: Array<{ resource: string; rows: number }>;
+  projects_without_names: Array<{ id: number; number: string; units: number; packets: number }>;
+  golive_date: string | null;
+  departments: string[];
+}
+
+/** One row of GET /api/jobs. The contract leaves the shape open. */
+export interface JobInfo {
+  job: string;
+  next_run_at: string | null;
+  interval_seconds: number | null;
+  paused: boolean;
+  last_run: JobRun | null;
+}
+
+export interface JobsAnswer {
+  jobs: JobInfo[];
+  heartbeat: WorkerStatus | null;
+}
+
+/** The feed settings row (feed.config.FeedSettings). */
+export interface FeedSettings {
+  interval_shop_minutes: number;
+  interval_off_minutes: number;
+  shop_days: number[];
+  shop_start: string;
+  shop_end: string;
+  window_back_days: number;
+  window_ahead_days: number;
+  golive_date: string | null;
+  departments: string[];
+  drawing_recheck_hours: number;
+  drawing_budget: number;
+  drawing_interval_minutes: number;
+  labor_tickets: boolean;
+  auto_release: boolean;
+  auto_release_max: number;
+  paused: boolean;
+}
+
+export interface FeedSettingsAnswer extends FeedSettings {
+  next_run_at?: string | null;
+  runs?: JobRun[];
+}
+
+export interface ResourceMapRow {
+  resource: string;
+  target: string | null;
+  confirmed: boolean;
+  rows_last_run: number | null;
+  updated_at: string | null;
+}