Files
PacketStation/server/packetstation/feed/fingerprint.py
T
spencermandClaude Fable 5.1 591fe90076
ci / frontend (push) Successful in 30s
ci / server (push) Successful in 4m12s
ci / image (push) Successful in 46s
ci / e2e (push) Successful in 1m24s
Phase 3: VISUAL feed and M-Files drawings in the worker
Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>
2026-10-16 14:45:38 -06:00

211 lines
6.9 KiB
Python

"""The candidate fingerprint: what VISUAL says about a packet, hashed per part, to decide update, CO or nothing.
Parts (each the sha256 hex of its canonical JSON):
- major: `legs` (the sorted leg keys), `qty` (per leg), `rev`, `drawing` (upper-cased, spaces removed), `bom`
(sorted `[item, part_id, qty]`, the old matSig);
- minor: `tgt` (target hours per leg), `want` (the want date).
Never in it: rel, folded, partspecs, title, statuses, issued, whs, loc, orders, labor, dept, op. A candidate whose
materials are unknown (`bom is None`) has no `bom` part; `classify` skips a part the new side lacks and `merge`
carries the old one forward, so a failed materials read never looks like a BOM change.
"""
from __future__ import annotations
import hashlib
import json
import unicodedata
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import date, datetime
from decimal import Decimal
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from .candidates import Candidate
MAJOR = ("legs", "qty", "rev", "drawing", "bom")
MINOR = ("tgt", "want")
PARTS = MAJOR + MINOR
# ---------------------------------------------------------------- normalisation
def norm_text(value: Any) -> str | None:
"""NFC, CRLF and CR to LF, trimmed; '' and None both None."""
if value is None:
return None
s = value if isinstance(value, str) else str(value)
s = unicodedata.normalize("NFC", s.replace("\r\n", "\n").replace("\r", "\n")).strip()
return s or None
def norm_id(value: Any) -> str | None:
"""An ID (base, lot, split, sub, part, drawing): normalised text, upper-cased. Stays text."""
s = norm_text(value)
return s.upper() if s is not None else None
def norm_decimal(value: Any) -> str | None:
"""A number as text with no exponent and no trailing zeros: 2, 2.0 and Decimal('2.00') are all '2'."""
if value is None or value == "":
return None
d = value if isinstance(value, Decimal) else Decimal(str(value))
if d.is_nan() or d.is_infinite():
return None
if d == 0:
return "0"
s = format(d.normalize(), "f")
if "." in s:
s = s.rstrip("0").rstrip(".")
return s
def normalise(value: Any) -> Any:
"""The canonical form of a JSON-like value."""
if value is None:
return None
if isinstance(value, bool):
return value
if isinstance(value, str):
return norm_text(value)
if isinstance(value, int | float | Decimal):
return norm_decimal(value)
if isinstance(value, datetime):
return value.isoformat()
if isinstance(value, date):
return value.isoformat()
if isinstance(value, Mapping):
return {str(k): normalise(v) for k, v in value.items()}
if isinstance(value, list | tuple | set | frozenset):
return [normalise(v) for v in value]
return norm_text(str(value))
def canonical(value: Any) -> bytes:
"""json.dumps(sort_keys, compact separators, ensure_ascii False) of the normalised value, UTF-8."""
return json.dumps(normalise(value), sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
def sha256_hex(value: Any) -> str:
return hashlib.sha256(canonical(value)).hexdigest()
def drawing_key(dwg: Any) -> str | None:
"""A drawing number for comparison: upper-cased, every whitespace removed."""
s = norm_text(dwg)
if s is None:
return None
s = "".join(s.split()).upper()
return s or None
def leg_key(base: Any, lot: Any, split: Any, sub: Any, seq: Any) -> str:
"""'BASE|LOT|SPLIT|SUB|SEQ' with the ID parts upper-cased (as text) and seq as an integer."""
parts = [norm_id(x) or "" for x in (base, lot, split, sub)]
return "|".join([*parts, str(int(seq)) if seq is not None and seq != "" else ""])
# ---------------------------------------------------------------- the fingerprint
@dataclass(frozen=True)
class Fingerprint:
full: str
parts: dict[str, str]
@dataclass(frozen=True)
class Change:
major: frozenset[str]
minor: frozenset[str]
@property
def changed(self) -> bool:
return bool(self.major or self.minor)
@property
def is_minor(self) -> bool:
return bool(self.minor) and not self.major
def part_values(c: Candidate) -> dict[str, Any]:
"""The normalised value of every part, before hashing (tests and the diff text read these)."""
legs = sorted(
(
[
norm_id(lg.key.base) or "",
norm_id(lg.key.lot) or "",
norm_id(lg.key.split) or "",
norm_id(lg.key.sub) or "",
int(lg.seq),
]
for lg in c.legs
),
key=lambda x: (x[0], x[1], x[2], x[3], x[4]),
)
qty: dict[str, Any] = {}
tgt: dict[str, Any] = {}
for lg in c.legs:
k = leg_key(lg.key.base, lg.key.lot, lg.key.split, lg.key.sub, lg.seq)
qty[k] = norm_decimal(lg.qty)
tgt[k] = norm_decimal(lg.tgt)
values: dict[str, Any] = {
"legs": legs,
"qty": qty,
"rev": norm_text(c.rev),
"drawing": drawing_key(c.drawing),
"tgt": tgt,
"want": c.want.isoformat() if c.want else None,
}
if c.bom is not None:
values["bom"] = bom_sig(c.bom)
return values
def bom_sig(bom: Any) -> list[list[Any]]:
"""The old matSig: sorted [item, part_id, qty] of every line (BomLine objects or dicts)."""
out: list[list[Any]] = []
for b in bom or []:
get = b.get if isinstance(b, Mapping) else lambda k, _b=b: getattr(_b, k, None)
out.append([norm_text(get("item")) or "", norm_id(get("part_id")) or "", norm_decimal(get("qty"))])
out.sort(key=lambda x: (x[0], x[1], x[2] or ""))
return out
def _full(parts: Mapping[str, str]) -> str:
return sha256_hex(dict(parts))
def compute(c: Candidate) -> Fingerprint:
parts = {name: sha256_hex(v) for name, v in part_values(c).items()}
return Fingerprint(full=_full(parts), parts=parts)
def merge(before: Mapping[str, str] | None, after: Fingerprint) -> Fingerprint:
"""Carry forward parts the new side does not know (the BOM when materials are unknown this run)."""
if not before:
return after
parts = dict(after.parts)
for name in PARTS:
if name not in parts and name in before:
parts[name] = str(before[name])
if parts == after.parts:
return after
return Fingerprint(full=_full(parts), parts=parts)
def classify(before: Mapping[str, str] | None, after: Mapping[str, str]) -> Change:
"""Which parts changed. A part the new side lacks (unknown this run) is skipped. No `before`: all changed."""
major: set[str] = set()
minor: set[str] = set()
for name in PARTS:
if name not in after:
continue
if before is not None and before.get(name) == after[name]:
continue
(major if name in MAJOR else minor).add(name)
return Change(frozenset(major), frozenset(minor))