Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
2026-10-16 14:45:38 -06:00
parent 16950055e3
commit 591fe90076
103 changed files with 16063 additions and 151 deletions
@@ -0,0 +1,62 @@
+"""Redaction for job errors and log lines: no password, key or token ever reaches job_runs or a log."""
+
+from __future__ import annotations
+
+import os
+import re
+
+MAX_ERRORS = 20
+MAX_ERROR_CHARS = 500
+
+# Environment variables whose values are secrets when set (MFilesKit reads MFILES_PASSWORD; VisualKit may read
+# VISUAL_PASSWORD). They are read only to strip them from text, never logged.
+SECRET_ENV = ("VISUAL_PASSWORD", "MFILES_PASSWORD", "PGPASSWORD", "LDAP_PASSWORD")
+SECRET_FILE_ENV = ("VISUAL_PASSWORD_FILE", "MFILES_PASSWORD_FILE")
+
+_PATTERNS = (
+    re.compile(r"(?i)\b(password|passwd|pwd|secret|token|api[_-]?key)\s*[=:]\s*(\"[^\"]*\"|'[^']*'|[^\s;,&]+)"),
+    re.compile(r"(?i)\bauthorization\s*:\s*[^\r\n]*"),
+    re.compile(r"(?i)\bx-authentication\s*:\s*[^\r\n]*"),
+    re.compile(r"(?i)(\w+://[^:/\s@]+:)([^@\s]+)(@)"),
+)
+
+
+def _secrets() -> list[str]:
+    out = []
+    for name in SECRET_ENV:
+        v = os.environ.get(name)
+        if v and len(v) >= 4:
+            out.append(v)
+    for name in SECRET_FILE_ENV:
+        path = os.environ.get(name)
+        if not path:
+            continue
+        try:
+            with open(path, encoding="utf-8") as f:
+                v = f.read().rstrip("\r\n")
+        except OSError:
+            continue
+        if v and len(v) >= 4:
+            out.append(v)
+    return out
+
+
+def redact(text: str) -> str:
+    """Strip anything that looks like a credential, plus the VISUAL and M-Files passwords if set."""
+    s = str(text)
+    for secret in _secrets():
+        s = s.replace(secret, "[redacted]")
+    s = _PATTERNS[0].sub(lambda m: f"{m.group(1)}=[redacted]", s)
+    s = _PATTERNS[1].sub("Authorization: [redacted]", s)
+    s = _PATTERNS[2].sub("X-Authentication: [redacted]", s)
+    s = _PATTERNS[3].sub(lambda m: f"{m.group(1)}[redacted]{m.group(3)}", s)
+    return s
+
+
+def clip(errors: list[str]) -> list[str]:
+    """At most MAX_ERRORS redacted strings of at most MAX_ERROR_CHARS each."""
+    out = []
+    for e in errors[:MAX_ERRORS]:
+        r = redact(e)
+        out.append(r if len(r) <= MAX_ERROR_CHARS else r[: MAX_ERROR_CHARS - 3] + "...")
+    return out