Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

server/packetstation/domain/assets.py

@@ -18,6 +18,7 @@ from pathlib import Path
 from typing import Any, BinaryIO
 
 from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
 
 from .. import audit
 from ..api.errors import ApiError
@@ -69,7 +70,35 @@ def path_of(settings: Settings, key: str) -> Path:
 
 
 def pdf_pages(path: Path) -> int:
-    """Page count through pypdf; 422 bad_pdf when it does not parse or has no pages."""
+    """Page count through pypdfium2, or pypdf when pypdfium2 is missing or cannot read the file.
+
+    422 bad_pdf when neither parses it or it has no pages.
+    """
+    n = _pages_pdfium(path)
+    if n is None:
+        n = _pages_pypdf(path)
+    if n < 1:
+        raise ApiError(422, "bad_pdf", "That PDF has no pages.")
+    return n
+
+
+def _pages_pdfium(path: Path) -> int | None:
+    try:
+        import pypdfium2
+    except ImportError:
+        return None
+    try:
+        doc = pypdfium2.PdfDocument(str(path))
+    except Exception as e:  # any pdfium failure falls back to pypdf
+        log.info("pypdfium2 could not read the PDF (%s); trying pypdf", type(e).__name__)
+        return None
+    try:
+        return len(doc)
+    finally:
+        doc.close()
+
+
+def _pages_pypdf(path: Path) -> int:
     from pypdf import PdfReader
     from pypdf.errors import PyPdfError
 
@@ -78,8 +107,6 @@ def pdf_pages(path: Path) -> int:
     except (PyPdfError, ValueError, KeyError, TypeError, OSError, IndexError, AttributeError) as e:
         log.info("upload is not a readable PDF: %s", type(e).__name__)
         raise ApiError(422, "bad_pdf", "That PDF could not be read.") from None
-    if n < 1:
-        raise ApiError(422, "bad_pdf", "That PDF has no pages.")
     return n
 
 
@@ -115,6 +142,40 @@ def _fsync_dir(path: Path) -> None:
         os.close(fd)
 
 
+def _existing(conn: Any, sha: bytes, size: int) -> dict[str, Any] | None:
+    """The stored asset with the same bytes (same sha256 and size), oldest first."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(
+            "SELECT * FROM assets WHERE sha256 = %s AND size = %s ORDER BY uploaded_at LIMIT 1", (sha, size)
+        ).fetchone()
+    return dict(r) if r else None
+
+
+def _place(settings: Settings, tmp: Path) -> tuple[uuid.UUID, str, Path]:
+    """Rename a fsynced file to ASSETS_DIR/<id[:2]>/<id>, then fsync the directory. Returns (id, key, path)."""
+    aid = uuid.uuid4()
+    key = storage_key(aid)
+    final = path_of(settings, key)
+    final.parent.mkdir(parents=True, exist_ok=True)
+    os.replace(tmp, final)
+    _fsync_dir(final.parent)
+    return aid, key, final
+
+
+def _insert(conn: Any, values: tuple[Any, ...]) -> dict[str, Any]:
+    """Insert one assets row from (id, sha256, size, mime, name, key, source, source_ref, pages, by, packet_id)."""
+    v = list(values)
+    if v[7] is not None and not isinstance(v[7], Jsonb):
+        v[7] = Jsonb(v[7])
+    with conn.cursor(row_factory=dict_row) as cur:
+        row = cur.execute(
+            "INSERT INTO assets (id, sha256, size, mime, original_name, storage_key, source, source_ref, pages,"
+            " uploaded_by, packet_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING *",
+            v,
+        ).fetchone()
+    return dict(row)
+
+
 def store_upload(
     conn: Any,
     settings: Settings,
@@ -166,29 +227,15 @@ def store_upload(
         if mime is None or size == 0:
             raise ApiError(415, "unsupported_type", "Only PDF, PNG and JPEG files can be stored.")
         sha = digest.digest()
-        with conn.cursor(row_factory=dict_row) as cur:
-            existing = cur.execute(
-                "SELECT * FROM assets WHERE sha256 = %s AND size = %s ORDER BY uploaded_at LIMIT 1", (sha, size)
-            ).fetchone()
+        existing = _existing(conn, sha, size)
         if existing is not None:
-            return public(dict(existing))
+            return public(existing)
         pages = pdf_pages(tmp) if mime == PDF else None
-        aid = uuid.uuid4()
-        key = storage_key(aid)
-        final = path_of(settings, key)
-        final.parent.mkdir(parents=True, exist_ok=True)
-        os.replace(tmp, final)
+        aid, key, placed = _place(settings, tmp)
         keep = True
-        placed = final
-        _fsync_dir(final.parent)
         name = safe_name(original_name)
         with conn.transaction():
-            with conn.cursor(row_factory=dict_row) as cur:
-                row = cur.execute(
-                    "INSERT INTO assets (id, sha256, size, mime, original_name, storage_key, source, source_ref, pages,"
-                    " uploaded_by, packet_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING *",
-                    (aid, sha, size, mime, name, key, source, source_ref, pages, user, packet_id),
-                ).fetchone()
+            row = _insert(conn, (aid, sha, size, mime, name, key, source, source_ref, pages, user, packet_id))
             audit.append(
                 conn,
                 action="asset.uploaded",
@@ -213,6 +260,69 @@ def store_upload(
                 tmp.unlink()
 
 
+def store_file(
+    conn: Any,
+    settings: Settings,
+    path: Path,
+    *,
+    mime: str = PDF,
+    original_name: Any,
+    source: str,
+    source_ref: Any = None,
+    uploaded_by: Any = None,
+    packet_id: int | None = None,
+) -> dict[str, Any]:
+    """Store a file the server wrote itself (the worker's M-Files download). Returns the full assets row.
+
+    The file is consumed: renamed into ASSETS_DIR when its bytes are new, deleted when the same bytes are already
+    stored (same sha256 and size; that row comes back). Write it under ASSETS_DIR (ASSETS_DIR/tmp) so the rename
+    stays on one file system. It is hashed in chunks, its first bytes must match `mime`, and it is fsynced before
+    the rename. The row carries one extra key, `reused`. No audit row: the caller writes its own.
+    Raises 413 too_large, 415 unsupported_type, 422 bad_pdf.
+    """
+    path = Path(path)
+    limit = int(settings.upload_max_bytes)
+    placed: Path | None = None
+    done = False
+    try:
+        digest = hashlib.sha256()
+        size = 0
+        head = b""
+        with path.open("r+b") as f:
+            while True:
+                chunk = f.read(CHUNK)
+                if not chunk:
+                    break
+                if len(head) < HEAD:
+                    head += chunk[: HEAD - len(head)]
+                size += len(chunk)
+                if size > limit:
+                    raise ApiError(413, "too_large", f"The file is over the {limit // (1024 * 1024)} MB limit.")
+                digest.update(chunk)
+            os.fsync(f.fileno())
+        if size == 0 or sniff(head) != mime:
+            raise ApiError(415, "unsupported_type", f"The file is not {EXT.get(mime, mime)}.")
+        sha = digest.digest()
+        existing = _existing(conn, sha, size)
+        if existing is not None:
+            return {**existing, "reused": True}
+        pages = pdf_pages(path) if mime == PDF else None
+        aid, key, placed = _place(settings, path)
+        name = safe_name(original_name)
+        with conn.transaction():
+            row = _insert(conn, (aid, sha, size, mime, name, key, source, source_ref, pages, uploaded_by, packet_id))
+        done = True
+        return {**row, "reused": False}
+    except BaseException:
+        if placed is not None and not done:  # the row was not written: do not leave an orphan file
+            with contextlib.suppress(OSError):
+                placed.unlink()
+        raise
+    finally:
+        with contextlib.suppress(OSError):
+            path.unlink()
+
+
 def open_blob(settings: Settings, row: dict[str, Any]) -> Path:
     """The stored file of an asset row; 404 when it is missing on disk."""
     p = path_of(settings, str(row["storage_key"]))