Archive copy must succeed before a packet counts as filed (Gitea issue 57)

FILE checks the archive folder and copies the packet inside the filing
transaction. A failed check or copy rolls the filing back, so the packet
stays in its previous stage; the person filing gets 503 archive_failed,
and in its own transaction the server writes an archive.filing_blocked
audit row and raises the critical archive_failed condition to Admin for
that packet (who tried, folder, error). The next successful filing clears
it. The archive-copy job now only catches up filings made while the
archive was off. PLAN question 53, CHANGELOG, FEED and SECURITY updated.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/packetstation/api/packets.py

@@ -18,9 +18,9 @@ from psycopg.types.json import Jsonb
 from pydantic import BaseModel, Field
 
 from .. import audit, events, policy
+from ..domain import archive, packets, search, shop
 from ..domain import assets as asset_store
 from ..domain import ncr as ncrs
-from ..domain import packets, search, shop
 from ..domain import qc as qc_dom
 from ..domain.pipeline import (
     Expect,
@@ -543,6 +543,10 @@ TRANSITION_FIELDS: dict[policy.Action, tuple[str, ...]] = {
     A.FILE: ("note",),
     A.ADMIN_REOPEN: ("reason",),
 }
+MSG_ARCHIVE = (
+    "The packet could not be copied to the archive: {reason}. Nothing was filed, and Admin has been told."
+    " File it again once the archive works."
+)
 DEPT_ACTIONS = (A.RELEASE_TO_SHOP, A.ROUTE_FROM_REVIEW, A.QC_ACCEPT_AND_ROUTE)
 
 
@@ -591,29 +595,34 @@ def transition(
         else:
             del extra["stamp"]
     content = {"action": str(action), "packet_id": packet_id, "expect": body.expect.model_dump(), **extra}
-    with packet_write(
-        d,
-        p,
-        packet_id,
-        action,
-        request=request,
-        extra=extra,
-        it_reason=body.it_reason,
-        sign=body.sign,
-        expect=body.expect,
-        sign_content=content,
-        policy_principal=pp,
-    ) as w:
-        if action is A.ASSIGN:
-            ok = conn.execute(
-                "SELECT 1 FROM stations WHERE id = %s AND dept_code = %s AND active",
-                (extra["station_id"], w.state.dept),
-            ).fetchone()
-            if ok is None:
-                raise ApiError(422, "invalid_station", "Choose a station in this department.")
-            active_user(conn, extra["welder_id"], "welder")
-            shop.welder_check(conn, w.state.dept, str(extra["welder_id"]), int(extra["station_id"]))
-        apply_transition(w)
+    try:
+        with packet_write(
+            d,
+            p,
+            packet_id,
+            action,
+            request=request,
+            extra=extra,
+            it_reason=body.it_reason,
+            sign=body.sign,
+            expect=body.expect,
+            sign_content=content,
+            policy_principal=pp,
+        ) as w:
+            if action is A.ASSIGN:
+                ok = conn.execute(
+                    "SELECT 1 FROM stations WHERE id = %s AND dept_code = %s AND active",
+                    (extra["station_id"], w.state.dept),
+                ).fetchone()
+                if ok is None:
+                    raise ApiError(422, "invalid_station", "Choose a station in this department.")
+                active_user(conn, extra["welder_id"], "welder")
+                shop.welder_check(conn, w.state.dept, str(extra["welder_id"]), int(extra["station_id"]))
+            apply_transition(w)
+    except archive.FilingCopyFailed as e:
+        # The filing rolled back (Gitea issue 57); record it and tell Admin in a transaction of its own.
+        archive.record_filing_failure(conn, p, packet_id, e)
+        raise ApiError(503, "archive_failed", MSG_ARCHIVE.format(reason=e.message.rstrip("."))) from None
     return _read(d, pp, packet_id)
 
 

server/packetstation/domain/archive.py

@@ -6,9 +6,14 @@ A path that leaves the root is refused: empty, `..`, an absolute path elsewhere,
 the root, or a folder that does not exist. The check writes a probe file, reads it back and removes it. Nothing here
 creates folders or deletes anything in the archive.
 
-FILE inserts a job request for `archive-copy` in its own transaction. The worker job (worker/jobs/archive_copy.py)
-queues every unbroken seal with a data book that has no `archive_copies` row, then copies the pending and due failed
-rows. A failed copy never blocks filing: it is retried with a backoff and raised to Admin as `archive_failed`.
+FILE copies the packet into the archive inside its own transaction (Gitea issue 57, Spencer 2026-11-03): the copy
+must succeed before the packet counts as filed. A failed folder check or copy rolls the whole filing back, so the
+packet stays where it was and can be filed again. The failure is then recorded in its own transaction and raised to
+Admin as the `archive_failed` condition with subject `filing:<packet id>`, cleared when that packet files.
+
+The worker job (worker/jobs/archive_copy.py) copies the seals filed while the archive was off: it queues every
+unbroken seal with a data book that has no `archive_copies` row, then copies the pending and due failed rows with a
+backoff, and raises `archive_failed` (subject `archive`) to Admin.
 
 Folder layout under the archive folder: `<project>/<unit>/<data book name>-seal<seal id>/` with the data book PDF,
 the packet's files (drawing page files, packet documents, the Q.C. folder documents of its unit and project, MTRs)
@@ -45,6 +50,8 @@ JOB = "archive-copy"
 JOB_CHANNEL = "ps_job_request"
 EVENT = "archive_failed"
 EVENT_SUBJECT = "archive"
+FILING_SUBJECT = "filing:{packet_id}"
+FILING_TITLE = "Filing blocked: archive copy failed"
 PROBE_PREFIX = ".packetstation-probe-"
 MANIFEST = "manifest.json"
 DIR_MAX = 400
@@ -72,6 +79,16 @@ class CopyError(Exception):
     """One copy failed. The message goes into archive_copies.last_error and the audit row."""
 
 
+class FilingCopyFailed(Exception):
+    """FILE could not copy the packet into the archive. The filing rolls back; the message is for the person filing
+    and for Admin."""
+
+    def __init__(self, message: str, archive_dir: str | None):
+        super().__init__(message)
+        self.message = message
+        self.archive_dir = archive_dir
+
+
 @dataclass(frozen=True)
 class CheckResult:
     ok: bool
@@ -320,20 +337,104 @@ def request_copy(conn: Any, user_id: Any = None, note: str | None = None) -> Non
     conn.execute("SELECT ps_notify(%s, %s)", (JOB_CHANNEL, json.dumps({"job": JOB}, separators=(",", ":"))))
 
 
-def on_filed(conn: Any, settings: Settings, user_id: Any, packet_id: int) -> bool:
-    """FILE's hook: request a prompt copy when the archive is on and a folder is saved. Never raises; a failure
-    here only means the copy waits for the next scheduled run. Returns whether a request went in."""
-    if root_of(settings) is None:
-        return False
+def copy_at_filing(conn: Any, settings: Settings, packet_id: int, seal_id: int) -> dict[str, Any] | None:
+    """FILE's copy, in the filing transaction: check the folder, copy the seal, record the copy as done.
+    Returns {folder, files}, or None when the archive is off (ARCHIVE_ROOT unset or no folder saved).
+    Raises FilingCopyFailed, which rolls the filing back."""
+    root = root_of(settings)
+    if root is None:
+        return None
+    archive_dir = load(conn)[0]
+    if archive_dir is None:
+        return None
+    rel, result = check(root, archive_dir)
+    if not result.ok or rel is None:
+        raise FilingCopyFailed(f"The archive folder check failed: {result.message}", archive_dir)
+    base = root.joinpath(*rel.split("/"))
+    try:
+        folder, sha, files = copy_seal(conn, settings, base, rel, seal_id)
+    except CopyError as e:
+        raise FilingCopyFailed(str(e), archive_dir) from None
+    except Exception as e:  # anything unexpected blocks the filing the same way
+        log.exception("archive copy of packet %s at filing failed", packet_id)
+        raise FilingCopyFailed(f"{type(e).__name__}: {e}", archive_dir) from None
+    with conn.transaction():
+        conn.execute(
+            "INSERT INTO archive_copies (seal_id, packet_id, status, attempts, folder, manifest_sha256, done_at)"
+            " VALUES (%s, %s, 'done', 1, %s, %s, now())",
+            (seal_id, packet_id, folder, sha),
+        )
+        audit.append(
+            conn,
+            action="archive.copied",
+            subject_type="seal",
+            subject_id=str(seal_id),
+            packet_id=int(packet_id),
+            detail={"seal_id": seal_id, "folder": folder, "files": files, "manifest_sha256": sha, "source": "filing"},
+        )
+    from ..notify import resolve_event
+
+    resolve_event(
+        conn, EVENT, subject=FILING_SUBJECT.format(packet_id=packet_id), body="The packet was filed and copied."
+    )
+    return {"folder": folder, "files": files}
+
+
+def record_filing_failure(conn: Any, who: Any, packet_id: int, failure: FilingCopyFailed) -> None:
+    """After the filing rolled back: one `archive.filing_blocked` audit row and the Admin alert, in their own
+    transaction. Never raises; the person filing gets the refusal either way."""
+    from ..notify import raise_event
+
     try:
         with conn.transaction():
-            if load(conn)[0] is None:
-                return False
-            request_copy(conn, user_id, f"filed packet {packet_id}")
-        return True
+            row = conn.execute(
+                "SELECT pr.number, u.lot, p.drawing, p.status, p.dept_code FROM packets p"
+                " JOIN units u ON u.id = p.unit_id JOIN projects pr ON pr.id = u.project_id WHERE p.id = %s",
+                (packet_id,),
+            ).fetchone()
+            label = packet_label(*row[:3]) if row else f"packet {packet_id}"
+            user_id = getattr(who, "user_id", None)
+            name = conn.execute("SELECT display_name FROM users WHERE id = %s", (user_id,)).fetchone()
+            by = name[0] if name else None
+            status = str(row[3]) if row else None
+            dept = row[4] if row else None
+            detail = {
+                "packet_id": packet_id,
+                "packet_label": label,
+                "status": status,
+                "dept": dept,
+                "archive_dir": failure.archive_dir,
+                "error": failure.message,
+                "by": by,
+            }
+            audit.append(
+                conn,
+                action="archive.filing_blocked",
+                user_id=user_id,
+                session_id=getattr(who, "session_id", None),
+                subject_type="packet",
+                subject_id=str(packet_id),
+                packet_id=int(packet_id),
+                detail=detail,
+                ip=getattr(who, "ip", None),
+            )
+            where = f"{status or 'where it was'}{' in ' + dept if dept else ''}"
+            raise_event(
+                conn,
+                EVENT,
+                subject=FILING_SUBJECT.format(packet_id=packet_id),
+                title=f"{FILING_TITLE}: {label}",
+                body=(
+                    f"{by or 'Someone'} tried to file {label}, but the packet could not be copied to the archive,"
+                    f" so it was not filed. It stays {where} and can be filed again once the archive works.\n\n"
+                    f"Archive folder: {failure.archive_dir or '(none)'}\nError: {failure.message}"
+                ),
+                detail=detail,
+                severity="critical",
+                condition=True,
+            )
     except Exception:
-        log.exception("archive copy request for packet %s not made; the scheduled run picks it up", packet_id)
-        return False
+        log.exception("the blocked filing of packet %s was not recorded", packet_id)
 
 
 def reset_backoff(conn: Any) -> int:

server/packetstation/domain/pipeline.py

@@ -833,7 +833,8 @@ def apply_transition(w: Write) -> None:
 def _file(w: Write) -> None:
     """FILE: the data book date, the seal, and the data book built and stored on the seal, all in this transaction.
 
-    A data book that cannot be built answers 500 databook_failed and the whole filing rolls back."""
+    A data book that cannot be built answers 500 databook_failed and the whole filing rolls back. So does an archive
+    copy that fails (archive.FilingCopyFailed, answered 503 archive_failed by the transition endpoint)."""
     key = _seal_key(w)
     _set_packet(w, "databook_at = now()")
     row = packets.load_row(w.conn, w.id)
@@ -850,11 +851,13 @@ def _file(w: Write) -> None:
     w.log_also("Data book filed", f"{pages} pages")
     w.detail["seal_id"] = int(seal_id)
     w.detail["databook_pages"] = pages
-    # The archive copy runs in the worker after this commits. A failed copy never blocks filing.
+    # The archive copy must succeed before the packet counts as filed (Gitea issue 57). A failure raises
+    # archive.FilingCopyFailed and the whole filing rolls back; the transition endpoint records it for Admin.
     from . import archive
 
-    if archive.on_filed(w.conn, _settings(w), w.user_id, w.id):
-        w.detail["archive_requested"] = True
+    copied = archive.copy_at_filing(w.conn, _settings(w), w.id, int(seal_id))
+    if copied is not None:
+        w.detail["archive_folder"] = copied["folder"]
 
 
 def _file_databook(w: Write, row: Mapping[str, Any], seal: dict[str, Any], seal_id: int) -> int:

server/packetstation/notify/catalog.py

@@ -75,8 +75,8 @@ _TYPES = (
     EventType(
         "archive_failed",
         "Archive copy failing",
-        "A filed packet could not be copied to the archive folder after three tries, or the archive folder check"
-        " failed.",
+        "A packet could not be filed because its archive copy failed, an earlier filing could not be copied after"
+        " three tries, or the archive folder check failed.",
     ),
 )
 

server/packetstation/worker/jobs/archive_copy.py

@@ -1,10 +1,12 @@
-"""The archive copy (Gitea issue 7), every 15 minutes and on request (FILE asks for a run when it commits).
+"""The archive copy (Gitea issue 7) for seals filed while the archive was off, every 15 minutes and on request.
 
-Off when ARCHIVE_ROOT is unset, and idle until the Admin saves a folder. Each run checks the folder (the probe), queues
-every unbroken seal with a data book that has no `archive_copies` row (so turning the archive on later copies earlier
-filings too), then copies the pending rows and the failed rows that are due. A failed copy waits 5, 15, 60 minutes,
-then an hour between tries. The `archive_failed` condition (subject `archive`) is raised when a copy has failed three
-times or the folder check fails, and cleared when no copy is failed. A failed copy never touches the packet.
+FILE copies its own packet in the filing transaction and refuses to file when the copy fails (Gitea issue 57; see
+domain/archive.py). This job catches up the rest: off when ARCHIVE_ROOT is unset, and idle until the Admin saves a
+folder. Each run checks the folder (the probe), queues every unbroken seal with a data book that has no
+`archive_copies` row (so turning the archive on later copies earlier filings too), then copies the pending rows and
+the failed rows that are due. A failed copy waits 5, 15, 60 minutes, then an hour between tries. The `archive_failed`
+condition (subject `archive`) is raised when a copy has failed three times or the folder check fails, and cleared when
+no copy is failed. These packets are already filed, so a failed copy here never touches the packet.
 """
 
 from __future__ import annotations