Archive location (Gitea issue 7): ARCHIVE_ROOT, Admin file-locations API, archive-copy job, doctor check

A filed packet's sealed data book, documents and a manifest are copied to the archive folder by the worker. A failed copy never blocks filing: it is retried with backoff and raises the archive_failed condition. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-30 17:50:04 -06:00
parent 3fdf063b70
commit da31b0a87a
21 changed files with 1537 additions and 11 deletions
@@ -106,6 +106,7 @@ LAZY = (
    "backup_check",
    "notify_send",
    "notify_digest",
+    "archive_copy",
 )


@@ -0,0 +1,101 @@
+"""The archive copy (Gitea issue 7), every 15 minutes and on request (FILE asks for a run when it commits).
+
+Off when ARCHIVE_ROOT is unset, and idle until the Admin saves a folder. Each run checks the folder (the probe), queues
+every unbroken seal with a data book that has no `archive_copies` row (so turning the archive on later copies earlier
+filings too), then copies the pending rows and the failed rows that are due. A failed copy waits 5, 15, 60 minutes,
+then an hour between tries. The `archive_failed` condition (subject `archive`) is raised when a copy has failed three
+times or the folder check fails, and cleared when no copy is failed. A failed copy never touches the packet.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from ...domain import archive
+from ...feed.config import FeedSettings
+from .. import jobs
+from . import Counts, JobContext, JobSpec
+
+NAME = archive.JOB
+INTERVAL_SECONDS = 15 * 60
+BATCH = 200
+TITLE = "Archive copy failing"
+
+
+def _check_changed(ctx: JobContext, result: archive.CheckResult, archive_dir: str) -> None:
+    """Audit the job's own folder check only when its result differs from the last check on record."""
+    last = archive.last_check(ctx.conn)
+    if last is None or bool(last["ok"]) != result.ok or last.get("archive_dir") != archive_dir:
+        archive.audit_check(ctx.conn, result, archive_dir=archive_dir, source="job")
+
+
+def run(ctx: JobContext) -> Counts:
+    root = archive.root_of(ctx.settings)
+    counts: Counts = {"enabled": root is not None, "queued": 0, "copied": 0, "failed": 0}
+    if root is None:
+        return counts
+    archive_dir, _, _, _ = archive.load(ctx.conn)
+    counts["archive_dir"] = archive_dir
+    if archive_dir is None:
+        counts["skipped"] = "no archive folder saved"
+        return counts
+    rel, result = archive.check(root, archive_dir)
+    _check_changed(ctx, result, archive_dir)
+    if not result.ok or rel is None:
+        counts["check"] = result.message
+        ctx.error(f"archive folder check failed: {result.message}")
+        jobs.raise_event(
+            ctx.conn,
+            archive.EVENT,
+            subject=archive.EVENT_SUBJECT,
+            title=TITLE,
+            body=f"The archive folder check failed: {result.message}",
+            detail={"archive_dir": archive_dir, "message": result.message},
+            severity="critical",
+            condition=True,
+        )
+        return counts
+    base = root.joinpath(*rel.split("/"))
+    with ctx.conn.transaction():
+        counts["queued"] = archive.queue_missing(ctx.conn)
+    for row in archive.due_rows(ctx.conn, BATCH):
+        try:
+            folder, sha, files = archive.copy_seal(ctx.conn, ctx.settings, base, rel, int(row["seal_id"]))
+        except archive.CopyError as e:
+            attempts = archive.mark_failed(ctx.conn, row, str(e))
+            counts["failed"] += 1
+            ctx.error(f"seal {row['seal_id']}: {e} (try {attempts})", degrade=True)
+            continue
+        except Exception as e:  # anything unexpected is one failed copy, never a stopped run
+            attempts = archive.mark_failed(ctx.conn, row, f"{type(e).__name__}: {e}")
+            counts["failed"] += 1
+            ctx.log.exception("archive copy of seal %s failed", row["seal_id"])
+            ctx.error(f"seal {row['seal_id']}: {type(e).__name__} (try {attempts})", degrade=True)
+            continue
+        archive.mark_done(ctx.conn, row, folder, sha, files)
+        counts["copied"] += 1
+    st = archive.copies_status(ctx.conn)
+    counts.update(pending=st["pending"], failed_total=st["failed"], done_total=st["done"])
+    stuck = archive.stuck_failures(ctx.conn)
+    if stuck:
+        lines = [f"{f['packet_label']} (seal {f['seal_id']}): {f['last_error']}" for f in stuck[:10]]
+        jobs.raise_event(
+            ctx.conn,
+            archive.EVENT,
+            subject=archive.EVENT_SUBJECT,
+            title=TITLE,
+            body=f"{len(stuck)} filed packet(s) could not be copied to the archive:\n" + "\n".join(lines),
+            detail={"failed": st["failed"], "seal_ids": [f["seal_id"] for f in stuck]},
+            severity="critical",
+            condition=True,
+        )
+    elif st["failed"] == 0:
+        jobs.resolve_event(ctx.conn, archive.EVENT, subject=archive.EVENT_SUBJECT, title=TITLE)
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    return INTERVAL_SECONDS
+
+
+SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)