"""The archive copy (Gitea issue 7), every 15 minutes and on request (FILE asks for a run when it commits). Off when ARCHIVE_ROOT is unset, and idle until the Admin saves a folder. Each run checks the folder (the probe), queues every unbroken seal with a data book that has no `archive_copies` row (so turning the archive on later copies earlier filings too), then copies the pending rows and the failed rows that are due. A failed copy waits 5, 15, 60 minutes, then an hour between tries. The `archive_failed` condition (subject `archive`) is raised when a copy has failed three times or the folder check fails, and cleared when no copy is failed. A failed copy never touches the packet. """ from __future__ import annotations from datetime import datetime from ...domain import archive from ...feed.config import FeedSettings from .. import jobs from . import Counts, JobContext, JobSpec NAME = archive.JOB INTERVAL_SECONDS = 15 * 60 BATCH = 200 TITLE = "Archive copy failing" def _check_changed(ctx: JobContext, result: archive.CheckResult, archive_dir: str) -> None: """Audit the job's own folder check only when its result differs from the last check on record.""" last = archive.last_check(ctx.conn) if last is None or bool(last["ok"]) != result.ok or last.get("archive_dir") != archive_dir: archive.audit_check(ctx.conn, result, archive_dir=archive_dir, source="job") def run(ctx: JobContext) -> Counts: root = archive.root_of(ctx.settings) counts: Counts = {"enabled": root is not None, "queued": 0, "copied": 0, "failed": 0} if root is None: return counts archive_dir, _, _, _ = archive.load(ctx.conn) counts["archive_dir"] = archive_dir if archive_dir is None: counts["skipped"] = "no archive folder saved" return counts rel, result = archive.check(root, archive_dir) _check_changed(ctx, result, archive_dir) if not result.ok or rel is None: counts["check"] = result.message ctx.error(f"archive folder check failed: {result.message}") jobs.raise_event( ctx.conn, archive.EVENT, subject=archive.EVENT_SUBJECT, title=TITLE, body=f"The archive folder check failed: {result.message}", detail={"archive_dir": archive_dir, "message": result.message}, severity="critical", condition=True, ) return counts base = root.joinpath(*rel.split("/")) with ctx.conn.transaction(): counts["queued"] = archive.queue_missing(ctx.conn) for row in archive.due_rows(ctx.conn, BATCH): try: folder, sha, files = archive.copy_seal(ctx.conn, ctx.settings, base, rel, int(row["seal_id"])) except archive.CopyError as e: attempts = archive.mark_failed(ctx.conn, row, str(e)) counts["failed"] += 1 ctx.error(f"seal {row['seal_id']}: {e} (try {attempts})", degrade=True) continue except Exception as e: # anything unexpected is one failed copy, never a stopped run attempts = archive.mark_failed(ctx.conn, row, f"{type(e).__name__}: {e}") counts["failed"] += 1 ctx.log.exception("archive copy of seal %s failed", row["seal_id"]) ctx.error(f"seal {row['seal_id']}: {type(e).__name__} (try {attempts})", degrade=True) continue archive.mark_done(ctx.conn, row, folder, sha, files) counts["copied"] += 1 st = archive.copies_status(ctx.conn) counts.update(pending=st["pending"], failed_total=st["failed"], done_total=st["done"]) stuck = archive.stuck_failures(ctx.conn) if stuck: lines = [f"{f['packet_label']} (seal {f['seal_id']}): {f['last_error']}" for f in stuck[:10]] jobs.raise_event( ctx.conn, archive.EVENT, subject=archive.EVENT_SUBJECT, title=TITLE, body=f"{len(stuck)} filed packet(s) could not be copied to the archive:\n" + "\n".join(lines), detail={"failed": st["failed"], "seal_ids": [f["seal_id"] for f in stuck]}, severity="critical", condition=True, ) elif st["failed"] == 0: jobs.resolve_event(ctx.conn, archive.EVENT, subject=archive.EVENT_SUBJECT, title=TITLE) return counts def cadence(_: FeedSettings, now_site: datetime) -> int: return INTERVAL_SECONDS SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)