PacketStation/server/packetstation/worker/jobs/archive_copy.py

"""The archive copy (Gitea issue 7) for seals filed while the archive was off, every 15 minutes and on request.

FILE copies its own packet in the filing transaction and refuses to file when the copy fails (Gitea issue 57; see
domain/archive.py). This job catches up the rest: off when ARCHIVE_ROOT is unset, and idle until the Admin saves a
folder. Each run checks the folder (the probe), queues every unbroken seal with a data book that has no
`archive_copies` row (so turning the archive on later copies earlier filings too), then copies the pending rows and
the failed rows that are due. A failed copy waits 5, 15, 60 minutes, then an hour between tries. The `archive_failed`
condition (subject `archive`) is raised when a copy has failed three times or the folder check fails, and cleared when
no copy is failed. These packets are already filed, so a failed copy here never touches the packet.
"""

from __future__ import annotations

from datetime import datetime

from ...domain import archive
from ...feed.config import FeedSettings
from .. import jobs
from . import Counts, JobContext, JobSpec

NAME = archive.JOB
INTERVAL_SECONDS = 15 * 60
BATCH = 200
TITLE = "Archive copy failing"


def _check_changed(ctx: JobContext, result: archive.CheckResult, archive_dir: str) -> None:
    """Audit the job's own folder check only when its result differs from the last check on record."""
    last = archive.last_check(ctx.conn)
    if last is None or bool(last["ok"]) != result.ok or last.get("archive_dir") != archive_dir:
        archive.audit_check(ctx.conn, result, archive_dir=archive_dir, source="job")


def run(ctx: JobContext) -> Counts:
    root = archive.root_of(ctx.settings)
    counts: Counts = {"enabled": root is not None, "queued": 0, "copied": 0, "failed": 0}
    if root is None:
        return counts
    archive_dir, _, _, _ = archive.load(ctx.conn)
    counts["archive_dir"] = archive_dir
    if archive_dir is None:
        counts["skipped"] = "no archive folder saved"
        return counts
    rel, result = archive.check(root, archive_dir)
    _check_changed(ctx, result, archive_dir)
    if not result.ok or rel is None:
        counts["check"] = result.message
        ctx.error(f"archive folder check failed: {result.message}")
        jobs.raise_event(
            ctx.conn,
            archive.EVENT,
            subject=archive.EVENT_SUBJECT,
            title=TITLE,
            body=f"The archive folder check failed: {result.message}",
            detail={"archive_dir": archive_dir, "message": result.message},
            severity="critical",
            condition=True,
        )
        return counts
    base = root.joinpath(*rel.split("/"))
    with ctx.conn.transaction():
        counts["queued"] = archive.queue_missing(ctx.conn)
    for row in archive.due_rows(ctx.conn, BATCH):
        try:
            folder, sha, files = archive.copy_seal(ctx.conn, ctx.settings, base, rel, int(row["seal_id"]))
        except archive.CopyError as e:
            attempts = archive.mark_failed(ctx.conn, row, str(e))
            counts["failed"] += 1
            ctx.error(f"seal {row['seal_id']}: {e} (try {attempts})", degrade=True)
            continue
        except Exception as e:  # anything unexpected is one failed copy, never a stopped run
            attempts = archive.mark_failed(ctx.conn, row, f"{type(e).__name__}: {e}")
            counts["failed"] += 1
            ctx.log.exception("archive copy of seal %s failed", row["seal_id"])
            ctx.error(f"seal {row['seal_id']}: {type(e).__name__} (try {attempts})", degrade=True)
            continue
        archive.mark_done(ctx.conn, row, folder, sha, files)
        counts["copied"] += 1
    st = archive.copies_status(ctx.conn)
    counts.update(pending=st["pending"], failed_total=st["failed"], done_total=st["done"])
    stuck = archive.stuck_failures(ctx.conn)
    if stuck:
        lines = [f"{f['packet_label']} (seal {f['seal_id']}): {f['last_error']}" for f in stuck[:10]]
        jobs.raise_event(
            ctx.conn,
            archive.EVENT,
            subject=archive.EVENT_SUBJECT,
            title=TITLE,
            body=f"{len(stuck)} filed packet(s) could not be copied to the archive:\n" + "\n".join(lines),
            detail={"failed": st["failed"], "seal_ids": [f["seal_id"] for f in stuck]},
            severity="critical",
            condition=True,
        )
    elif st["failed"] == 0:
        jobs.resolve_event(ctx.conn, archive.EVENT, subject=archive.EVENT_SUBJECT, title=TITLE)
    return counts


def cadence(_: FeedSettings, now_site: datetime) -> int:
    return INTERVAL_SECONDS


SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)