"""The backup check, daily at 07:15 site time. BACKUP_DIR is this stack's backup folder on the host (/var/backups/packetstation//), mounted read-only. The host's backup script writes `*.dump` files and the `last-success` and `last-failure` markers there. The check raises the `backup_stale` condition (subject: the stack) when the newest dump is missing, empty or older than 26 hours, or when `last-failure` is newer than `last-success`. Otherwise it resolves it. It only reads. """ from __future__ import annotations from datetime import UTC, datetime, time from pathlib import Path from ...feed.config import FeedSettings from .. import jobs from . import Counts, JobContext, JobSpec, seconds_until AT = time(7, 15) NAME = "backup-check" MAX_AGE_HOURS = 26.0 SUCCESS = "last-success" FAILURE = "last-failure" def _mtime(p: Path) -> datetime | None: try: return datetime.fromtimestamp(p.stat().st_mtime, UTC) except OSError: return None def inspect(folder: Path, now: datetime | None = None) -> tuple[Counts, list[str]]: """(counts, problems) for a backup folder. No problems means the backups are fine.""" now = now or datetime.now(UTC) counts: Counts = {"folder": str(folder), "dumps": 0, "newest": None, "age_hours": None, "size": None} if not folder.is_dir(): return counts, [f"The backup folder {folder} is not there."] dumps: list[tuple[datetime, Path, int]] = [] for p in folder.glob("*.dump"): try: st = p.stat() except OSError: continue if p.is_file(): dumps.append((datetime.fromtimestamp(st.st_mtime, UTC), p, st.st_size)) counts["dumps"] = len(dumps) problems: list[str] = [] if not dumps: problems.append("No backup dump is in the folder.") else: at, path, size = max(dumps, key=lambda d: (d[0], d[1].name)) age = (now - at).total_seconds() / 3600 counts.update(newest=path.name, newest_at=at.isoformat(), age_hours=round(age, 1), size=size) if size == 0: problems.append(f"The newest dump {path.name} is empty.") if age > MAX_AGE_HOURS: problems.append(f"The newest dump {path.name} is {age:.0f} hours old.") ok_at, fail_at = _mtime(folder / SUCCESS), _mtime(folder / FAILURE) counts["last_success"] = ok_at.isoformat() if ok_at else None counts["last_failure"] = fail_at.isoformat() if fail_at else None if fail_at is not None and (ok_at is None or fail_at > ok_at): problems.append("The last backup run failed (last-failure is newer than last-success).") return counts, problems def run(ctx: JobContext) -> Counts: stack = str(ctx.settings.stack) counts, problems = inspect(Path(ctx.settings.backup_dir)) counts["problems"] = problems title = f"Backup failed or stale ({stack})" with ctx.conn.transaction(): if problems: jobs.raise_event( ctx.conn, "backup_stale", subject=stack, title=title, body="\n".join(problems), detail={k: v for k, v in counts.items() if k != "problems"}, severity="critical", condition=True, ) else: jobs.resolve_event(ctx.conn, "backup_stale", subject=stack, title=title) return counts def cadence(_: FeedSettings, now_site: datetime) -> int: return seconds_until(now_site, AT) SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)