93 lines
3.5 KiB
Python
93 lines
3.5 KiB
Python
 
|
"""The backup check, daily at 07:15 site time.
|
||
|
|
|
||
|
|
BACKUP_DIR is this stack's backup folder on the host (/var/backups/packetstation/<stack>/), mounted read-only.
|
||
|
|
The host's backup script writes `*.dump` files and the `last-success` and `last-failure` markers there. The
|
||
|
|
check raises the `backup_stale` condition (subject: the stack) when the newest dump is missing, empty or older
|
||
|
|
than 26 hours, or when `last-failure` is newer than `last-success`. Otherwise it resolves it. It only reads.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from datetime import UTC, datetime, time
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from ...feed.config import FeedSettings
|
||
|
|
from .. import jobs
|
||
|
|
from . import Counts, JobContext, JobSpec, seconds_until
|
||
|
|
|
||
|
|
AT = time(7, 15)
|
||
|
|
NAME = "backup-check"
|
||
|
|
MAX_AGE_HOURS = 26.0
|
||
|
|
SUCCESS = "last-success"
|
||
|
|
FAILURE = "last-failure"
|
||
|
|
|
||
|
|
|
||
|
|
def _mtime(p: Path) -> datetime | None:
|
||
|
|
try:
|
||
|
|
return datetime.fromtimestamp(p.stat().st_mtime, UTC)
|
||
|
|
except OSError:
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def inspect(folder: Path, now: datetime | None = None) -> tuple[Counts, list[str]]:
|
||
|
|
"""(counts, problems) for a backup folder. No problems means the backups are fine."""
|
||
|
|
now = now or datetime.now(UTC)
|
||
|
|
counts: Counts = {"folder": str(folder), "dumps": 0, "newest": None, "age_hours": None, "size": None}
|
||
|
|
if not folder.is_dir():
|
||
|
|
return counts, [f"The backup folder {folder} is not there."]
|
||
|
|
dumps: list[tuple[datetime, Path, int]] = []
|
||
|
|
for p in folder.glob("*.dump"):
|
||
|
|
try:
|
||
|
|
st = p.stat()
|
||
|
|
except OSError:
|
||
|
|
continue
|
||
|
|
if p.is_file():
|
||
|
|
dumps.append((datetime.fromtimestamp(st.st_mtime, UTC), p, st.st_size))
|
||
|
|
counts["dumps"] = len(dumps)
|
||
|
|
problems: list[str] = []
|
||
|
|
if not dumps:
|
||
|
|
problems.append("No backup dump is in the folder.")
|
||
|
|
else:
|
||
|
|
at, path, size = max(dumps, key=lambda d: (d[0], d[1].name))
|
||
|
|
age = (now - at).total_seconds() / 3600
|
||
|
|
counts.update(newest=path.name, newest_at=at.isoformat(), age_hours=round(age, 1), size=size)
|
||
|
|
if size == 0:
|
||
|
|
problems.append(f"The newest dump {path.name} is empty.")
|
||
|
|
if age > MAX_AGE_HOURS:
|
||
|
|
problems.append(f"The newest dump {path.name} is {age:.0f} hours old.")
|
||
|
|
ok_at, fail_at = _mtime(folder / SUCCESS), _mtime(folder / FAILURE)
|
||
|
|
counts["last_success"] = ok_at.isoformat() if ok_at else None
|
||
|
|
counts["last_failure"] = fail_at.isoformat() if fail_at else None
|
||
|
|
if fail_at is not None and (ok_at is None or fail_at > ok_at):
|
||
|
|
problems.append("The last backup run failed (last-failure is newer than last-success).")
|
||
|
|
return counts, problems
|
||
|
|
|
||
|
|
|
||
|
|
def run(ctx: JobContext) -> Counts:
|
||
|
|
stack = str(ctx.settings.stack)
|
||
|
|
counts, problems = inspect(Path(ctx.settings.backup_dir))
|
||
|
|
counts["problems"] = problems
|
||
|
|
title = f"Backup failed or stale ({stack})"
|
||
|
|
with ctx.conn.transaction():
|
||
|
|
if problems:
|
||
|
|
jobs.raise_event(
|
||
|
|
ctx.conn,
|
||
|
|
"backup_stale",
|
||
|
|
subject=stack,
|
||
|
|
title=title,
|
||
|
|
body="\n".join(problems),
|
||
|
|
detail={k: v for k, v in counts.items() if k != "problems"},
|
||
|
|
severity="critical",
|
||
|
|
condition=True,
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
jobs.resolve_event(ctx.conn, "backup_stale", subject=stack, title=title)
|
||
|
|
return counts
|
||
|
|
|
||
|
|
|
||
|
|
def cadence(_: FeedSettings, now_site: datetime) -> int:
|
||
|
|
return seconds_until(now_site, AT)
|
||
|
|
|
||
|
|
|
||
|
|
SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)
|