PacketStation/server/packetstation/worker/jobs/housekeeping.py

"""Housekeeping, nightly at 03:30 site time: prune the VISUAL change rows and gone cache rows, old job runs and
old claimed run requests. It never touches packets, the audit log or anything append-only. Then it checks the assets
storage against its warning threshold (assets.check_quota opens or resolves the alert)."""

from __future__ import annotations

from datetime import datetime, time

from ...domain import assets
from ...feed.config import FeedSettings
from . import Counts, JobContext, JobSpec, seconds_until

AT = time(3, 30)
CACHE_DAYS = 90
RUNS_DAYS = 180
REQUESTS_DAYS = 30

STEPS: tuple[tuple[str, str, int], ...] = (
    ("row_changes", "DELETE FROM visual_row_changes WHERE at < now() - make_interval(days => %s)", CACHE_DAYS),
    (
        "material_changes",
        "DELETE FROM visual_material_changes WHERE at < now() - make_interval(days => %s)",
        CACHE_DAYS,
    ),
    (
        "gone_rows",
        "DELETE FROM visual_rows WHERE gone_batch IN"
        " (SELECT id FROM visual_batches WHERE started_at < now() - make_interval(days => %s))",
        CACHE_DAYS,
    ),
    (
        "gone_materials",
        "DELETE FROM visual_materials WHERE gone_batch IN"
        " (SELECT id FROM visual_batches WHERE started_at < now() - make_interval(days => %s))",
        CACHE_DAYS,
    ),
    (
        "job_runs",
        "DELETE FROM job_runs WHERE started_at < now() - make_interval(days => %s) AND status <> 'running'",
        RUNS_DAYS,
    ),
    (
        "job_requests",
        "DELETE FROM job_requests WHERE claimed_at IS NOT NULL AND claimed_at < now() - make_interval(days => %s)",
        REQUESTS_DAYS,
    ),
)


def run(ctx: JobContext) -> Counts:
    counts: Counts = {}
    for name, sql, days in STEPS:
        with ctx.conn.transaction():
            counts[name] = ctx.conn.execute(sql, (days,)).rowcount
    try:
        usage = assets.check_quota(ctx.conn, ctx.settings)
    except Exception as e:  # the storage check never fails the pruning
        ctx.log.exception("the storage quota check failed")
        counts["quota_error"] = type(e).__name__
    else:
        counts["quota_pct"] = usage["pct"]
        counts["quota_warn"] = usage["warn"]
        if usage["alert"]:
            counts["quota_alert"] = usage["alert"]
    return counts


def cadence(_: FeedSettings, now_site: datetime) -> int:
    return seconds_until(now_site, AT)


SPEC = JobSpec(name="housekeeping", run=run, cadence=cadence, pausable=False)