Archive location (Gitea issue 7): ARCHIVE_ROOT, Admin file-locations API, archive-copy job, doctor check
A filed packet's sealed data book, documents and a manifest are copied to the archive folder by the worker. A failed copy never blocks filing: it is retried with backoff and raises the archive_failed condition. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,141 @@
|
||||
"""Admin: File locations (Gitea issue 7). The archive folder under ARCHIVE_ROOT, its check, and the archive copies.
|
||||
|
||||
Every route needs the admin role and a password session. Writes answer 409 `archive_off` while ARCHIVE_ROOT is unset.
|
||||
The check and the save are audited (`archive.checked`, `admin.file_locations_updated`); "Retry now" is audited as
|
||||
`admin.archive_retry` through the job request.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from ..domain import archive
|
||||
from .deps import Db, Principal, admin_principal, get_db, settings_of
|
||||
from .errors import ApiError
|
||||
from .jobs import request_run
|
||||
from .packets import WRITE
|
||||
|
||||
router = APIRouter(prefix="/api/admin/file-locations", tags=["admin"])
|
||||
|
||||
|
||||
class CheckBody(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
archive_dir: str = Field(max_length=archive.DIR_MAX + 100)
|
||||
|
||||
|
||||
class PutBody(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
archive_dir: str = Field(max_length=archive.DIR_MAX + 100)
|
||||
version: int = Field(ge=0)
|
||||
|
||||
|
||||
def _out(conn: Any, request: Request) -> dict[str, Any]:
|
||||
root = archive.root_of(settings_of(request))
|
||||
archive_dir, version, _, _ = archive.load(conn)
|
||||
resolved = None
|
||||
if root is not None and archive_dir is not None:
|
||||
try:
|
||||
resolved = str(archive.resolve_dir(root, archive_dir)[1])
|
||||
except archive.PathRefused:
|
||||
resolved = None
|
||||
last = archive.last_check(conn)
|
||||
return {
|
||||
"enabled": root is not None,
|
||||
"root": str(root) if root is not None else None,
|
||||
"archive_dir": archive_dir,
|
||||
"version": version,
|
||||
"resolved": resolved,
|
||||
"last_check": {k: last[k] for k in ("ok", "at", "by", "message")} if last else None,
|
||||
"copies": archive.copies_status(conn),
|
||||
}
|
||||
|
||||
|
||||
def _root_or_409(request: Request) -> Any:
|
||||
root = archive.root_of(settings_of(request))
|
||||
if root is None:
|
||||
raise ApiError(409, "archive_off", archive.OFF_MESSAGE)
|
||||
return root
|
||||
|
||||
|
||||
@router.get("")
|
||||
def file_locations(
|
||||
request: Request,
|
||||
p: Principal = Depends(admin_principal), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> dict[str, Any]:
|
||||
return _out(d.conn, request)
|
||||
|
||||
|
||||
@router.post("/check", dependencies=WRITE)
|
||||
async def file_locations_check(
|
||||
body: CheckBody,
|
||||
request: Request,
|
||||
p: Principal = Depends(admin_principal), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> dict[str, Any]:
|
||||
root = _root_or_409(request)
|
||||
rel, result = await run_in_threadpool(archive.check, root, body.archive_dir)
|
||||
await run_in_threadpool(
|
||||
archive.audit_check, d.conn, result, archive_dir=rel or body.archive_dir, source="admin", who=p
|
||||
)
|
||||
if rel is None:
|
||||
# The path rules refused it (not a failed probe): 422 with the reason, still audited above.
|
||||
raise ApiError(422, "bad_path", result.message)
|
||||
return result.as_dict()
|
||||
|
||||
|
||||
@router.put("", dependencies=WRITE)
|
||||
async def file_locations_put(
|
||||
body: PutBody,
|
||||
request: Request,
|
||||
p: Principal = Depends(admin_principal), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> Any:
|
||||
root = _root_or_409(request)
|
||||
conn = d.conn
|
||||
rel, result = await run_in_threadpool(archive.check, root, body.archive_dir)
|
||||
await run_in_threadpool(
|
||||
archive.audit_check, conn, result, archive_dir=rel or body.archive_dir, source="save", who=p
|
||||
)
|
||||
if rel is None:
|
||||
raise ApiError(422, "bad_path", result.message)
|
||||
if not result.ok:
|
||||
raise ApiError(422, "check_failed", result.message, extra={"check": result.as_dict()})
|
||||
|
||||
def save() -> dict[str, Any]:
|
||||
try:
|
||||
archive.save(conn, who=p, archive_dir=rel, version=body.version)
|
||||
except archive.VersionConflict:
|
||||
raise ApiError(
|
||||
409,
|
||||
"version_conflict",
|
||||
"Someone else changed the file locations. Reload and try again.",
|
||||
extra={"current": jsonable_encoder(_out(conn, request))},
|
||||
) from None
|
||||
with conn.transaction():
|
||||
archive.request_copy(conn, p.user_id, "archive folder saved")
|
||||
return _out(conn, request)
|
||||
|
||||
return await run_in_threadpool(save)
|
||||
|
||||
|
||||
@router.post("/retry", dependencies=WRITE)
|
||||
def file_locations_retry(
|
||||
request: Request,
|
||||
p: Principal = Depends(admin_principal), # noqa: B008
|
||||
d: Db = Depends(get_db), # noqa: B008
|
||||
) -> JSONResponse:
|
||||
_root_or_409(request)
|
||||
conn = d.conn
|
||||
with conn.transaction():
|
||||
reset = archive.reset_backoff(conn)
|
||||
out = request_run(conn, p, archive.JOB, note="Retry now (File locations)", action="admin.archive_retry")
|
||||
return JSONResponse(status_code=202, content={**out, "reset": reset})
|
||||
@@ -213,7 +213,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
m.set_defaults(func=cmd_migrate)
|
||||
d = sub.add_parser("doctor", help="Check configuration and every dependency")
|
||||
d.add_argument("--json", action="store_true", help="Print the checks as a JSON list")
|
||||
d.add_argument("--skip", metavar="LIST", help="Comma list of: visual,mfiles,ldap,smtp,db,volume")
|
||||
d.add_argument("--skip", metavar="LIST", help="Comma list of: visual,mfiles,ldap,smtp,db,volume,clamav,archive")
|
||||
d.add_argument("--send-test", metavar="ADDR", help="Also send one test message to ADDR")
|
||||
d.set_defaults(func=cmd_doctor)
|
||||
g = sub.add_parser("ad-groups", help="The AD groups Packet Station maps to roles")
|
||||
|
||||
@@ -23,7 +23,7 @@ from . import __version__
|
||||
from .settings import SecretFileError, Settings
|
||||
|
||||
PASS, WARN, FAIL, SKIP = "PASS", "WARN", "FAIL", "SKIP"
|
||||
SKIPPABLE = ("visual", "mfiles", "ldap", "smtp", "db", "volume", "clamav")
|
||||
SKIPPABLE = ("visual", "mfiles", "ldap", "smtp", "db", "volume", "clamav", "archive")
|
||||
FREE_WARN_BYTES = 2 * 1024**3
|
||||
EXPECTED_PS_GROUPS = 50
|
||||
IN_CHAIN = "1.2.840.113556.1.4.1941"
|
||||
@@ -261,6 +261,46 @@ def check_volume(s: Settings) -> list[Check]:
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- archive
|
||||
|
||||
|
||||
def check_archive(s: Settings) -> list[Check]:
|
||||
"""The archive folder check (probe write, read back, delete) against the saved folder, audited like the Admin's."""
|
||||
import psycopg
|
||||
|
||||
from . import db
|
||||
from .domain import archive
|
||||
|
||||
root = archive.root_of(s)
|
||||
if root is None:
|
||||
return [Check("archive", SKIP, "ARCHIVE_ROOT not set: filed packets are not copied to an archive")]
|
||||
try:
|
||||
with db.connect_as("app", s, autocommit=True) as conn:
|
||||
archive_dir = archive.load(conn)[0]
|
||||
except (SecretFileError, psycopg.Error) as e:
|
||||
return [Check("archive", FAIL, f"can't read the saved archive folder: {_first_line(e)}")]
|
||||
if archive_dir is None:
|
||||
return [Check("archive", SKIP, f"ARCHIVE_ROOT={root}, no archive folder saved (Admin, File locations)")]
|
||||
_, result = archive.check(root, archive_dir)
|
||||
note = ""
|
||||
try:
|
||||
with db.connect_as("app", s, autocommit=True) as conn:
|
||||
archive.audit_check(conn, result, archive_dir=archive_dir, source="doctor")
|
||||
except (SecretFileError, psycopg.Error) as e:
|
||||
note = f" (not audited: {_first_line(e)})"
|
||||
if not result.ok:
|
||||
return [
|
||||
Check(
|
||||
"archive",
|
||||
FAIL,
|
||||
f"{archive_dir} under {root}: {result.message}{note}",
|
||||
"is the share mounted and writable by uid 10001?",
|
||||
)
|
||||
]
|
||||
free = f", {result.free_bytes / 1024**3:.1f} GB free" if result.free_bytes is not None else ""
|
||||
return [Check("archive", WARN if note else PASS, f"{result.resolved} writable{free}{note}")]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- clamav
|
||||
|
||||
|
||||
@@ -439,6 +479,7 @@ def run(
|
||||
("db", check_db),
|
||||
("volume", check_volume),
|
||||
("clamav", check_clamav),
|
||||
("archive", check_archive),
|
||||
("ldap", check_ldap),
|
||||
("smtp", lambda st: check_smtp(st, send_test)),
|
||||
]
|
||||
|
||||
@@ -0,0 +1,646 @@
|
||||
"""The archive location (Gitea issue 7): filed packets are copied to a folder under ARCHIVE_ROOT.
|
||||
|
||||
ARCHIVE_ROOT is a fixed path on the server (an NFS mount). Unset or blank turns the feature off. The Admin picks the
|
||||
archive folder as a path under the root; the `files` settings row stores it relative (`{"archive_dir": "packets"}`).
|
||||
A path that leaves the root is refused: empty, `..`, an absolute path elsewhere, NUL, a link that resolves outside
|
||||
the root, or a folder that does not exist. The check writes a probe file, reads it back and removes it. Nothing here
|
||||
creates folders or deletes anything in the archive.
|
||||
|
||||
FILE inserts a job request for `archive-copy` in its own transaction. The worker job (worker/jobs/archive_copy.py)
|
||||
queues every unbroken seal with a data book that has no `archive_copies` row, then copies the pending and due failed
|
||||
rows. A failed copy never blocks filing: it is retried with a backoff and raised to Admin as `archive_failed`.
|
||||
|
||||
Folder layout under the archive folder: `<project>/<unit>/<data book name>-seal<seal id>/` with the data book PDF,
|
||||
the packet's files (drawing page files, packet documents, the Q.C. folder documents of its unit and project, MTRs)
|
||||
and `manifest.json`. Each file goes to a temp name, is fsynced and renamed. An existing identical file counts as
|
||||
done; a different one is never overwritten.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path, PurePath
|
||||
from typing import Any
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from .. import audit
|
||||
from ..settings import Settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
KEY = "files"
|
||||
JOB = "archive-copy"
|
||||
JOB_CHANNEL = "ps_job_request"
|
||||
EVENT = "archive_failed"
|
||||
EVENT_SUBJECT = "archive"
|
||||
PROBE_PREFIX = ".packetstation-probe-"
|
||||
MANIFEST = "manifest.json"
|
||||
DIR_MAX = 400
|
||||
FAILS_TO_RAISE = 3
|
||||
# Minutes before the next try after the 1st, 2nd, 3rd failure; hourly after that.
|
||||
BACKOFF_MINUTES = (5, 15, 60)
|
||||
RECENT_FAILURES = 20
|
||||
CHUNK = 1024 * 1024
|
||||
OFF_MESSAGE = "The archive is off. Set ARCHIVE_ROOT on the server first."
|
||||
|
||||
_UNSAFE = re.compile(r'[\x00-\x1f\x7f<>:"/\\|?*]+')
|
||||
|
||||
|
||||
class PathRefused(ValueError):
|
||||
"""An archive folder the rules refuse. The message is for the Admin."""
|
||||
|
||||
|
||||
class VersionConflict(Exception):
|
||||
def __init__(self, version: int):
|
||||
super().__init__(f"version is now {version}")
|
||||
self.version = version
|
||||
|
||||
|
||||
class CopyError(Exception):
|
||||
"""One copy failed. The message goes into archive_copies.last_error and the audit row."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CheckResult:
|
||||
ok: bool
|
||||
resolved: str | None
|
||||
free_bytes: int | None
|
||||
message: str
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {"ok": self.ok, "resolved": self.resolved, "free_bytes": self.free_bytes, "message": self.message}
|
||||
|
||||
|
||||
def root_of(settings: Settings) -> Path | None:
|
||||
"""ARCHIVE_ROOT, or None when the feature is off."""
|
||||
root = settings.archive_root
|
||||
if root is None or not str(root).strip():
|
||||
return None
|
||||
return Path(root)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- path rules
|
||||
|
||||
|
||||
def _parts(text: str) -> list[str]:
|
||||
return [p for p in re.split(r"[\\/]+", text) if p not in ("", ".")]
|
||||
|
||||
|
||||
def _is_absolute(text: str) -> bool:
|
||||
return text.startswith(("/", "\\")) or PurePath(text).is_absolute()
|
||||
|
||||
|
||||
def relative_dir(root: Path, archive_dir: str) -> str:
|
||||
"""The folder as a clean relative path under the root ("a/b"). Raises PathRefused. Lexical only."""
|
||||
text = (archive_dir or "").strip()
|
||||
if not text:
|
||||
raise PathRefused("Enter a folder under the archive root.")
|
||||
if "\x00" in text:
|
||||
raise PathRefused("The folder name holds a NUL character.")
|
||||
if len(text) > DIR_MAX:
|
||||
raise PathRefused(f"The folder path is longer than {DIR_MAX} characters.")
|
||||
if ".." in re.split(r"[\\/]+", text):
|
||||
raise PathRefused("The folder may not contain '..'.")
|
||||
if _is_absolute(text):
|
||||
root_parts = _parts(os.path.normpath(str(root)))
|
||||
given = _parts(os.path.normpath(text))
|
||||
norm = (lambda xs: [x.lower() for x in xs]) if os.name == "nt" else (lambda xs: xs)
|
||||
if norm(given[: len(root_parts)]) != norm(root_parts):
|
||||
raise PathRefused(f"An absolute path must lie under the archive root {root}.")
|
||||
rel = given[len(root_parts) :]
|
||||
if not rel:
|
||||
raise PathRefused("Pick a folder under the archive root, not the root itself.")
|
||||
return "/".join(rel)
|
||||
return "/".join(_parts(text))
|
||||
|
||||
|
||||
def resolve_dir(root: Path, archive_dir: str) -> tuple[str, Path]:
|
||||
"""(relative folder, real path) for an archive folder. Raises PathRefused when the rules refuse it."""
|
||||
rel = relative_dir(root, archive_dir)
|
||||
try:
|
||||
real_root = root.resolve(strict=True)
|
||||
except OSError:
|
||||
raise PathRefused(f"The archive root {root} is not there. Is the share mounted?") from None
|
||||
if not real_root.is_dir():
|
||||
raise PathRefused(f"The archive root {root} is not a folder.")
|
||||
target = root.joinpath(*rel.split("/"))
|
||||
try:
|
||||
real = target.resolve(strict=True)
|
||||
except OSError:
|
||||
raise PathRefused(f"The folder {rel} does not exist under {root}. Create it on the share first.") from None
|
||||
if real != real_root and not real.is_relative_to(real_root):
|
||||
raise PathRefused(f"The folder {rel} points outside the archive root.")
|
||||
if real == real_root:
|
||||
raise PathRefused("Pick a folder under the archive root, not the root itself.")
|
||||
if not real.is_dir():
|
||||
raise PathRefused(f"{rel} is not a folder.")
|
||||
return rel, real
|
||||
|
||||
|
||||
def probe(folder: Path) -> CheckResult:
|
||||
"""Write a probe file in the folder, fsync, read it back, compare and remove it."""
|
||||
name = folder / f"{PROBE_PREFIX}{secrets.token_hex(8)}"
|
||||
data = secrets.token_bytes(64)
|
||||
try:
|
||||
with open(name, "xb") as f:
|
||||
f.write(data)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
try:
|
||||
back = name.read_bytes()
|
||||
finally:
|
||||
name.unlink()
|
||||
except OSError as e:
|
||||
with contextlib.suppress(OSError):
|
||||
name.unlink()
|
||||
return CheckResult(False, str(folder), None, f"Cannot write in {folder}: {e.strerror or type(e).__name__}.")
|
||||
if back != data:
|
||||
return CheckResult(False, str(folder), None, f"The probe file read back from {folder} differs.")
|
||||
try:
|
||||
free = int(shutil.disk_usage(folder).free)
|
||||
except OSError:
|
||||
free = None
|
||||
return CheckResult(True, str(folder), free, "Write, read back and delete all worked.")
|
||||
|
||||
|
||||
def check(root: Path | None, archive_dir: str) -> tuple[str | None, CheckResult]:
|
||||
"""(relative folder or None, result) for a candidate folder. Never raises for a refused path."""
|
||||
if root is None:
|
||||
return None, CheckResult(False, None, None, OFF_MESSAGE)
|
||||
try:
|
||||
rel, real = resolve_dir(root, archive_dir)
|
||||
except PathRefused as e:
|
||||
return None, CheckResult(False, None, None, str(e))
|
||||
return rel, probe(real)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the settings row
|
||||
|
||||
|
||||
def load(conn: Any) -> tuple[str | None, int, Any, Any]:
|
||||
"""(archive_dir or None, version, updated_at, updated_by). Version 0 when the row is missing."""
|
||||
row = conn.execute("SELECT value, version, updated_at, updated_by FROM settings WHERE key = %s", (KEY,)).fetchone()
|
||||
if row is None:
|
||||
return None, 0, None, None
|
||||
value = row[0] if isinstance(row[0], dict) else {}
|
||||
d = value.get("archive_dir")
|
||||
return (str(d) if isinstance(d, str) and d.strip() else None), int(row[1]), row[2], row[3]
|
||||
|
||||
|
||||
def save(conn: Any, *, who: Any, archive_dir: str, version: int) -> int:
|
||||
"""Compare-and-set the folder (already checked). Returns the new version. Audited with the old and new value."""
|
||||
with conn.transaction():
|
||||
row = conn.execute("SELECT value, version FROM settings WHERE key = %s FOR UPDATE", (KEY,)).fetchone()
|
||||
cur_version = int(row[1]) if row else 0
|
||||
if int(version) != cur_version:
|
||||
raise VersionConflict(cur_version)
|
||||
value = dict(row[0]) if row and isinstance(row[0], dict) else {}
|
||||
old = value.get("archive_dir")
|
||||
value["archive_dir"] = archive_dir
|
||||
new_version = conn.execute(
|
||||
"INSERT INTO settings (key, value, version, updated_at, updated_by) VALUES (%s, %s, 1, now(), %s)"
|
||||
" ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1,"
|
||||
" updated_at = now(), updated_by = EXCLUDED.updated_by RETURNING version",
|
||||
(KEY, Jsonb(value), who.user_id),
|
||||
).fetchone()[0]
|
||||
audit.append(
|
||||
conn,
|
||||
action="admin.file_locations_updated",
|
||||
user_id=who.user_id,
|
||||
session_id=getattr(who, "session_id", None),
|
||||
role_snapshot=getattr(who, "role_snapshot", None),
|
||||
subject_type="settings",
|
||||
subject_id=KEY,
|
||||
detail={"archive_dir": {"from": old, "to": archive_dir}},
|
||||
ip=getattr(who, "ip", None),
|
||||
)
|
||||
return int(new_version)
|
||||
|
||||
|
||||
def audit_check(conn: Any, result: CheckResult, *, archive_dir: str | None, source: str, who: Any = None) -> None:
|
||||
"""One `archive.checked` audit row: source admin (the Check button), save, or job (the worker's own check)."""
|
||||
audit.append(
|
||||
conn,
|
||||
action="archive.checked",
|
||||
user_id=getattr(who, "user_id", None),
|
||||
session_id=getattr(who, "session_id", None),
|
||||
role_snapshot=getattr(who, "role_snapshot", None),
|
||||
subject_type="archive",
|
||||
subject_id=(archive_dir or "")[:200] or None,
|
||||
detail={"source": source, "archive_dir": archive_dir, **result.as_dict()},
|
||||
ip=getattr(who, "ip", None),
|
||||
)
|
||||
|
||||
|
||||
def last_check(conn: Any) -> dict[str, Any] | None:
|
||||
"""The newest check from the audit log: {ok, at, by, message, archive_dir, source}."""
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
r = cur.execute(
|
||||
"SELECT e.at, e.detail, u.display_name FROM audit_events e LEFT JOIN users u ON u.id = e.user_id"
|
||||
" WHERE e.action = 'archive.checked' ORDER BY e.id DESC LIMIT 1"
|
||||
).fetchone()
|
||||
if r is None:
|
||||
return None
|
||||
d = r["detail"] or {}
|
||||
return {
|
||||
"ok": bool(d.get("ok")),
|
||||
"at": r["at"],
|
||||
"by": r["display_name"] or ("Worker" if d.get("source") == "job" else None),
|
||||
"message": d.get("message") or "",
|
||||
"archive_dir": d.get("archive_dir"),
|
||||
"source": d.get("source"),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- copies: status and requests
|
||||
|
||||
|
||||
def copies_status(conn: Any) -> dict[str, Any]:
|
||||
"""{pending, failed, done, recent_failures: [...]}."""
|
||||
counts = {"pending": 0, "failed": 0, "done": 0}
|
||||
for status, n in conn.execute("SELECT status, count(*) FROM archive_copies GROUP BY status").fetchall():
|
||||
counts[str(status)] = int(n)
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
rows = cur.execute(
|
||||
"SELECT c.seal_id, c.packet_id, c.attempts, c.last_error, c.next_try_at,"
|
||||
" pr.number AS project, u.lot, p.drawing"
|
||||
" FROM archive_copies c JOIN packets p ON p.id = c.packet_id JOIN units u ON u.id = p.unit_id"
|
||||
" JOIN projects pr ON pr.id = u.project_id"
|
||||
" WHERE c.status = 'failed' ORDER BY c.next_try_at NULLS FIRST, c.id LIMIT %s",
|
||||
(RECENT_FAILURES,),
|
||||
).fetchall()
|
||||
return {
|
||||
**counts,
|
||||
"recent_failures": [
|
||||
{
|
||||
"seal_id": r["seal_id"],
|
||||
"packet_id": r["packet_id"],
|
||||
"packet_label": packet_label(r["project"], r["lot"], r["drawing"]),
|
||||
"attempts": r["attempts"],
|
||||
"last_error": r["last_error"],
|
||||
"next_try_at": r["next_try_at"],
|
||||
}
|
||||
for r in rows
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def stuck_failures(conn: Any) -> list[dict[str, Any]]:
|
||||
"""The failed copies that have failed FAILS_TO_RAISE times or more, oldest first."""
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
rows = cur.execute(
|
||||
"SELECT c.seal_id, c.packet_id, c.attempts, c.last_error, pr.number AS project, u.lot, p.drawing"
|
||||
" FROM archive_copies c JOIN packets p ON p.id = c.packet_id JOIN units u ON u.id = p.unit_id"
|
||||
" JOIN projects pr ON pr.id = u.project_id"
|
||||
" WHERE c.status = 'failed' AND c.attempts >= %s ORDER BY c.id",
|
||||
(FAILS_TO_RAISE,),
|
||||
).fetchall()
|
||||
return [{**r, "packet_label": packet_label(r["project"], r["lot"], r["drawing"])} for r in rows]
|
||||
|
||||
|
||||
def packet_label(project: Any, lot: Any, drawing: Any) -> str:
|
||||
return " ".join(x for x in (str(project or ""), f"Lot {lot}" if lot else "", str(drawing or "")) if x)
|
||||
|
||||
|
||||
def request_copy(conn: Any, user_id: Any = None, note: str | None = None) -> None:
|
||||
"""Ask the worker for an archive-copy run: a job request and a NOTIFY, in the caller's transaction."""
|
||||
conn.execute("INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)", (JOB, user_id, note))
|
||||
conn.execute("SELECT ps_notify(%s, %s)", (JOB_CHANNEL, json.dumps({"job": JOB}, separators=(",", ":"))))
|
||||
|
||||
|
||||
def on_filed(conn: Any, settings: Settings, user_id: Any, packet_id: int) -> bool:
|
||||
"""FILE's hook: request a prompt copy when the archive is on and a folder is saved. Never raises; a failure
|
||||
here only means the copy waits for the next scheduled run. Returns whether a request went in."""
|
||||
if root_of(settings) is None:
|
||||
return False
|
||||
try:
|
||||
with conn.transaction():
|
||||
if load(conn)[0] is None:
|
||||
return False
|
||||
request_copy(conn, user_id, f"filed packet {packet_id}")
|
||||
return True
|
||||
except Exception:
|
||||
log.exception("archive copy request for packet %s not made; the scheduled run picks it up", packet_id)
|
||||
return False
|
||||
|
||||
|
||||
def reset_backoff(conn: Any) -> int:
|
||||
"""Make every failed copy due now. Returns how many."""
|
||||
return int(conn.execute("UPDATE archive_copies SET next_try_at = now() WHERE status = 'failed'").rowcount or 0)
|
||||
|
||||
|
||||
def queue_missing(conn: Any) -> int:
|
||||
"""Queue every unbroken seal with a data book and no archive_copies row. Returns how many."""
|
||||
cur = conn.execute(
|
||||
"INSERT INTO archive_copies (seal_id, packet_id)"
|
||||
" SELECT s.id, s.packet_id FROM packet_seals s"
|
||||
" WHERE s.broken_at IS NULL AND s.databook_asset_id IS NOT NULL"
|
||||
" AND NOT EXISTS (SELECT 1 FROM archive_copies c WHERE c.seal_id = s.id)"
|
||||
" ORDER BY s.id ON CONFLICT (seal_id) DO NOTHING"
|
||||
)
|
||||
return int(cur.rowcount or 0)
|
||||
|
||||
|
||||
def due_rows(conn: Any, limit: int) -> list[dict[str, Any]]:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
rows: list[dict[str, Any]] = cur.execute(
|
||||
"SELECT * FROM archive_copies WHERE status = 'pending'"
|
||||
" OR (status = 'failed' AND (next_try_at IS NULL OR next_try_at <= now()))"
|
||||
" ORDER BY (status = 'failed'), id LIMIT %s",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return rows
|
||||
|
||||
|
||||
def backoff(attempts: int) -> timedelta:
|
||||
"""The wait after the given number of failed attempts: 5, 15, 60 minutes, then hourly."""
|
||||
i = max(1, attempts) - 1
|
||||
return timedelta(minutes=BACKOFF_MINUTES[i] if i < len(BACKOFF_MINUTES) else 60)
|
||||
|
||||
|
||||
def mark_done(conn: Any, row: Mapping[str, Any], folder: str, manifest_sha: str, files: int) -> None:
|
||||
with conn.transaction():
|
||||
conn.execute(
|
||||
"UPDATE archive_copies SET status = 'done', attempts = attempts + 1, last_error = NULL, next_try_at = NULL,"
|
||||
" folder = %s, manifest_sha256 = %s, done_at = now() WHERE id = %s",
|
||||
(folder, manifest_sha, row["id"]),
|
||||
)
|
||||
audit.append(
|
||||
conn,
|
||||
action="archive.copied",
|
||||
subject_type="seal",
|
||||
subject_id=str(row["seal_id"]),
|
||||
packet_id=int(row["packet_id"]),
|
||||
detail={"seal_id": row["seal_id"], "folder": folder, "files": files, "manifest_sha256": manifest_sha},
|
||||
)
|
||||
|
||||
|
||||
def mark_failed(conn: Any, row: Mapping[str, Any], error: str) -> int:
|
||||
"""Record one failed attempt. Returns the attempts so far."""
|
||||
attempts = int(row["attempts"] or 0) + 1
|
||||
error = " ".join(error.split())[:500]
|
||||
with conn.transaction():
|
||||
conn.execute(
|
||||
"UPDATE archive_copies SET status = 'failed', attempts = %s, last_error = %s,"
|
||||
" next_try_at = now() + %s WHERE id = %s",
|
||||
(attempts, error, backoff(attempts), row["id"]),
|
||||
)
|
||||
audit.append(
|
||||
conn,
|
||||
action="archive.copy_failed",
|
||||
subject_type="seal",
|
||||
subject_id=str(row["seal_id"]),
|
||||
packet_id=int(row["packet_id"]),
|
||||
detail={"seal_id": row["seal_id"], "attempts": attempts, "error": error},
|
||||
)
|
||||
return attempts
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- one copy
|
||||
|
||||
|
||||
def safe_part(text: Any, fallback: str) -> str:
|
||||
"""One path component: no separators or reserved characters, no leading dots, at most 120 characters."""
|
||||
s = _UNSAFE.sub("_", str(text or "")).strip(" .")
|
||||
return s[:120].strip(" .") or fallback
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
asset_id: str
|
||||
kind: str # databook, page, doc, qc_doc, mtr
|
||||
name: str
|
||||
sha256: str
|
||||
size: int
|
||||
path: Path
|
||||
|
||||
|
||||
def _unique(name: str, taken: set[str]) -> str:
|
||||
stem, dot, ext = name.rpartition(".")
|
||||
if not dot:
|
||||
stem, ext = name, ""
|
||||
out, n = name, 2
|
||||
while out.lower() in taken or out.lower() == MANIFEST:
|
||||
out = f"{stem} ({n}).{ext}" if ext else f"{stem} ({n})"
|
||||
n += 1
|
||||
taken.add(out.lower())
|
||||
return out
|
||||
|
||||
|
||||
def _asset_rows(conn: Any, ids: Iterable[Any]) -> dict[str, dict[str, Any]]:
|
||||
keys = list(dict.fromkeys(str(i) for i in ids if i))
|
||||
if not keys:
|
||||
return {}
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
rows = cur.execute("SELECT * FROM assets WHERE id = ANY(%s::uuid[])", (keys,)).fetchall()
|
||||
return {str(r["id"]): dict(r) for r in rows}
|
||||
|
||||
|
||||
def packet_files(conn: Any, packet_id: int, unit_id: int, project_id: int) -> list[tuple[str, str]]:
|
||||
"""(asset id, kind) of the packet's files in a fixed order, each asset once: drawing page files (live and void),
|
||||
packet documents (not removed), the current Q.C. folder documents of its unit and project, MTRs."""
|
||||
out: list[tuple[str, str]] = []
|
||||
q: list[tuple[str, str, tuple[Any, ...]]] = [
|
||||
(
|
||||
"page",
|
||||
"SELECT asset_id FROM packet_pages WHERE packet_id = %s AND asset_id IS NOT NULL"
|
||||
" GROUP BY asset_id ORDER BY min(n)",
|
||||
(packet_id,),
|
||||
),
|
||||
(
|
||||
"doc",
|
||||
"SELECT asset_id FROM packet_docs WHERE packet_id = %s AND removed_at IS NULL ORDER BY added_at, id",
|
||||
(packet_id,),
|
||||
),
|
||||
(
|
||||
"qc_doc",
|
||||
"SELECT asset_id FROM qc_docs WHERE superseded_by IS NULL AND (unit_id = %s OR project_id = %s)"
|
||||
" ORDER BY (unit_id IS NULL), added_at, id",
|
||||
(unit_id, project_id),
|
||||
),
|
||||
(
|
||||
"mtr",
|
||||
"SELECT mtr_asset_id FROM bom_lines WHERE packet_id = %s AND mtr_asset_id IS NOT NULL ORDER BY item",
|
||||
(packet_id,),
|
||||
),
|
||||
]
|
||||
seen: set[str] = set()
|
||||
for kind, sql, params in q:
|
||||
for (aid,) in conn.execute(sql, params).fetchall():
|
||||
key = str(aid)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
out.append((key, kind))
|
||||
return out
|
||||
|
||||
|
||||
def _sha_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(CHUNK), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def _fsync_dir(path: Path) -> None:
|
||||
if os.name != "posix":
|
||||
return
|
||||
fd = os.open(str(path), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _write_once(folder: Path, name: str, source: Path | None, data: bytes | None, sha: str) -> None:
|
||||
"""Put one file in the folder: temp name, fsync, rename. Identical existing file: done. Different: refused."""
|
||||
target = folder / name
|
||||
if target.exists():
|
||||
if target.is_file() and _sha_file(target) == sha:
|
||||
return
|
||||
raise CopyError(f"A different file is already at {name} in the archive; it was not overwritten.")
|
||||
tmp = folder / f".{name[:80]}.tmp-{secrets.token_hex(6)}"
|
||||
h = hashlib.sha256()
|
||||
try:
|
||||
with open(tmp, "xb") as out:
|
||||
if data is not None:
|
||||
out.write(data)
|
||||
h.update(data)
|
||||
else:
|
||||
assert source is not None
|
||||
with open(source, "rb") as src:
|
||||
for chunk in iter(lambda: src.read(CHUNK), b""):
|
||||
out.write(chunk)
|
||||
h.update(chunk)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
if h.hexdigest() != sha:
|
||||
raise CopyError(f"{name} changed while it was copied (sha256 differs from the asset record).")
|
||||
if target.exists():
|
||||
raise CopyError(f"A file appeared at {name} during the copy; it was not overwritten.")
|
||||
os.rename(tmp, target)
|
||||
finally:
|
||||
with contextlib.suppress(OSError):
|
||||
tmp.unlink()
|
||||
|
||||
|
||||
def copy_seal(conn: Any, settings: Settings, base: Path, rel_base: str, seal_id: int) -> tuple[str, str, int]:
|
||||
"""Copy one filed seal into the archive folder `base` (its relative path `rel_base`).
|
||||
Returns (folder relative to ARCHIVE_ROOT, manifest sha256, files). Raises CopyError."""
|
||||
from . import assets
|
||||
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
s = cur.execute(
|
||||
"SELECT s.*, p.drawing, p.rev, p.unit_id, u.lot, u.project_id, pr.number AS project,"
|
||||
" pr.number || u.lot AS unit_number, sb.display_name AS sealed_by_name,"
|
||||
" t.by AS filed_by, fb.display_name AS filed_by_name, t.at AS filed_at"
|
||||
" FROM packet_seals s JOIN packets p ON p.id = s.packet_id JOIN units u ON u.id = p.unit_id"
|
||||
" JOIN projects pr ON pr.id = u.project_id LEFT JOIN users sb ON sb.id = s.sealed_by"
|
||||
" LEFT JOIN LATERAL (SELECT x.by, x.at FROM packet_transitions x WHERE x.packet_id = s.packet_id"
|
||||
" AND x.action = 'file' AND x.at <= s.sealed_at + interval '1 minute' ORDER BY x.id DESC LIMIT 1) t"
|
||||
" ON true"
|
||||
" LEFT JOIN users fb ON fb.id = t.by WHERE s.id = %s",
|
||||
(seal_id,),
|
||||
).fetchone()
|
||||
if s is None:
|
||||
raise CopyError(f"Seal {seal_id} is not there.")
|
||||
if s["databook_asset_id"] is None:
|
||||
raise CopyError(f"Seal {seal_id} has no data book.")
|
||||
rows = _asset_rows(conn, [s["databook_asset_id"]])
|
||||
book = rows.get(str(s["databook_asset_id"]))
|
||||
if book is None:
|
||||
raise CopyError("The data book is not in the asset store.")
|
||||
book_name = assets.safe_name(book.get("original_name")) or "Data_Book.pdf"
|
||||
stem = book_name[:-4] if book_name.lower().endswith(".pdf") else book_name
|
||||
rel_parts = [
|
||||
safe_part(s["project"], "project"),
|
||||
safe_part(s["unit_number"], "unit"),
|
||||
safe_part(f"{stem}-seal{seal_id}", f"seal{seal_id}"),
|
||||
]
|
||||
folder = base.joinpath(*rel_parts)
|
||||
|
||||
ids = [(str(s["databook_asset_id"]), "databook")]
|
||||
ids += [
|
||||
x for x in packet_files(conn, int(s["packet_id"]), int(s["unit_id"]), int(s["project_id"])) if x[0] != ids[0][0]
|
||||
]
|
||||
rows.update(_asset_rows(conn, [a for a, _ in ids[1:]]))
|
||||
taken: set[str] = set()
|
||||
items: list[Item] = []
|
||||
for aid, kind in ids:
|
||||
r = rows.get(aid)
|
||||
if r is None:
|
||||
raise CopyError(f"Asset {aid} ({kind}) is not in the asset store.")
|
||||
try:
|
||||
path = assets.open_blob(settings, r)
|
||||
except Exception:
|
||||
raise CopyError(f"The stored file of asset {aid} ({kind}) is missing.") from None
|
||||
name = assets.safe_name(r.get("original_name")) or f"{aid}{assets.EXT.get(str(r['mime']), '')}"
|
||||
if kind == "databook":
|
||||
name = book_name
|
||||
items.append(Item(aid, kind, _unique(name, taken), bytes(r["sha256"]).hex(), int(r["size"]), path))
|
||||
|
||||
manifest = {
|
||||
"packet_id": int(s["packet_id"]),
|
||||
"packet_label": packet_label(s["project"], s["lot"], s["drawing"]),
|
||||
"project": s["project"],
|
||||
"unit": s["unit_number"],
|
||||
"drawing": s["drawing"],
|
||||
"rev": s["rev"],
|
||||
"seal_id": int(seal_id),
|
||||
"seal_hash": bytes(s["hash"]).hex(),
|
||||
"key_id": s["key_id"],
|
||||
"sealed_at": _iso(s["sealed_at"]),
|
||||
"sealed_by": s["sealed_by_name"],
|
||||
"filed_at": _iso(s["filed_at"] or s["sealed_at"]),
|
||||
"filed_by": s["filed_by_name"] or s["sealed_by_name"],
|
||||
"build": settings.ps_build,
|
||||
"files": [
|
||||
{"name": i.name, "kind": i.kind, "sha256": i.sha256, "size": i.size, "asset_id": i.asset_id} for i in items
|
||||
],
|
||||
}
|
||||
data = (json.dumps(manifest, indent=2, ensure_ascii=False) + "\n").encode("utf-8")
|
||||
try:
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
existing = folder / MANIFEST
|
||||
if existing.exists():
|
||||
same = _same_manifest(existing, manifest)
|
||||
if same is not None:
|
||||
return "/".join([rel_base, *rel_parts]), same, len(items)
|
||||
raise CopyError(f"A different {MANIFEST} is already in {'/'.join(rel_parts)}; it was not overwritten.")
|
||||
for i in items:
|
||||
_write_once(folder, i.name, i.path, None, i.sha256)
|
||||
sha = hashlib.sha256(data).hexdigest()
|
||||
_write_once(folder, MANIFEST, None, data, sha)
|
||||
_fsync_dir(folder)
|
||||
except OSError as e:
|
||||
raise CopyError(f"Cannot write in the archive: {e.strerror or type(e).__name__}.") from None
|
||||
return "/".join([rel_base, *rel_parts]), sha, len(items)
|
||||
|
||||
|
||||
def _same_manifest(path: Path, manifest: Mapping[str, Any]) -> str | None:
|
||||
"""The sha256 of the manifest on disk when it describes the same copy (the build may differ), else None."""
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
old = json.loads(raw)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
keys = ("packet_id", "seal_id", "seal_hash", "files")
|
||||
if isinstance(old, dict) and all(old.get(k) == manifest.get(k) for k in keys):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
return None
|
||||
|
||||
|
||||
def _iso(v: Any) -> str | None:
|
||||
return v.isoformat() if isinstance(v, datetime) else (str(v) if v is not None else None)
|
||||
@@ -850,6 +850,11 @@ def _file(w: Write) -> None:
|
||||
w.log_also("Data book filed", f"{pages} pages")
|
||||
w.detail["seal_id"] = int(seal_id)
|
||||
w.detail["databook_pages"] = pages
|
||||
# The archive copy runs in the worker after this commits. A failed copy never blocks filing.
|
||||
from . import archive
|
||||
|
||||
if archive.on_filed(w.conn, _settings(w), w.user_id, w.id):
|
||||
w.detail["archive_requested"] = True
|
||||
|
||||
|
||||
def _file_databook(w: Write, row: Mapping[str, Any], seal: dict[str, Any], seal_id: int) -> int:
|
||||
|
||||
@@ -97,6 +97,7 @@ def create_app(
|
||||
from .api import (
|
||||
account,
|
||||
admin,
|
||||
admin_files,
|
||||
admin_org,
|
||||
admin_security,
|
||||
assets,
|
||||
@@ -125,6 +126,7 @@ def create_app(
|
||||
admin,
|
||||
admin_org,
|
||||
admin_security,
|
||||
admin_files,
|
||||
departments,
|
||||
events_api,
|
||||
projects,
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
-- 0012_archive: phase 10. The archive location (Gitea issue 7). Runs as ps_migrate; 0001's default privileges give
|
||||
-- ps_app SELECT, INSERT, UPDATE, DELETE on the new table.
|
||||
|
||||
-- One row per filed seal to copy to the archive folder. The archive-copy job queues every unbroken seal with a data
|
||||
-- book that has no row yet, then copies the pending rows and the failed rows that are due (next_try_at).
|
||||
-- folder is the copy's folder relative to ARCHIVE_ROOT, set when the copy is done. manifest_sha256 is the sha256 of
|
||||
-- the manifest.json written there. A copy is never removed; a refile after a reopen has its own seal and row.
|
||||
CREATE TABLE archive_copies (
|
||||
id bigserial PRIMARY KEY,
|
||||
seal_id bigint NOT NULL UNIQUE REFERENCES packet_seals (id),
|
||||
packet_id bigint NOT NULL REFERENCES packets (id),
|
||||
status text NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'done', 'failed')),
|
||||
attempts int NOT NULL DEFAULT 0,
|
||||
last_error text,
|
||||
next_try_at timestamptz,
|
||||
folder text,
|
||||
manifest_sha256 text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
done_at timestamptz,
|
||||
CHECK (status <> 'done' OR (folder IS NOT NULL AND done_at IS NOT NULL))
|
||||
);
|
||||
CREATE INDEX archive_copies_due ON archive_copies (next_try_at, id) WHERE status <> 'done';
|
||||
CREATE INDEX archive_copies_packet ON archive_copies (packet_id);
|
||||
|
||||
-- The job, so "Retry now" and FILE's request are accepted before the worker has started once. The notification
|
||||
-- rule for archive_failed is seeded by the code (notify/seeding.py) like every other event type.
|
||||
INSERT INTO job_schedule (job) VALUES ('archive-copy') ON CONFLICT (job) DO NOTHING;
|
||||
|
||||
-- Explicit, after the default privileges: the app reads, queues and updates copies; it never deletes one.
|
||||
REVOKE DELETE, TRUNCATE ON archive_copies FROM ps_app;
|
||||
GRANT SELECT, INSERT, UPDATE ON archive_copies TO ps_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ps_app;
|
||||
@@ -72,6 +72,12 @@ _TYPES = (
|
||||
),
|
||||
EventType("version_deployed", "Version deployed", "The worker started on a new build."),
|
||||
EventType("assets_quota", "Assets storage warning", "The assets storage passed its warning threshold."),
|
||||
EventType(
|
||||
"archive_failed",
|
||||
"Archive copy failing",
|
||||
"A filed packet could not be copied to the archive folder after three tries, or the archive folder check"
|
||||
" failed.",
|
||||
),
|
||||
)
|
||||
|
||||
EVENT_TYPES: dict[str, EventType] = {t.key: t for t in _TYPES}
|
||||
|
||||
@@ -33,7 +33,7 @@ def read_secret(path: Path | str, what: str = "secret") -> str:
|
||||
return value
|
||||
|
||||
|
||||
_OPTIONAL_PATHS = ("ldap_ca_bundle", "smtp_ca_bundle")
|
||||
_OPTIONAL_PATHS = ("ldap_ca_bundle", "smtp_ca_bundle", "archive_root")
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
@@ -61,6 +61,8 @@ class Settings(BaseSettings):
|
||||
|
||||
assets_dir: Path = Path("/data/assets")
|
||||
backup_dir: Path = Path("/backups")
|
||||
# The archive share (Gitea issue 7). Unset or blank turns the archive off. The Admin picks a folder under it.
|
||||
archive_root: Path | None = None
|
||||
session_key_file: Path = Path("/run/secrets/session_key")
|
||||
seal_key_file: Path = Path("/run/secrets/seal_key")
|
||||
static_dir: Path = Path("/app/static")
|
||||
@@ -125,7 +127,7 @@ class Settings(BaseSettings):
|
||||
if isinstance(data, dict):
|
||||
data = dict(data)
|
||||
for key in list(data):
|
||||
if key.lower() in _OPTIONAL_PATHS and data[key] in ("", None):
|
||||
if key.lower() in _OPTIONAL_PATHS and (data[key] is None or not str(data[key]).strip()):
|
||||
data[key] = None
|
||||
for key in list(data):
|
||||
if key.lower() in ("stack", "feed_source", "drawing_source") and isinstance(data[key], str):
|
||||
|
||||
@@ -106,6 +106,7 @@ LAZY = (
|
||||
"backup_check",
|
||||
"notify_send",
|
||||
"notify_digest",
|
||||
"archive_copy",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
"""The archive copy (Gitea issue 7), every 15 minutes and on request (FILE asks for a run when it commits).
|
||||
|
||||
Off when ARCHIVE_ROOT is unset, and idle until the Admin saves a folder. Each run checks the folder (the probe), queues
|
||||
every unbroken seal with a data book that has no `archive_copies` row (so turning the archive on later copies earlier
|
||||
filings too), then copies the pending rows and the failed rows that are due. A failed copy waits 5, 15, 60 minutes,
|
||||
then an hour between tries. The `archive_failed` condition (subject `archive`) is raised when a copy has failed three
|
||||
times or the folder check fails, and cleared when no copy is failed. A failed copy never touches the packet.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from ...domain import archive
|
||||
from ...feed.config import FeedSettings
|
||||
from .. import jobs
|
||||
from . import Counts, JobContext, JobSpec
|
||||
|
||||
NAME = archive.JOB
|
||||
INTERVAL_SECONDS = 15 * 60
|
||||
BATCH = 200
|
||||
TITLE = "Archive copy failing"
|
||||
|
||||
|
||||
def _check_changed(ctx: JobContext, result: archive.CheckResult, archive_dir: str) -> None:
|
||||
"""Audit the job's own folder check only when its result differs from the last check on record."""
|
||||
last = archive.last_check(ctx.conn)
|
||||
if last is None or bool(last["ok"]) != result.ok or last.get("archive_dir") != archive_dir:
|
||||
archive.audit_check(ctx.conn, result, archive_dir=archive_dir, source="job")
|
||||
|
||||
|
||||
def run(ctx: JobContext) -> Counts:
|
||||
root = archive.root_of(ctx.settings)
|
||||
counts: Counts = {"enabled": root is not None, "queued": 0, "copied": 0, "failed": 0}
|
||||
if root is None:
|
||||
return counts
|
||||
archive_dir, _, _, _ = archive.load(ctx.conn)
|
||||
counts["archive_dir"] = archive_dir
|
||||
if archive_dir is None:
|
||||
counts["skipped"] = "no archive folder saved"
|
||||
return counts
|
||||
rel, result = archive.check(root, archive_dir)
|
||||
_check_changed(ctx, result, archive_dir)
|
||||
if not result.ok or rel is None:
|
||||
counts["check"] = result.message
|
||||
ctx.error(f"archive folder check failed: {result.message}")
|
||||
jobs.raise_event(
|
||||
ctx.conn,
|
||||
archive.EVENT,
|
||||
subject=archive.EVENT_SUBJECT,
|
||||
title=TITLE,
|
||||
body=f"The archive folder check failed: {result.message}",
|
||||
detail={"archive_dir": archive_dir, "message": result.message},
|
||||
severity="critical",
|
||||
condition=True,
|
||||
)
|
||||
return counts
|
||||
base = root.joinpath(*rel.split("/"))
|
||||
with ctx.conn.transaction():
|
||||
counts["queued"] = archive.queue_missing(ctx.conn)
|
||||
for row in archive.due_rows(ctx.conn, BATCH):
|
||||
try:
|
||||
folder, sha, files = archive.copy_seal(ctx.conn, ctx.settings, base, rel, int(row["seal_id"]))
|
||||
except archive.CopyError as e:
|
||||
attempts = archive.mark_failed(ctx.conn, row, str(e))
|
||||
counts["failed"] += 1
|
||||
ctx.error(f"seal {row['seal_id']}: {e} (try {attempts})", degrade=True)
|
||||
continue
|
||||
except Exception as e: # anything unexpected is one failed copy, never a stopped run
|
||||
attempts = archive.mark_failed(ctx.conn, row, f"{type(e).__name__}: {e}")
|
||||
counts["failed"] += 1
|
||||
ctx.log.exception("archive copy of seal %s failed", row["seal_id"])
|
||||
ctx.error(f"seal {row['seal_id']}: {type(e).__name__} (try {attempts})", degrade=True)
|
||||
continue
|
||||
archive.mark_done(ctx.conn, row, folder, sha, files)
|
||||
counts["copied"] += 1
|
||||
st = archive.copies_status(ctx.conn)
|
||||
counts.update(pending=st["pending"], failed_total=st["failed"], done_total=st["done"])
|
||||
stuck = archive.stuck_failures(ctx.conn)
|
||||
if stuck:
|
||||
lines = [f"{f['packet_label']} (seal {f['seal_id']}): {f['last_error']}" for f in stuck[:10]]
|
||||
jobs.raise_event(
|
||||
ctx.conn,
|
||||
archive.EVENT,
|
||||
subject=archive.EVENT_SUBJECT,
|
||||
title=TITLE,
|
||||
body=f"{len(stuck)} filed packet(s) could not be copied to the archive:\n" + "\n".join(lines),
|
||||
detail={"failed": st["failed"], "seal_ids": [f["seal_id"] for f in stuck]},
|
||||
severity="critical",
|
||||
condition=True,
|
||||
)
|
||||
elif st["failed"] == 0:
|
||||
jobs.resolve_event(ctx.conn, archive.EVENT, subject=archive.EVENT_SUBJECT, title=TITLE)
|
||||
return counts
|
||||
|
||||
|
||||
def cadence(_: FeedSettings, now_site: datetime) -> int:
|
||||
return INTERVAL_SECONDS
|
||||
|
||||
|
||||
SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)
|
||||
Reference in New Issue
Block a user