"""The archive location (Gitea issue 7): filed packets are copied to a folder under ARCHIVE_ROOT. ARCHIVE_ROOT is a fixed path on the server (an NFS mount). Unset or blank turns the feature off. The Admin picks the archive folder as a path under the root; the `files` settings row stores it relative (`{"archive_dir": "packets"}`). A path that leaves the root is refused: empty, `..`, an absolute path elsewhere, NUL, a link that resolves outside the root, or a folder that does not exist. The check writes a probe file, reads it back and removes it. Nothing here creates folders or deletes anything in the archive. FILE inserts a job request for `archive-copy` in its own transaction. The worker job (worker/jobs/archive_copy.py) queues every unbroken seal with a data book that has no `archive_copies` row, then copies the pending and due failed rows. A failed copy never blocks filing: it is retried with a backoff and raised to Admin as `archive_failed`. Folder layout under the archive folder: `//-seal/` with the data book PDF, the packet's files (drawing page files, packet documents, the Q.C. folder documents of its unit and project, MTRs) and `manifest.json`. Each file goes to a temp name, is fsynced and renamed. An existing identical file counts as done; a different one is never overwritten. """ from __future__ import annotations import contextlib import hashlib import json import logging import os import re import secrets import shutil from collections.abc import Iterable, Mapping from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path, PurePath from typing import Any from psycopg.rows import dict_row from psycopg.types.json import Jsonb from .. import audit from ..settings import Settings log = logging.getLogger(__name__) KEY = "files" JOB = "archive-copy" JOB_CHANNEL = "ps_job_request" EVENT = "archive_failed" EVENT_SUBJECT = "archive" PROBE_PREFIX = ".packetstation-probe-" MANIFEST = "manifest.json" DIR_MAX = 400 FAILS_TO_RAISE = 3 # Minutes before the next try after the 1st, 2nd, 3rd failure; hourly after that. BACKOFF_MINUTES = (5, 15, 60) RECENT_FAILURES = 20 CHUNK = 1024 * 1024 OFF_MESSAGE = "The archive is off. Set ARCHIVE_ROOT on the server first." _UNSAFE = re.compile(r'[\x00-\x1f\x7f<>:"/\\|?*]+') class PathRefused(ValueError): """An archive folder the rules refuse. The message is for the Admin.""" class VersionConflict(Exception): def __init__(self, version: int): super().__init__(f"version is now {version}") self.version = version class CopyError(Exception): """One copy failed. The message goes into archive_copies.last_error and the audit row.""" @dataclass(frozen=True) class CheckResult: ok: bool resolved: str | None free_bytes: int | None message: str def as_dict(self) -> dict[str, Any]: return {"ok": self.ok, "resolved": self.resolved, "free_bytes": self.free_bytes, "message": self.message} def root_of(settings: Settings) -> Path | None: """ARCHIVE_ROOT, or None when the feature is off.""" root = settings.archive_root if root is None or not str(root).strip(): return None return Path(root) # ---------------------------------------------------------------- path rules def _parts(text: str) -> list[str]: return [p for p in re.split(r"[\\/]+", text) if p not in ("", ".")] def _is_absolute(text: str) -> bool: return text.startswith(("/", "\\")) or PurePath(text).is_absolute() def relative_dir(root: Path, archive_dir: str) -> str: """The folder as a clean relative path under the root ("a/b"). Raises PathRefused. Lexical only.""" text = (archive_dir or "").strip() if not text: raise PathRefused("Enter a folder under the archive root.") if "\x00" in text: raise PathRefused("The folder name holds a NUL character.") if len(text) > DIR_MAX: raise PathRefused(f"The folder path is longer than {DIR_MAX} characters.") if ".." in re.split(r"[\\/]+", text): raise PathRefused("The folder may not contain '..'.") if _is_absolute(text): root_parts = _parts(os.path.normpath(str(root))) given = _parts(os.path.normpath(text)) norm = (lambda xs: [x.lower() for x in xs]) if os.name == "nt" else (lambda xs: xs) if norm(given[: len(root_parts)]) != norm(root_parts): raise PathRefused(f"An absolute path must lie under the archive root {root}.") rel = given[len(root_parts) :] if not rel: raise PathRefused("Pick a folder under the archive root, not the root itself.") return "/".join(rel) return "/".join(_parts(text)) def resolve_dir(root: Path, archive_dir: str) -> tuple[str, Path]: """(relative folder, real path) for an archive folder. Raises PathRefused when the rules refuse it.""" rel = relative_dir(root, archive_dir) try: real_root = root.resolve(strict=True) except OSError: raise PathRefused(f"The archive root {root} is not there. Is the share mounted?") from None if not real_root.is_dir(): raise PathRefused(f"The archive root {root} is not a folder.") target = root.joinpath(*rel.split("/")) try: real = target.resolve(strict=True) except OSError: raise PathRefused(f"The folder {rel} does not exist under {root}. Create it on the share first.") from None if real != real_root and not real.is_relative_to(real_root): raise PathRefused(f"The folder {rel} points outside the archive root.") if real == real_root: raise PathRefused("Pick a folder under the archive root, not the root itself.") if not real.is_dir(): raise PathRefused(f"{rel} is not a folder.") return rel, real def probe(folder: Path) -> CheckResult: """Write a probe file in the folder, fsync, read it back, compare and remove it.""" name = folder / f"{PROBE_PREFIX}{secrets.token_hex(8)}" data = secrets.token_bytes(64) try: with open(name, "xb") as f: f.write(data) f.flush() os.fsync(f.fileno()) try: back = name.read_bytes() finally: name.unlink() except OSError as e: with contextlib.suppress(OSError): name.unlink() return CheckResult(False, str(folder), None, f"Cannot write in {folder}: {e.strerror or type(e).__name__}.") if back != data: return CheckResult(False, str(folder), None, f"The probe file read back from {folder} differs.") try: free = int(shutil.disk_usage(folder).free) except OSError: free = None return CheckResult(True, str(folder), free, "Write, read back and delete all worked.") def check(root: Path | None, archive_dir: str) -> tuple[str | None, CheckResult]: """(relative folder or None, result) for a candidate folder. Never raises for a refused path.""" if root is None: return None, CheckResult(False, None, None, OFF_MESSAGE) try: rel, real = resolve_dir(root, archive_dir) except PathRefused as e: return None, CheckResult(False, None, None, str(e)) return rel, probe(real) # ---------------------------------------------------------------- the settings row def load(conn: Any) -> tuple[str | None, int, Any, Any]: """(archive_dir or None, version, updated_at, updated_by). Version 0 when the row is missing.""" row = conn.execute("SELECT value, version, updated_at, updated_by FROM settings WHERE key = %s", (KEY,)).fetchone() if row is None: return None, 0, None, None value = row[0] if isinstance(row[0], dict) else {} d = value.get("archive_dir") return (str(d) if isinstance(d, str) and d.strip() else None), int(row[1]), row[2], row[3] def save(conn: Any, *, who: Any, archive_dir: str, version: int) -> int: """Compare-and-set the folder (already checked). Returns the new version. Audited with the old and new value.""" with conn.transaction(): row = conn.execute("SELECT value, version FROM settings WHERE key = %s FOR UPDATE", (KEY,)).fetchone() cur_version = int(row[1]) if row else 0 if int(version) != cur_version: raise VersionConflict(cur_version) value = dict(row[0]) if row and isinstance(row[0], dict) else {} old = value.get("archive_dir") value["archive_dir"] = archive_dir new_version = conn.execute( "INSERT INTO settings (key, value, version, updated_at, updated_by) VALUES (%s, %s, 1, now(), %s)" " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1," " updated_at = now(), updated_by = EXCLUDED.updated_by RETURNING version", (KEY, Jsonb(value), who.user_id), ).fetchone()[0] audit.append( conn, action="admin.file_locations_updated", user_id=who.user_id, session_id=getattr(who, "session_id", None), role_snapshot=getattr(who, "role_snapshot", None), subject_type="settings", subject_id=KEY, detail={"archive_dir": {"from": old, "to": archive_dir}}, ip=getattr(who, "ip", None), ) return int(new_version) def audit_check(conn: Any, result: CheckResult, *, archive_dir: str | None, source: str, who: Any = None) -> None: """One `archive.checked` audit row: source admin (the Check button), save, or job (the worker's own check).""" audit.append( conn, action="archive.checked", user_id=getattr(who, "user_id", None), session_id=getattr(who, "session_id", None), role_snapshot=getattr(who, "role_snapshot", None), subject_type="archive", subject_id=(archive_dir or "")[:200] or None, detail={"source": source, "archive_dir": archive_dir, **result.as_dict()}, ip=getattr(who, "ip", None), ) def last_check(conn: Any) -> dict[str, Any] | None: """The newest check from the audit log: {ok, at, by, message, archive_dir, source}.""" with conn.cursor(row_factory=dict_row) as cur: r = cur.execute( "SELECT e.at, e.detail, u.display_name FROM audit_events e LEFT JOIN users u ON u.id = e.user_id" " WHERE e.action = 'archive.checked' ORDER BY e.id DESC LIMIT 1" ).fetchone() if r is None: return None d = r["detail"] or {} return { "ok": bool(d.get("ok")), "at": r["at"], "by": r["display_name"] or ("Worker" if d.get("source") == "job" else None), "message": d.get("message") or "", "archive_dir": d.get("archive_dir"), "source": d.get("source"), } # ---------------------------------------------------------------- copies: status and requests def copies_status(conn: Any) -> dict[str, Any]: """{pending, failed, done, recent_failures: [...]}.""" counts = {"pending": 0, "failed": 0, "done": 0} for status, n in conn.execute("SELECT status, count(*) FROM archive_copies GROUP BY status").fetchall(): counts[str(status)] = int(n) with conn.cursor(row_factory=dict_row) as cur: rows = cur.execute( "SELECT c.seal_id, c.packet_id, c.attempts, c.last_error, c.next_try_at," " pr.number AS project, u.lot, p.drawing" " FROM archive_copies c JOIN packets p ON p.id = c.packet_id JOIN units u ON u.id = p.unit_id" " JOIN projects pr ON pr.id = u.project_id" " WHERE c.status = 'failed' ORDER BY c.next_try_at NULLS FIRST, c.id LIMIT %s", (RECENT_FAILURES,), ).fetchall() return { **counts, "recent_failures": [ { "seal_id": r["seal_id"], "packet_id": r["packet_id"], "packet_label": packet_label(r["project"], r["lot"], r["drawing"]), "attempts": r["attempts"], "last_error": r["last_error"], "next_try_at": r["next_try_at"], } for r in rows ], } def stuck_failures(conn: Any) -> list[dict[str, Any]]: """The failed copies that have failed FAILS_TO_RAISE times or more, oldest first.""" with conn.cursor(row_factory=dict_row) as cur: rows = cur.execute( "SELECT c.seal_id, c.packet_id, c.attempts, c.last_error, pr.number AS project, u.lot, p.drawing" " FROM archive_copies c JOIN packets p ON p.id = c.packet_id JOIN units u ON u.id = p.unit_id" " JOIN projects pr ON pr.id = u.project_id" " WHERE c.status = 'failed' AND c.attempts >= %s ORDER BY c.id", (FAILS_TO_RAISE,), ).fetchall() return [{**r, "packet_label": packet_label(r["project"], r["lot"], r["drawing"])} for r in rows] def packet_label(project: Any, lot: Any, drawing: Any) -> str: return " ".join(x for x in (str(project or ""), f"Lot {lot}" if lot else "", str(drawing or "")) if x) def request_copy(conn: Any, user_id: Any = None, note: str | None = None) -> None: """Ask the worker for an archive-copy run: a job request and a NOTIFY, in the caller's transaction.""" conn.execute("INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)", (JOB, user_id, note)) conn.execute("SELECT ps_notify(%s, %s)", (JOB_CHANNEL, json.dumps({"job": JOB}, separators=(",", ":")))) def on_filed(conn: Any, settings: Settings, user_id: Any, packet_id: int) -> bool: """FILE's hook: request a prompt copy when the archive is on and a folder is saved. Never raises; a failure here only means the copy waits for the next scheduled run. Returns whether a request went in.""" if root_of(settings) is None: return False try: with conn.transaction(): if load(conn)[0] is None: return False request_copy(conn, user_id, f"filed packet {packet_id}") return True except Exception: log.exception("archive copy request for packet %s not made; the scheduled run picks it up", packet_id) return False def reset_backoff(conn: Any) -> int: """Make every failed copy due now. Returns how many.""" return int(conn.execute("UPDATE archive_copies SET next_try_at = now() WHERE status = 'failed'").rowcount or 0) def queue_missing(conn: Any) -> int: """Queue every unbroken seal with a data book and no archive_copies row. Returns how many.""" cur = conn.execute( "INSERT INTO archive_copies (seal_id, packet_id)" " SELECT s.id, s.packet_id FROM packet_seals s" " WHERE s.broken_at IS NULL AND s.databook_asset_id IS NOT NULL" " AND NOT EXISTS (SELECT 1 FROM archive_copies c WHERE c.seal_id = s.id)" " ORDER BY s.id ON CONFLICT (seal_id) DO NOTHING" ) return int(cur.rowcount or 0) def due_rows(conn: Any, limit: int) -> list[dict[str, Any]]: with conn.cursor(row_factory=dict_row) as cur: rows: list[dict[str, Any]] = cur.execute( "SELECT * FROM archive_copies WHERE status = 'pending'" " OR (status = 'failed' AND (next_try_at IS NULL OR next_try_at <= now()))" " ORDER BY (status = 'failed'), id LIMIT %s", (limit,), ).fetchall() return rows def backoff(attempts: int) -> timedelta: """The wait after the given number of failed attempts: 5, 15, 60 minutes, then hourly.""" i = max(1, attempts) - 1 return timedelta(minutes=BACKOFF_MINUTES[i] if i < len(BACKOFF_MINUTES) else 60) def mark_done(conn: Any, row: Mapping[str, Any], folder: str, manifest_sha: str, files: int) -> None: with conn.transaction(): conn.execute( "UPDATE archive_copies SET status = 'done', attempts = attempts + 1, last_error = NULL, next_try_at = NULL," " folder = %s, manifest_sha256 = %s, done_at = now() WHERE id = %s", (folder, manifest_sha, row["id"]), ) audit.append( conn, action="archive.copied", subject_type="seal", subject_id=str(row["seal_id"]), packet_id=int(row["packet_id"]), detail={"seal_id": row["seal_id"], "folder": folder, "files": files, "manifest_sha256": manifest_sha}, ) def mark_failed(conn: Any, row: Mapping[str, Any], error: str) -> int: """Record one failed attempt. Returns the attempts so far.""" attempts = int(row["attempts"] or 0) + 1 error = " ".join(error.split())[:500] with conn.transaction(): conn.execute( "UPDATE archive_copies SET status = 'failed', attempts = %s, last_error = %s," " next_try_at = now() + %s WHERE id = %s", (attempts, error, backoff(attempts), row["id"]), ) audit.append( conn, action="archive.copy_failed", subject_type="seal", subject_id=str(row["seal_id"]), packet_id=int(row["packet_id"]), detail={"seal_id": row["seal_id"], "attempts": attempts, "error": error}, ) return attempts # ---------------------------------------------------------------- one copy def safe_part(text: Any, fallback: str) -> str: """One path component: no separators or reserved characters, no leading dots, at most 120 characters.""" s = _UNSAFE.sub("_", str(text or "")).strip(" .") return s[:120].strip(" .") or fallback @dataclass class Item: asset_id: str kind: str # databook, page, doc, qc_doc, mtr name: str sha256: str size: int path: Path def _unique(name: str, taken: set[str]) -> str: stem, dot, ext = name.rpartition(".") if not dot: stem, ext = name, "" out, n = name, 2 while out.lower() in taken or out.lower() == MANIFEST: out = f"{stem} ({n}).{ext}" if ext else f"{stem} ({n})" n += 1 taken.add(out.lower()) return out def _asset_rows(conn: Any, ids: Iterable[Any]) -> dict[str, dict[str, Any]]: keys = list(dict.fromkeys(str(i) for i in ids if i)) if not keys: return {} with conn.cursor(row_factory=dict_row) as cur: rows = cur.execute("SELECT * FROM assets WHERE id = ANY(%s::uuid[])", (keys,)).fetchall() return {str(r["id"]): dict(r) for r in rows} def packet_files(conn: Any, packet_id: int, unit_id: int, project_id: int) -> list[tuple[str, str]]: """(asset id, kind) of the packet's files in a fixed order, each asset once: drawing page files (live and void), packet documents (not removed), the current Q.C. folder documents of its unit and project, MTRs.""" out: list[tuple[str, str]] = [] q: list[tuple[str, str, tuple[Any, ...]]] = [ ( "page", "SELECT asset_id FROM packet_pages WHERE packet_id = %s AND asset_id IS NOT NULL" " GROUP BY asset_id ORDER BY min(n)", (packet_id,), ), ( "doc", "SELECT asset_id FROM packet_docs WHERE packet_id = %s AND removed_at IS NULL ORDER BY added_at, id", (packet_id,), ), ( "qc_doc", "SELECT asset_id FROM qc_docs WHERE superseded_by IS NULL AND (unit_id = %s OR project_id = %s)" " ORDER BY (unit_id IS NULL), added_at, id", (unit_id, project_id), ), ( "mtr", "SELECT mtr_asset_id FROM bom_lines WHERE packet_id = %s AND mtr_asset_id IS NOT NULL ORDER BY item", (packet_id,), ), ] seen: set[str] = set() for kind, sql, params in q: for (aid,) in conn.execute(sql, params).fetchall(): key = str(aid) if key not in seen: seen.add(key) out.append((key, kind)) return out def _sha_file(path: Path) -> str: h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(CHUNK), b""): h.update(chunk) return h.hexdigest() def _fsync_dir(path: Path) -> None: if os.name != "posix": return fd = os.open(str(path), os.O_RDONLY) try: os.fsync(fd) finally: os.close(fd) def _write_once(folder: Path, name: str, source: Path | None, data: bytes | None, sha: str) -> None: """Put one file in the folder: temp name, fsync, rename. Identical existing file: done. Different: refused.""" target = folder / name if target.exists(): if target.is_file() and _sha_file(target) == sha: return raise CopyError(f"A different file is already at {name} in the archive; it was not overwritten.") tmp = folder / f".{name[:80]}.tmp-{secrets.token_hex(6)}" h = hashlib.sha256() try: with open(tmp, "xb") as out: if data is not None: out.write(data) h.update(data) else: assert source is not None with open(source, "rb") as src: for chunk in iter(lambda: src.read(CHUNK), b""): out.write(chunk) h.update(chunk) out.flush() os.fsync(out.fileno()) if h.hexdigest() != sha: raise CopyError(f"{name} changed while it was copied (sha256 differs from the asset record).") if target.exists(): raise CopyError(f"A file appeared at {name} during the copy; it was not overwritten.") os.rename(tmp, target) finally: with contextlib.suppress(OSError): tmp.unlink() def copy_seal(conn: Any, settings: Settings, base: Path, rel_base: str, seal_id: int) -> tuple[str, str, int]: """Copy one filed seal into the archive folder `base` (its relative path `rel_base`). Returns (folder relative to ARCHIVE_ROOT, manifest sha256, files). Raises CopyError.""" from . import assets with conn.cursor(row_factory=dict_row) as cur: s = cur.execute( "SELECT s.*, p.drawing, p.rev, p.unit_id, u.lot, u.project_id, pr.number AS project," " pr.number || u.lot AS unit_number, sb.display_name AS sealed_by_name," " t.by AS filed_by, fb.display_name AS filed_by_name, t.at AS filed_at" " FROM packet_seals s JOIN packets p ON p.id = s.packet_id JOIN units u ON u.id = p.unit_id" " JOIN projects pr ON pr.id = u.project_id LEFT JOIN users sb ON sb.id = s.sealed_by" " LEFT JOIN LATERAL (SELECT x.by, x.at FROM packet_transitions x WHERE x.packet_id = s.packet_id" " AND x.action = 'file' AND x.at <= s.sealed_at + interval '1 minute' ORDER BY x.id DESC LIMIT 1) t" " ON true" " LEFT JOIN users fb ON fb.id = t.by WHERE s.id = %s", (seal_id,), ).fetchone() if s is None: raise CopyError(f"Seal {seal_id} is not there.") if s["databook_asset_id"] is None: raise CopyError(f"Seal {seal_id} has no data book.") rows = _asset_rows(conn, [s["databook_asset_id"]]) book = rows.get(str(s["databook_asset_id"])) if book is None: raise CopyError("The data book is not in the asset store.") book_name = assets.safe_name(book.get("original_name")) or "Data_Book.pdf" stem = book_name[:-4] if book_name.lower().endswith(".pdf") else book_name rel_parts = [ safe_part(s["project"], "project"), safe_part(s["unit_number"], "unit"), safe_part(f"{stem}-seal{seal_id}", f"seal{seal_id}"), ] folder = base.joinpath(*rel_parts) ids = [(str(s["databook_asset_id"]), "databook")] ids += [ x for x in packet_files(conn, int(s["packet_id"]), int(s["unit_id"]), int(s["project_id"])) if x[0] != ids[0][0] ] rows.update(_asset_rows(conn, [a for a, _ in ids[1:]])) taken: set[str] = set() items: list[Item] = [] for aid, kind in ids: r = rows.get(aid) if r is None: raise CopyError(f"Asset {aid} ({kind}) is not in the asset store.") try: path = assets.open_blob(settings, r) except Exception: raise CopyError(f"The stored file of asset {aid} ({kind}) is missing.") from None name = assets.safe_name(r.get("original_name")) or f"{aid}{assets.EXT.get(str(r['mime']), '')}" if kind == "databook": name = book_name items.append(Item(aid, kind, _unique(name, taken), bytes(r["sha256"]).hex(), int(r["size"]), path)) manifest = { "packet_id": int(s["packet_id"]), "packet_label": packet_label(s["project"], s["lot"], s["drawing"]), "project": s["project"], "unit": s["unit_number"], "drawing": s["drawing"], "rev": s["rev"], "seal_id": int(seal_id), "seal_hash": bytes(s["hash"]).hex(), "key_id": s["key_id"], "sealed_at": _iso(s["sealed_at"]), "sealed_by": s["sealed_by_name"], "filed_at": _iso(s["filed_at"] or s["sealed_at"]), "filed_by": s["filed_by_name"] or s["sealed_by_name"], "build": settings.ps_build, "files": [ {"name": i.name, "kind": i.kind, "sha256": i.sha256, "size": i.size, "asset_id": i.asset_id} for i in items ], } data = (json.dumps(manifest, indent=2, ensure_ascii=False) + "\n").encode("utf-8") try: folder.mkdir(parents=True, exist_ok=True) existing = folder / MANIFEST if existing.exists(): same = _same_manifest(existing, manifest) if same is not None: return "/".join([rel_base, *rel_parts]), same, len(items) raise CopyError(f"A different {MANIFEST} is already in {'/'.join(rel_parts)}; it was not overwritten.") for i in items: _write_once(folder, i.name, i.path, None, i.sha256) sha = hashlib.sha256(data).hexdigest() _write_once(folder, MANIFEST, None, data, sha) _fsync_dir(folder) except OSError as e: raise CopyError(f"Cannot write in the archive: {e.strerror or type(e).__name__}.") from None return "/".join([rel_base, *rel_parts]), sha, len(items) def _same_manifest(path: Path, manifest: Mapping[str, Any]) -> str | None: """The sha256 of the manifest on disk when it describes the same copy (the build may differ), else None.""" try: raw = path.read_bytes() old = json.loads(raw) except (OSError, ValueError): return None keys = ("packet_id", "seal_id", "seal_hash", "files") if isinstance(old, dict) and all(old.get(k) == manifest.get(k) for k in keys): return hashlib.sha256(raw).hexdigest() return None def _iso(v: Any) -> str | None: return v.isoformat() if isinstance(v, datetime) else (str(v) if v is not None else None)