spencermandClaude Opus 5.5
da31b0a87a
A filed packet's sealed data book, documents and a manifest are copied to the archive folder by the worker. A failed copy never blocks filing: it is retried with backoff and raises the archive_failed condition. Co-Authored-By: Claude Opus 5.5 <[email protected]>
647 lines
26 KiB
Python
647 lines
26 KiB
Python
"""The archive location (Gitea issue 7): filed packets are copied to a folder under ARCHIVE_ROOT.
|
|
|
|
ARCHIVE_ROOT is a fixed path on the server (an NFS mount). Unset or blank turns the feature off. The Admin picks the
|
|
archive folder as a path under the root; the `files` settings row stores it relative (`{"archive_dir": "packets"}`).
|
|
A path that leaves the root is refused: empty, `..`, an absolute path elsewhere, NUL, a link that resolves outside
|
|
the root, or a folder that does not exist. The check writes a probe file, reads it back and removes it. Nothing here
|
|
creates folders or deletes anything in the archive.
|
|
|
|
FILE inserts a job request for `archive-copy` in its own transaction. The worker job (worker/jobs/archive_copy.py)
|
|
queues every unbroken seal with a data book that has no `archive_copies` row, then copies the pending and due failed
|
|
rows. A failed copy never blocks filing: it is retried with a backoff and raised to Admin as `archive_failed`.
|
|
|
|
Folder layout under the archive folder: `<project>/<unit>/<data book name>-seal<seal id>/` with the data book PDF,
|
|
the packet's files (drawing page files, packet documents, the Q.C. folder documents of its unit and project, MTRs)
|
|
and `manifest.json`. Each file goes to a temp name, is fsynced and renamed. An existing identical file counts as
|
|
done; a different one is never overwritten.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
from collections.abc import Iterable, Mapping
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path, PurePath
|
|
from typing import Any
|
|
|
|
from psycopg.rows import dict_row
|
|
from psycopg.types.json import Jsonb
|
|
|
|
from .. import audit
|
|
from ..settings import Settings
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
KEY = "files"
|
|
JOB = "archive-copy"
|
|
JOB_CHANNEL = "ps_job_request"
|
|
EVENT = "archive_failed"
|
|
EVENT_SUBJECT = "archive"
|
|
PROBE_PREFIX = ".packetstation-probe-"
|
|
MANIFEST = "manifest.json"
|
|
DIR_MAX = 400
|
|
FAILS_TO_RAISE = 3
|
|
# Minutes before the next try after the 1st, 2nd, 3rd failure; hourly after that.
|
|
BACKOFF_MINUTES = (5, 15, 60)
|
|
RECENT_FAILURES = 20
|
|
CHUNK = 1024 * 1024
|
|
OFF_MESSAGE = "The archive is off. Set ARCHIVE_ROOT on the server first."
|
|
|
|
_UNSAFE = re.compile(r'[\x00-\x1f\x7f<>:"/\\|?*]+')
|
|
|
|
|
|
class PathRefused(ValueError):
|
|
"""An archive folder the rules refuse. The message is for the Admin."""
|
|
|
|
|
|
class VersionConflict(Exception):
|
|
def __init__(self, version: int):
|
|
super().__init__(f"version is now {version}")
|
|
self.version = version
|
|
|
|
|
|
class CopyError(Exception):
|
|
"""One copy failed. The message goes into archive_copies.last_error and the audit row."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CheckResult:
|
|
ok: bool
|
|
resolved: str | None
|
|
free_bytes: int | None
|
|
message: str
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {"ok": self.ok, "resolved": self.resolved, "free_bytes": self.free_bytes, "message": self.message}
|
|
|
|
|
|
def root_of(settings: Settings) -> Path | None:
|
|
"""ARCHIVE_ROOT, or None when the feature is off."""
|
|
root = settings.archive_root
|
|
if root is None or not str(root).strip():
|
|
return None
|
|
return Path(root)
|
|
|
|
|
|
# ---------------------------------------------------------------- path rules
|
|
|
|
|
|
def _parts(text: str) -> list[str]:
|
|
return [p for p in re.split(r"[\\/]+", text) if p not in ("", ".")]
|
|
|
|
|
|
def _is_absolute(text: str) -> bool:
|
|
return text.startswith(("/", "\\")) or PurePath(text).is_absolute()
|
|
|
|
|
|
def relative_dir(root: Path, archive_dir: str) -> str:
|
|
"""The folder as a clean relative path under the root ("a/b"). Raises PathRefused. Lexical only."""
|
|
text = (archive_dir or "").strip()
|
|
if not text:
|
|
raise PathRefused("Enter a folder under the archive root.")
|
|
if "\x00" in text:
|
|
raise PathRefused("The folder name holds a NUL character.")
|
|
if len(text) > DIR_MAX:
|
|
raise PathRefused(f"The folder path is longer than {DIR_MAX} characters.")
|
|
if ".." in re.split(r"[\\/]+", text):
|
|
raise PathRefused("The folder may not contain '..'.")
|
|
if _is_absolute(text):
|
|
root_parts = _parts(os.path.normpath(str(root)))
|
|
given = _parts(os.path.normpath(text))
|
|
norm = (lambda xs: [x.lower() for x in xs]) if os.name == "nt" else (lambda xs: xs)
|
|
if norm(given[: len(root_parts)]) != norm(root_parts):
|
|
raise PathRefused(f"An absolute path must lie under the archive root {root}.")
|
|
rel = given[len(root_parts) :]
|
|
if not rel:
|
|
raise PathRefused("Pick a folder under the archive root, not the root itself.")
|
|
return "/".join(rel)
|
|
return "/".join(_parts(text))
|
|
|
|
|
|
def resolve_dir(root: Path, archive_dir: str) -> tuple[str, Path]:
|
|
"""(relative folder, real path) for an archive folder. Raises PathRefused when the rules refuse it."""
|
|
rel = relative_dir(root, archive_dir)
|
|
try:
|
|
real_root = root.resolve(strict=True)
|
|
except OSError:
|
|
raise PathRefused(f"The archive root {root} is not there. Is the share mounted?") from None
|
|
if not real_root.is_dir():
|
|
raise PathRefused(f"The archive root {root} is not a folder.")
|
|
target = root.joinpath(*rel.split("/"))
|
|
try:
|
|
real = target.resolve(strict=True)
|
|
except OSError:
|
|
raise PathRefused(f"The folder {rel} does not exist under {root}. Create it on the share first.") from None
|
|
if real != real_root and not real.is_relative_to(real_root):
|
|
raise PathRefused(f"The folder {rel} points outside the archive root.")
|
|
if real == real_root:
|
|
raise PathRefused("Pick a folder under the archive root, not the root itself.")
|
|
if not real.is_dir():
|
|
raise PathRefused(f"{rel} is not a folder.")
|
|
return rel, real
|
|
|
|
|
|
def probe(folder: Path) -> CheckResult:
|
|
"""Write a probe file in the folder, fsync, read it back, compare and remove it."""
|
|
name = folder / f"{PROBE_PREFIX}{secrets.token_hex(8)}"
|
|
data = secrets.token_bytes(64)
|
|
try:
|
|
with open(name, "xb") as f:
|
|
f.write(data)
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
try:
|
|
back = name.read_bytes()
|
|
finally:
|
|
name.unlink()
|
|
except OSError as e:
|
|
with contextlib.suppress(OSError):
|
|
name.unlink()
|
|
return CheckResult(False, str(folder), None, f"Cannot write in {folder}: {e.strerror or type(e).__name__}.")
|
|
if back != data:
|
|
return CheckResult(False, str(folder), None, f"The probe file read back from {folder} differs.")
|
|
try:
|
|
free = int(shutil.disk_usage(folder).free)
|
|
except OSError:
|
|
free = None
|
|
return CheckResult(True, str(folder), free, "Write, read back and delete all worked.")
|
|
|
|
|
|
def check(root: Path | None, archive_dir: str) -> tuple[str | None, CheckResult]:
|
|
"""(relative folder or None, result) for a candidate folder. Never raises for a refused path."""
|
|
if root is None:
|
|
return None, CheckResult(False, None, None, OFF_MESSAGE)
|
|
try:
|
|
rel, real = resolve_dir(root, archive_dir)
|
|
except PathRefused as e:
|
|
return None, CheckResult(False, None, None, str(e))
|
|
return rel, probe(real)
|
|
|
|
|
|
# ---------------------------------------------------------------- the settings row
|
|
|
|
|
|
def load(conn: Any) -> tuple[str | None, int, Any, Any]:
|
|
"""(archive_dir or None, version, updated_at, updated_by). Version 0 when the row is missing."""
|
|
row = conn.execute("SELECT value, version, updated_at, updated_by FROM settings WHERE key = %s", (KEY,)).fetchone()
|
|
if row is None:
|
|
return None, 0, None, None
|
|
value = row[0] if isinstance(row[0], dict) else {}
|
|
d = value.get("archive_dir")
|
|
return (str(d) if isinstance(d, str) and d.strip() else None), int(row[1]), row[2], row[3]
|
|
|
|
|
|
def save(conn: Any, *, who: Any, archive_dir: str, version: int) -> int:
|
|
"""Compare-and-set the folder (already checked). Returns the new version. Audited with the old and new value."""
|
|
with conn.transaction():
|
|
row = conn.execute("SELECT value, version FROM settings WHERE key = %s FOR UPDATE", (KEY,)).fetchone()
|
|
cur_version = int(row[1]) if row else 0
|
|
if int(version) != cur_version:
|
|
raise VersionConflict(cur_version)
|
|
value = dict(row[0]) if row and isinstance(row[0], dict) else {}
|
|
old = value.get("archive_dir")
|
|
value["archive_dir"] = archive_dir
|
|
new_version = conn.execute(
|
|
"INSERT INTO settings (key, value, version, updated_at, updated_by) VALUES (%s, %s, 1, now(), %s)"
|
|
" ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1,"
|
|
" updated_at = now(), updated_by = EXCLUDED.updated_by RETURNING version",
|
|
(KEY, Jsonb(value), who.user_id),
|
|
).fetchone()[0]
|
|
audit.append(
|
|
conn,
|
|
action="admin.file_locations_updated",
|
|
user_id=who.user_id,
|
|
session_id=getattr(who, "session_id", None),
|
|
role_snapshot=getattr(who, "role_snapshot", None),
|
|
subject_type="settings",
|
|
subject_id=KEY,
|
|
detail={"archive_dir": {"from": old, "to": archive_dir}},
|
|
ip=getattr(who, "ip", None),
|
|
)
|
|
return int(new_version)
|
|
|
|
|
|
def audit_check(conn: Any, result: CheckResult, *, archive_dir: str | None, source: str, who: Any = None) -> None:
|
|
"""One `archive.checked` audit row: source admin (the Check button), save, or job (the worker's own check)."""
|
|
audit.append(
|
|
conn,
|
|
action="archive.checked",
|
|
user_id=getattr(who, "user_id", None),
|
|
session_id=getattr(who, "session_id", None),
|
|
role_snapshot=getattr(who, "role_snapshot", None),
|
|
subject_type="archive",
|
|
subject_id=(archive_dir or "")[:200] or None,
|
|
detail={"source": source, "archive_dir": archive_dir, **result.as_dict()},
|
|
ip=getattr(who, "ip", None),
|
|
)
|
|
|
|
|
|
def last_check(conn: Any) -> dict[str, Any] | None:
|
|
"""The newest check from the audit log: {ok, at, by, message, archive_dir, source}."""
|
|
with conn.cursor(row_factory=dict_row) as cur:
|
|
r = cur.execute(
|
|
"SELECT e.at, e.detail, u.display_name FROM audit_events e LEFT JOIN users u ON u.id = e.user_id"
|
|
" WHERE e.action = 'archive.checked' ORDER BY e.id DESC LIMIT 1"
|
|
).fetchone()
|
|
if r is None:
|
|
return None
|
|
d = r["detail"] or {}
|
|
return {
|
|
"ok": bool(d.get("ok")),
|
|
"at": r["at"],
|
|
"by": r["display_name"] or ("Worker" if d.get("source") == "job" else None),
|
|
"message": d.get("message") or "",
|
|
"archive_dir": d.get("archive_dir"),
|
|
"source": d.get("source"),
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------- copies: status and requests
|
|
|
|
|
|
def copies_status(conn: Any) -> dict[str, Any]:
|
|
"""{pending, failed, done, recent_failures: [...]}."""
|
|
counts = {"pending": 0, "failed": 0, "done": 0}
|
|
for status, n in conn.execute("SELECT status, count(*) FROM archive_copies GROUP BY status").fetchall():
|
|
counts[str(status)] = int(n)
|
|
with conn.cursor(row_factory=dict_row) as cur:
|
|
rows = cur.execute(
|
|
"SELECT c.seal_id, c.packet_id, c.attempts, c.last_error, c.next_try_at,"
|
|
" pr.number AS project, u.lot, p.drawing"
|
|
" FROM archive_copies c JOIN packets p ON p.id = c.packet_id JOIN units u ON u.id = p.unit_id"
|
|
" JOIN projects pr ON pr.id = u.project_id"
|
|
" WHERE c.status = 'failed' ORDER BY c.next_try_at NULLS FIRST, c.id LIMIT %s",
|
|
(RECENT_FAILURES,),
|
|
).fetchall()
|
|
return {
|
|
**counts,
|
|
"recent_failures": [
|
|
{
|
|
"seal_id": r["seal_id"],
|
|
"packet_id": r["packet_id"],
|
|
"packet_label": packet_label(r["project"], r["lot"], r["drawing"]),
|
|
"attempts": r["attempts"],
|
|
"last_error": r["last_error"],
|
|
"next_try_at": r["next_try_at"],
|
|
}
|
|
for r in rows
|
|
],
|
|
}
|
|
|
|
|
|
def stuck_failures(conn: Any) -> list[dict[str, Any]]:
|
|
"""The failed copies that have failed FAILS_TO_RAISE times or more, oldest first."""
|
|
with conn.cursor(row_factory=dict_row) as cur:
|
|
rows = cur.execute(
|
|
"SELECT c.seal_id, c.packet_id, c.attempts, c.last_error, pr.number AS project, u.lot, p.drawing"
|
|
" FROM archive_copies c JOIN packets p ON p.id = c.packet_id JOIN units u ON u.id = p.unit_id"
|
|
" JOIN projects pr ON pr.id = u.project_id"
|
|
" WHERE c.status = 'failed' AND c.attempts >= %s ORDER BY c.id",
|
|
(FAILS_TO_RAISE,),
|
|
).fetchall()
|
|
return [{**r, "packet_label": packet_label(r["project"], r["lot"], r["drawing"])} for r in rows]
|
|
|
|
|
|
def packet_label(project: Any, lot: Any, drawing: Any) -> str:
|
|
return " ".join(x for x in (str(project or ""), f"Lot {lot}" if lot else "", str(drawing or "")) if x)
|
|
|
|
|
|
def request_copy(conn: Any, user_id: Any = None, note: str | None = None) -> None:
|
|
"""Ask the worker for an archive-copy run: a job request and a NOTIFY, in the caller's transaction."""
|
|
conn.execute("INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)", (JOB, user_id, note))
|
|
conn.execute("SELECT ps_notify(%s, %s)", (JOB_CHANNEL, json.dumps({"job": JOB}, separators=(",", ":"))))
|
|
|
|
|
|
def on_filed(conn: Any, settings: Settings, user_id: Any, packet_id: int) -> bool:
|
|
"""FILE's hook: request a prompt copy when the archive is on and a folder is saved. Never raises; a failure
|
|
here only means the copy waits for the next scheduled run. Returns whether a request went in."""
|
|
if root_of(settings) is None:
|
|
return False
|
|
try:
|
|
with conn.transaction():
|
|
if load(conn)[0] is None:
|
|
return False
|
|
request_copy(conn, user_id, f"filed packet {packet_id}")
|
|
return True
|
|
except Exception:
|
|
log.exception("archive copy request for packet %s not made; the scheduled run picks it up", packet_id)
|
|
return False
|
|
|
|
|
|
def reset_backoff(conn: Any) -> int:
|
|
"""Make every failed copy due now. Returns how many."""
|
|
return int(conn.execute("UPDATE archive_copies SET next_try_at = now() WHERE status = 'failed'").rowcount or 0)
|
|
|
|
|
|
def queue_missing(conn: Any) -> int:
|
|
"""Queue every unbroken seal with a data book and no archive_copies row. Returns how many."""
|
|
cur = conn.execute(
|
|
"INSERT INTO archive_copies (seal_id, packet_id)"
|
|
" SELECT s.id, s.packet_id FROM packet_seals s"
|
|
" WHERE s.broken_at IS NULL AND s.databook_asset_id IS NOT NULL"
|
|
" AND NOT EXISTS (SELECT 1 FROM archive_copies c WHERE c.seal_id = s.id)"
|
|
" ORDER BY s.id ON CONFLICT (seal_id) DO NOTHING"
|
|
)
|
|
return int(cur.rowcount or 0)
|
|
|
|
|
|
def due_rows(conn: Any, limit: int) -> list[dict[str, Any]]:
|
|
with conn.cursor(row_factory=dict_row) as cur:
|
|
rows: list[dict[str, Any]] = cur.execute(
|
|
"SELECT * FROM archive_copies WHERE status = 'pending'"
|
|
" OR (status = 'failed' AND (next_try_at IS NULL OR next_try_at <= now()))"
|
|
" ORDER BY (status = 'failed'), id LIMIT %s",
|
|
(limit,),
|
|
).fetchall()
|
|
return rows
|
|
|
|
|
|
def backoff(attempts: int) -> timedelta:
|
|
"""The wait after the given number of failed attempts: 5, 15, 60 minutes, then hourly."""
|
|
i = max(1, attempts) - 1
|
|
return timedelta(minutes=BACKOFF_MINUTES[i] if i < len(BACKOFF_MINUTES) else 60)
|
|
|
|
|
|
def mark_done(conn: Any, row: Mapping[str, Any], folder: str, manifest_sha: str, files: int) -> None:
|
|
with conn.transaction():
|
|
conn.execute(
|
|
"UPDATE archive_copies SET status = 'done', attempts = attempts + 1, last_error = NULL, next_try_at = NULL,"
|
|
" folder = %s, manifest_sha256 = %s, done_at = now() WHERE id = %s",
|
|
(folder, manifest_sha, row["id"]),
|
|
)
|
|
audit.append(
|
|
conn,
|
|
action="archive.copied",
|
|
subject_type="seal",
|
|
subject_id=str(row["seal_id"]),
|
|
packet_id=int(row["packet_id"]),
|
|
detail={"seal_id": row["seal_id"], "folder": folder, "files": files, "manifest_sha256": manifest_sha},
|
|
)
|
|
|
|
|
|
def mark_failed(conn: Any, row: Mapping[str, Any], error: str) -> int:
|
|
"""Record one failed attempt. Returns the attempts so far."""
|
|
attempts = int(row["attempts"] or 0) + 1
|
|
error = " ".join(error.split())[:500]
|
|
with conn.transaction():
|
|
conn.execute(
|
|
"UPDATE archive_copies SET status = 'failed', attempts = %s, last_error = %s,"
|
|
" next_try_at = now() + %s WHERE id = %s",
|
|
(attempts, error, backoff(attempts), row["id"]),
|
|
)
|
|
audit.append(
|
|
conn,
|
|
action="archive.copy_failed",
|
|
subject_type="seal",
|
|
subject_id=str(row["seal_id"]),
|
|
packet_id=int(row["packet_id"]),
|
|
detail={"seal_id": row["seal_id"], "attempts": attempts, "error": error},
|
|
)
|
|
return attempts
|
|
|
|
|
|
# ---------------------------------------------------------------- one copy
|
|
|
|
|
|
def safe_part(text: Any, fallback: str) -> str:
|
|
"""One path component: no separators or reserved characters, no leading dots, at most 120 characters."""
|
|
s = _UNSAFE.sub("_", str(text or "")).strip(" .")
|
|
return s[:120].strip(" .") or fallback
|
|
|
|
|
|
@dataclass
|
|
class Item:
|
|
asset_id: str
|
|
kind: str # databook, page, doc, qc_doc, mtr
|
|
name: str
|
|
sha256: str
|
|
size: int
|
|
path: Path
|
|
|
|
|
|
def _unique(name: str, taken: set[str]) -> str:
|
|
stem, dot, ext = name.rpartition(".")
|
|
if not dot:
|
|
stem, ext = name, ""
|
|
out, n = name, 2
|
|
while out.lower() in taken or out.lower() == MANIFEST:
|
|
out = f"{stem} ({n}).{ext}" if ext else f"{stem} ({n})"
|
|
n += 1
|
|
taken.add(out.lower())
|
|
return out
|
|
|
|
|
|
def _asset_rows(conn: Any, ids: Iterable[Any]) -> dict[str, dict[str, Any]]:
|
|
keys = list(dict.fromkeys(str(i) for i in ids if i))
|
|
if not keys:
|
|
return {}
|
|
with conn.cursor(row_factory=dict_row) as cur:
|
|
rows = cur.execute("SELECT * FROM assets WHERE id = ANY(%s::uuid[])", (keys,)).fetchall()
|
|
return {str(r["id"]): dict(r) for r in rows}
|
|
|
|
|
|
def packet_files(conn: Any, packet_id: int, unit_id: int, project_id: int) -> list[tuple[str, str]]:
|
|
"""(asset id, kind) of the packet's files in a fixed order, each asset once: drawing page files (live and void),
|
|
packet documents (not removed), the current Q.C. folder documents of its unit and project, MTRs."""
|
|
out: list[tuple[str, str]] = []
|
|
q: list[tuple[str, str, tuple[Any, ...]]] = [
|
|
(
|
|
"page",
|
|
"SELECT asset_id FROM packet_pages WHERE packet_id = %s AND asset_id IS NOT NULL"
|
|
" GROUP BY asset_id ORDER BY min(n)",
|
|
(packet_id,),
|
|
),
|
|
(
|
|
"doc",
|
|
"SELECT asset_id FROM packet_docs WHERE packet_id = %s AND removed_at IS NULL ORDER BY added_at, id",
|
|
(packet_id,),
|
|
),
|
|
(
|
|
"qc_doc",
|
|
"SELECT asset_id FROM qc_docs WHERE superseded_by IS NULL AND (unit_id = %s OR project_id = %s)"
|
|
" ORDER BY (unit_id IS NULL), added_at, id",
|
|
(unit_id, project_id),
|
|
),
|
|
(
|
|
"mtr",
|
|
"SELECT mtr_asset_id FROM bom_lines WHERE packet_id = %s AND mtr_asset_id IS NOT NULL ORDER BY item",
|
|
(packet_id,),
|
|
),
|
|
]
|
|
seen: set[str] = set()
|
|
for kind, sql, params in q:
|
|
for (aid,) in conn.execute(sql, params).fetchall():
|
|
key = str(aid)
|
|
if key not in seen:
|
|
seen.add(key)
|
|
out.append((key, kind))
|
|
return out
|
|
|
|
|
|
def _sha_file(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for chunk in iter(lambda: f.read(CHUNK), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def _fsync_dir(path: Path) -> None:
|
|
if os.name != "posix":
|
|
return
|
|
fd = os.open(str(path), os.O_RDONLY)
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def _write_once(folder: Path, name: str, source: Path | None, data: bytes | None, sha: str) -> None:
|
|
"""Put one file in the folder: temp name, fsync, rename. Identical existing file: done. Different: refused."""
|
|
target = folder / name
|
|
if target.exists():
|
|
if target.is_file() and _sha_file(target) == sha:
|
|
return
|
|
raise CopyError(f"A different file is already at {name} in the archive; it was not overwritten.")
|
|
tmp = folder / f".{name[:80]}.tmp-{secrets.token_hex(6)}"
|
|
h = hashlib.sha256()
|
|
try:
|
|
with open(tmp, "xb") as out:
|
|
if data is not None:
|
|
out.write(data)
|
|
h.update(data)
|
|
else:
|
|
assert source is not None
|
|
with open(source, "rb") as src:
|
|
for chunk in iter(lambda: src.read(CHUNK), b""):
|
|
out.write(chunk)
|
|
h.update(chunk)
|
|
out.flush()
|
|
os.fsync(out.fileno())
|
|
if h.hexdigest() != sha:
|
|
raise CopyError(f"{name} changed while it was copied (sha256 differs from the asset record).")
|
|
if target.exists():
|
|
raise CopyError(f"A file appeared at {name} during the copy; it was not overwritten.")
|
|
os.rename(tmp, target)
|
|
finally:
|
|
with contextlib.suppress(OSError):
|
|
tmp.unlink()
|
|
|
|
|
|
def copy_seal(conn: Any, settings: Settings, base: Path, rel_base: str, seal_id: int) -> tuple[str, str, int]:
|
|
"""Copy one filed seal into the archive folder `base` (its relative path `rel_base`).
|
|
Returns (folder relative to ARCHIVE_ROOT, manifest sha256, files). Raises CopyError."""
|
|
from . import assets
|
|
|
|
with conn.cursor(row_factory=dict_row) as cur:
|
|
s = cur.execute(
|
|
"SELECT s.*, p.drawing, p.rev, p.unit_id, u.lot, u.project_id, pr.number AS project,"
|
|
" pr.number || u.lot AS unit_number, sb.display_name AS sealed_by_name,"
|
|
" t.by AS filed_by, fb.display_name AS filed_by_name, t.at AS filed_at"
|
|
" FROM packet_seals s JOIN packets p ON p.id = s.packet_id JOIN units u ON u.id = p.unit_id"
|
|
" JOIN projects pr ON pr.id = u.project_id LEFT JOIN users sb ON sb.id = s.sealed_by"
|
|
" LEFT JOIN LATERAL (SELECT x.by, x.at FROM packet_transitions x WHERE x.packet_id = s.packet_id"
|
|
" AND x.action = 'file' AND x.at <= s.sealed_at + interval '1 minute' ORDER BY x.id DESC LIMIT 1) t"
|
|
" ON true"
|
|
" LEFT JOIN users fb ON fb.id = t.by WHERE s.id = %s",
|
|
(seal_id,),
|
|
).fetchone()
|
|
if s is None:
|
|
raise CopyError(f"Seal {seal_id} is not there.")
|
|
if s["databook_asset_id"] is None:
|
|
raise CopyError(f"Seal {seal_id} has no data book.")
|
|
rows = _asset_rows(conn, [s["databook_asset_id"]])
|
|
book = rows.get(str(s["databook_asset_id"]))
|
|
if book is None:
|
|
raise CopyError("The data book is not in the asset store.")
|
|
book_name = assets.safe_name(book.get("original_name")) or "Data_Book.pdf"
|
|
stem = book_name[:-4] if book_name.lower().endswith(".pdf") else book_name
|
|
rel_parts = [
|
|
safe_part(s["project"], "project"),
|
|
safe_part(s["unit_number"], "unit"),
|
|
safe_part(f"{stem}-seal{seal_id}", f"seal{seal_id}"),
|
|
]
|
|
folder = base.joinpath(*rel_parts)
|
|
|
|
ids = [(str(s["databook_asset_id"]), "databook")]
|
|
ids += [
|
|
x for x in packet_files(conn, int(s["packet_id"]), int(s["unit_id"]), int(s["project_id"])) if x[0] != ids[0][0]
|
|
]
|
|
rows.update(_asset_rows(conn, [a for a, _ in ids[1:]]))
|
|
taken: set[str] = set()
|
|
items: list[Item] = []
|
|
for aid, kind in ids:
|
|
r = rows.get(aid)
|
|
if r is None:
|
|
raise CopyError(f"Asset {aid} ({kind}) is not in the asset store.")
|
|
try:
|
|
path = assets.open_blob(settings, r)
|
|
except Exception:
|
|
raise CopyError(f"The stored file of asset {aid} ({kind}) is missing.") from None
|
|
name = assets.safe_name(r.get("original_name")) or f"{aid}{assets.EXT.get(str(r['mime']), '')}"
|
|
if kind == "databook":
|
|
name = book_name
|
|
items.append(Item(aid, kind, _unique(name, taken), bytes(r["sha256"]).hex(), int(r["size"]), path))
|
|
|
|
manifest = {
|
|
"packet_id": int(s["packet_id"]),
|
|
"packet_label": packet_label(s["project"], s["lot"], s["drawing"]),
|
|
"project": s["project"],
|
|
"unit": s["unit_number"],
|
|
"drawing": s["drawing"],
|
|
"rev": s["rev"],
|
|
"seal_id": int(seal_id),
|
|
"seal_hash": bytes(s["hash"]).hex(),
|
|
"key_id": s["key_id"],
|
|
"sealed_at": _iso(s["sealed_at"]),
|
|
"sealed_by": s["sealed_by_name"],
|
|
"filed_at": _iso(s["filed_at"] or s["sealed_at"]),
|
|
"filed_by": s["filed_by_name"] or s["sealed_by_name"],
|
|
"build": settings.ps_build,
|
|
"files": [
|
|
{"name": i.name, "kind": i.kind, "sha256": i.sha256, "size": i.size, "asset_id": i.asset_id} for i in items
|
|
],
|
|
}
|
|
data = (json.dumps(manifest, indent=2, ensure_ascii=False) + "\n").encode("utf-8")
|
|
try:
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
existing = folder / MANIFEST
|
|
if existing.exists():
|
|
same = _same_manifest(existing, manifest)
|
|
if same is not None:
|
|
return "/".join([rel_base, *rel_parts]), same, len(items)
|
|
raise CopyError(f"A different {MANIFEST} is already in {'/'.join(rel_parts)}; it was not overwritten.")
|
|
for i in items:
|
|
_write_once(folder, i.name, i.path, None, i.sha256)
|
|
sha = hashlib.sha256(data).hexdigest()
|
|
_write_once(folder, MANIFEST, None, data, sha)
|
|
_fsync_dir(folder)
|
|
except OSError as e:
|
|
raise CopyError(f"Cannot write in the archive: {e.strerror or type(e).__name__}.") from None
|
|
return "/".join([rel_base, *rel_parts]), sha, len(items)
|
|
|
|
|
|
def _same_manifest(path: Path, manifest: Mapping[str, Any]) -> str | None:
|
|
"""The sha256 of the manifest on disk when it describes the same copy (the build may differ), else None."""
|
|
try:
|
|
raw = path.read_bytes()
|
|
old = json.loads(raw)
|
|
except (OSError, ValueError):
|
|
return None
|
|
keys = ("packet_id", "seal_id", "seal_hash", "files")
|
|
if isinstance(old, dict) and all(old.get(k) == manifest.get(k) for k in keys):
|
|
return hashlib.sha256(raw).hexdigest()
|
|
return None
|
|
|
|
|
|
def _iso(v: Any) -> str | None:
|
|
return v.isoformat() if isinstance(v, datetime) else (str(v) if v is not None else None)
|