mirror of
https://github.com/CyberMind-FR/secubox-deb.git
synced 2026-07-29 09:14:33 +00:00
perf(profiles): batch the probes and cache the status (46s -> 0.03s)
GET /status took 46 SECONDS on the board and the panel was unusable. Cause: observe() ran 3 subprocesses per module (is-enabled, is-active, show) plus lxc-info, and load_routes() re-read the routes file once per module. With 187 modules that is ~560 spawns at ~29ms each, on a box running 118 services at load 5.4 on 4 cores. observe_all() now issues ONE systemctl-show for every unit (measured: 187 units in 0.28s), ONE LXC enumeration, and reads routes once. RSS comes from /proc/<MainPID>/status - a file read, not a spawn. web.py serves /status from a 60s cache refreshed off the event loop (asyncio.to_thread), per the project's own "double caching" pattern; the CLI still computes live, since a one-shot command gains nothing from staleness. /status reports cached_at + age_s rather than pretending to be live. Two traps, both covered by tests: - A bare template unit (name@.service) makes systemctl-show fail for the WHOLE batch - it returned 40 blocks of 188. Templates are excluded. - A unit missing from the batch output must become enabled=None/active=None (unknown), never False. This branch has fixed five defects of that class; a batch is a fresh way to commit the same one. Measured live on gk2 after deploy: 46s -> 2.57s cold, 0.025s warm; 187 modules; sockets 108 before and after; panel 1.6ms and API 94ms through nginx. The honest-unknown rule is visible in the real data: totals went from off:66/unknown:0 to off:62/unknown:4 - the old code was asserting "off" for the 4 template units it could not actually resolve. Co-Authored-By: Gerald KERMA <devel@cybermind.fr>
This commit is contained in:
parent
dfb01df1bb
commit
b7bd56b23a
|
|
@ -22,7 +22,7 @@ from pathlib import Path
|
|||
|
||||
from .diff import ProtectedViolation, plan_changes
|
||||
from .manifest import ManifestError, load_all
|
||||
from .observe import Actual, is_on, load_routes, observe
|
||||
from .observe import Actual, is_on, load_routes, observe_all
|
||||
from .scan import discover, write_drafts
|
||||
from .state import StateError, load_pins, load_profile
|
||||
|
||||
|
|
@ -35,7 +35,13 @@ def _paths(root: Path):
|
|||
|
||||
|
||||
def _observe_all(manifests, routes):
|
||||
return {mid: observe(m, routes=routes) for mid, m in manifests.items()}
|
||||
"""Point d'entrée partagé par `status`/`diff` (CLI) et par le calcul de
|
||||
cache de l'API web (voir api/web.py) — batché (observe.observe_all), pas
|
||||
une boucle observe() par module : sur 187 modules, la boucle coûtait
|
||||
~46s (560 sous-process) contre <1s batché (mesuré sur la board). observe()
|
||||
module par module reste la référence (tests, sondage d'un seul module) ;
|
||||
ce wrapper n'en change pas le contrat, seulement le coût."""
|
||||
return observe_all(manifests, routes=routes)
|
||||
|
||||
|
||||
def _active_profile_name(root: Path, override: str | None) -> str | None:
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ du projet), donc un état mémorisé ment.
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
|
@ -28,6 +29,26 @@ PROC = Path("/proc")
|
|||
ROUTES_FILE = Path("/etc/secubox/waf/haproxy-routes.json")
|
||||
|
||||
_TIMEOUT = 5
|
||||
_BATCH_TIMEOUT = 15
|
||||
|
||||
# Unit *templates* (bare "name@.service", no instance) are not runnable units:
|
||||
# `systemctl show` errors out on them ("neither a valid invocation ID nor
|
||||
# unit name") and — verified on the board — that error ABORTS the whole
|
||||
# batched invocation after the first bad unit, silently dropping every unit
|
||||
# that would have come after it in argv. They must never enter the batch;
|
||||
# they fall out of `unit_props` naturally and are reported unknown (None),
|
||||
# same as any other unit missing from the batch output.
|
||||
_TEMPLATE_UNIT_RE = re.compile(r"@\.service$")
|
||||
|
||||
# The full, exhaustive set of systemd UnitFileState values that make
|
||||
# `systemctl is-enabled` exit non-zero (verified against systemd's own
|
||||
# unit_file_state enum, and against the 4 states actually observed on the
|
||||
# board: disabled/enabled/masked/static). Everything else — notably
|
||||
# "static", "indirect", "generated", "alias" — is a genuine is-enabled
|
||||
# success (rc=0), so it must map to True, not False.
|
||||
_DISABLED_UNIT_FILE_STATES = frozenset({
|
||||
"disabled", "disabled-runtime", "masked", "masked-runtime", "invalid", "bad", "",
|
||||
})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -118,6 +139,146 @@ def observe(m: Manifest, *, run=_run_cmd, routes: set[str] | None = None) -> Act
|
|||
lxc_autostart=lxc_autostart, portal_routed=portal_routed, rss_kb=rss)
|
||||
|
||||
|
||||
def _parse_systemctl_show_blocks(output: str) -> dict[str, dict[str, str]]:
|
||||
"""`systemctl show <unit...> -p A -p B ...` prints one block per unit,
|
||||
blocks separated by a blank line. Property order WITHIN a block is not
|
||||
stable (observed MainPID before Id on the board) — parse by key, never
|
||||
by position. A unit with no `Id=` line (shouldn't happen, but a
|
||||
defensive read) contributes nothing rather than a bogus entry."""
|
||||
out: dict[str, dict[str, str]] = {}
|
||||
for block in output.split("\n\n"):
|
||||
props: dict[str, str] = {}
|
||||
for line in block.splitlines():
|
||||
key, sep, value = line.partition("=")
|
||||
if sep:
|
||||
props[key] = value
|
||||
unit_id = props.get("Id")
|
||||
if unit_id:
|
||||
out[unit_id] = props
|
||||
return out
|
||||
|
||||
|
||||
def _parse_lxc_ls_fancy(output: str) -> dict[str, tuple[bool | None, bool | None]]:
|
||||
"""`lxc-ls -f` prints a header line (NAME/STATE/AUTOSTART/GROUPS/IPV4/…)
|
||||
then one row per container, columns fixed-width-aligned under the
|
||||
header. Locate EVERY header column's start offset (not just the 3 we
|
||||
need) so each field's slice is bounded by the *next* column, wherever it
|
||||
is — bounding AUTOSTART's slice at end-of-line would swallow GROUPS/IPV4/
|
||||
IPV6/UNPRIVILEGED into it whenever AUTOSTART isn't the last header word."""
|
||||
lines = output.splitlines()
|
||||
if not lines:
|
||||
return {}
|
||||
header = lines[0]
|
||||
col_starts = [m.start() for m in re.finditer(r"\S+", header)]
|
||||
col_names = [m.group() for m in re.finditer(r"\S+", header)]
|
||||
if not {"NAME", "STATE", "AUTOSTART"} <= set(col_names):
|
||||
return {}
|
||||
|
||||
out: dict[str, tuple[bool | None, bool | None]] = {}
|
||||
for line in lines[1:]:
|
||||
if not line.strip():
|
||||
continue
|
||||
fields: dict[str, str] = {}
|
||||
for i, name in enumerate(col_names):
|
||||
start = col_starts[i]
|
||||
end = col_starts[i + 1] if i + 1 < len(col_starts) else len(line)
|
||||
fields[name] = line[start:end].strip()
|
||||
cname = fields.get("NAME")
|
||||
if not cname:
|
||||
continue
|
||||
state = fields.get("STATE", "").upper()
|
||||
running = ("RUNNING" in state) if state else None
|
||||
autostart_raw = fields.get("AUTOSTART", "").upper()
|
||||
if autostart_raw in ("1", "YES"):
|
||||
autostart: bool | None = True
|
||||
elif autostart_raw in ("0", "NO"):
|
||||
autostart = False
|
||||
else:
|
||||
autostart = None
|
||||
out[cname] = (running, autostart)
|
||||
return out
|
||||
|
||||
|
||||
def _run_batch(argv: list[str]) -> tuple[int | None, str]:
|
||||
try:
|
||||
p = subprocess.run(argv, capture_output=True, text=True, timeout=_BATCH_TIMEOUT)
|
||||
return p.returncode, p.stdout
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None, ""
|
||||
|
||||
|
||||
def observe_all(manifests: dict[str, Manifest], *, run=_run_batch,
|
||||
routes: set[str] | None = None) -> dict[str, Actual]:
|
||||
"""État réel de TOUS les modules — chemin rapide. Une seule commande
|
||||
`systemctl show` batchée pour toutes les units natives, une seule
|
||||
énumération `lxc-ls -f` pour tous les conteneurs, routes lues UNE fois
|
||||
(par l'appelant, ou ici si non fournies). `observe()` reste la référence
|
||||
module-par-module ; ceci ne la remplace pas, c'est le chemin utilisé
|
||||
quand on observe la totalité de l'inventaire (187 modules ~46s en boucle
|
||||
-> <1s batché, mesuré sur la board).
|
||||
|
||||
Invariant identique à observe() : une unit absente du batch (unit
|
||||
inconnue de systemd, template exclu, ou `systemctl show` n'a pas pu
|
||||
s'exécuter du tout) laisse enabled/active=None — jamais un False
|
||||
fabriqué. Idem lxc_running/lxc_autostart si le conteneur est absent de
|
||||
l'énumération lxc-ls."""
|
||||
resolved_routes = routes if routes is not None else load_routes()
|
||||
|
||||
units: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for m in manifests.values():
|
||||
unit = m.units[0] if m.units else None
|
||||
if unit and unit not in seen and not _TEMPLATE_UNIT_RE.search(unit):
|
||||
seen.add(unit)
|
||||
units.append(unit)
|
||||
|
||||
unit_props: dict[str, dict[str, str]] = {}
|
||||
if units:
|
||||
_, out = run(["systemctl", "show", *units, "-p", "Id", "-p", "UnitFileState",
|
||||
"-p", "ActiveState", "-p", "MainPID", "--no-pager"])
|
||||
if out:
|
||||
unit_props = _parse_systemctl_show_blocks(out)
|
||||
|
||||
lxc_names = {m.lxc for m in manifests.values() if m.runtime == "lxc" and m.lxc}
|
||||
lxc_state: dict[str, tuple[bool | None, bool | None]] = {}
|
||||
if lxc_names:
|
||||
_, out = run(["lxc-ls", "-f"])
|
||||
if out:
|
||||
lxc_state = _parse_lxc_ls_fancy(out)
|
||||
|
||||
result: dict[str, Actual] = {}
|
||||
for mid, m in manifests.items():
|
||||
unit = m.units[0] if m.units else None
|
||||
enabled = active = rss = None
|
||||
if unit:
|
||||
props = unit_props.get(unit)
|
||||
# `props is None` covers all three "couldn't determine" cases at
|
||||
# once: unknown unit, excluded template, or a batch that didn't
|
||||
# execute at all (rc=None, out="") — every one of them must stay
|
||||
# None, never become a fabricated False.
|
||||
if props is not None:
|
||||
ufs = props.get("UnitFileState")
|
||||
enabled = None if ufs is None else ufs not in _DISABLED_UNIT_FILE_STATES
|
||||
astate = props.get("ActiveState")
|
||||
active = None if astate is None else astate == "active"
|
||||
rss = _rss_kb(props.get("MainPID", ""))
|
||||
|
||||
lxc_running = lxc_autostart = None
|
||||
if m.runtime == "lxc" and m.lxc:
|
||||
state = lxc_state.get(m.lxc)
|
||||
if state is not None:
|
||||
lxc_running, lxc_autostart = state
|
||||
|
||||
portal_routed = None
|
||||
if m.portal_domain is not None and resolved_routes is not None:
|
||||
portal_routed = m.portal_domain in resolved_routes
|
||||
|
||||
result[mid] = Actual(enabled=enabled, active=active, lxc_running=lxc_running,
|
||||
lxc_autostart=lxc_autostart, portal_routed=portal_routed,
|
||||
rss_kb=rss)
|
||||
return result
|
||||
|
||||
|
||||
def is_on(a: Actual) -> bool:
|
||||
"""Un module est ON s'il est enabled ET actif. `enabled` seul survit au
|
||||
reboot mais ne sert à rien maintenant ; `active` seul ne survit pas."""
|
||||
|
|
|
|||
|
|
@ -29,9 +29,12 @@ Deux règles structurelles, indépendantes de diff.plan_changes :
|
|||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
|
@ -120,6 +123,140 @@ def _write_pins(path: Path, pins: dict) -> None:
|
|||
_atomic_write(path, "\n".join(lines) + "\n")
|
||||
|
||||
|
||||
def _build_status_payload(manifests: dict, actuals: dict) -> dict:
|
||||
modules = []
|
||||
totals = {"count": 0, "on": 0, "off": 0, "unknown": 0, "rss_kb": 0, "exposed_public": 0}
|
||||
by_category: dict = {}
|
||||
by_runtime: dict = {}
|
||||
by_exposure: dict = {}
|
||||
|
||||
def bump(bucket: dict, key: str, state: str, rss: Optional[int]) -> None:
|
||||
b = bucket.setdefault(key, {"count": 0, "on": 0, "off": 0, "unknown": 0, "rss_kb": 0})
|
||||
b["count"] += 1
|
||||
b[state] += 1
|
||||
if rss:
|
||||
b["rss_kb"] += rss
|
||||
|
||||
for mid, m in sorted(manifests.items()):
|
||||
a = actuals.get(mid, Actual())
|
||||
st = _tri_state(a)
|
||||
modules.append({
|
||||
"id": mid, "category": m.category, "runtime": m.runtime,
|
||||
"exposure": m.exposure, "priority": m.priority,
|
||||
"protected": m.protected, "on": st, "rss_kb": a.rss_kb,
|
||||
})
|
||||
totals["count"] += 1
|
||||
totals[st] += 1
|
||||
if a.rss_kb:
|
||||
totals["rss_kb"] += a.rss_kb
|
||||
if m.exposure == "public":
|
||||
totals["exposed_public"] += 1
|
||||
bump(by_category, m.category, st, a.rss_kb)
|
||||
bump(by_runtime, m.runtime, st, a.rss_kb)
|
||||
bump(by_exposure, m.exposure, st, a.rss_kb)
|
||||
|
||||
return {
|
||||
"modules": modules,
|
||||
"totals": totals,
|
||||
"by_category": by_category,
|
||||
"by_runtime": by_runtime,
|
||||
"by_exposure": by_exposure,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /status cache — pattern "double caching" imposé par le CLAUDE.md du projet
|
||||
# (Performance Patterns) : le calcul (observe_all -> systemctl/lxc-ls) tourne
|
||||
# hors de la boucle asyncio (asyncio.to_thread), jamais dessus. Ce service
|
||||
# tourne sur son propre socket, mais la board a déjà vu un appel bloquant sur
|
||||
# une boucle PARTAGÉE geler ~110 modules (voir aggregator) : même seul,
|
||||
# `observe_all` reste hors-loop par prudence/cohérence avec le reste du parc.
|
||||
#
|
||||
# TTL 60s. Clé = racine de config (str(root)) plutôt qu'une clé globale
|
||||
# unique : chaque test utilise un tmp_path distinct comme racine, donc chaque
|
||||
# test obtient naturellement un cache froid et recalcule sur ses propres
|
||||
# doubles — aucune fuite d'état entre tests, sans changer un seul test
|
||||
# existant. En production il n'y a qu'une racine (/etc/secubox), donc un seul
|
||||
# slot de cache actif.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_STATUS_CACHE_TTL_S = 60.0
|
||||
|
||||
|
||||
class _StatusCacheEntry:
|
||||
__slots__ = ("payload", "computed_at")
|
||||
|
||||
def __init__(self, payload: dict, computed_at: float):
|
||||
self.payload = payload
|
||||
self.computed_at = computed_at
|
||||
|
||||
|
||||
_status_cache: dict[str, _StatusCacheEntry] = {}
|
||||
_status_locks: dict[str, asyncio.Lock] = {}
|
||||
|
||||
|
||||
def _status_lock(key: str) -> asyncio.Lock:
|
||||
lock = _status_locks.get(key)
|
||||
if lock is None:
|
||||
lock = asyncio.Lock()
|
||||
_status_locks[key] = lock
|
||||
return lock
|
||||
|
||||
|
||||
def _compute_status_payload(root: Path) -> dict:
|
||||
"""Bloquant (subprocess via observe_all) — appeler uniquement via
|
||||
asyncio.to_thread, jamais directement dans une coroutine d'endpoint."""
|
||||
mod_dir, _prof_dir, _pins_file, _active_file = _cli._paths(root)
|
||||
manifests = _load_manifests_or_500(mod_dir)
|
||||
routes = load_routes()
|
||||
actuals = _cli._observe_all(manifests, routes)
|
||||
return _build_status_payload(manifests, actuals)
|
||||
|
||||
|
||||
def _with_freshness(entry: _StatusCacheEntry) -> dict:
|
||||
age_s = max(0.0, time.time() - entry.computed_at)
|
||||
return {
|
||||
**entry.payload,
|
||||
"cached_at": datetime.fromtimestamp(entry.computed_at, tz=timezone.utc).isoformat(),
|
||||
"age_s": round(age_s, 1),
|
||||
}
|
||||
|
||||
|
||||
async def _get_status_cached(root: Path) -> dict:
|
||||
key = str(root)
|
||||
entry = _status_cache.get(key)
|
||||
if entry is not None and (time.time() - entry.computed_at) < _STATUS_CACHE_TTL_S:
|
||||
return _with_freshness(entry)
|
||||
|
||||
async with _status_lock(key):
|
||||
# Un autre appel concurrent a pu rafraîchir pendant l'attente du lock.
|
||||
entry = _status_cache.get(key)
|
||||
if entry is not None and (time.time() - entry.computed_at) < _STATUS_CACHE_TTL_S:
|
||||
return _with_freshness(entry)
|
||||
# Cache froid : on calcule une fois plutôt que de renvoyer une
|
||||
# erreur — /status doit toujours répondre, même au tout premier appel.
|
||||
payload = await asyncio.to_thread(_compute_status_payload, root)
|
||||
entry = _StatusCacheEntry(payload=payload, computed_at=time.time())
|
||||
_status_cache[key] = entry
|
||||
return _with_freshness(entry)
|
||||
|
||||
|
||||
async def _status_cache_refresher() -> None:
|
||||
"""Rafraîchissement proactif en arrière-plan (motif CLAUDE.md) : une
|
||||
requête ne devrait quasiment jamais payer le coût du calcul, seulement le
|
||||
tout premier appel après démarrage. Une erreur de rafraîchissement ne
|
||||
doit ni planter le service ni effacer un cache encore valide — on
|
||||
réessaiera au tour suivant."""
|
||||
while True:
|
||||
try:
|
||||
root = _root()
|
||||
payload = await asyncio.to_thread(_compute_status_payload, root)
|
||||
_status_cache[str(root)] = _StatusCacheEntry(payload=payload, computed_at=time.time())
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(_STATUS_CACHE_TTL_S)
|
||||
|
||||
|
||||
def _load_manifests_or_500(mod_dir: Path) -> dict:
|
||||
try:
|
||||
return load_all(mod_dir)
|
||||
|
|
@ -153,52 +290,18 @@ def create_app() -> FastAPI:
|
|||
redoc_url=None,
|
||||
)
|
||||
|
||||
@app.on_event("startup")
|
||||
async def _start_status_refresher() -> None:
|
||||
# Fire-and-forget background task (motif CLAUDE.md « Performance
|
||||
# Patterns — Double Caching »). N'est pas nécessaire à la correction
|
||||
# de /status (voir _get_status_cached : cache froid = calcul direct,
|
||||
# jamais d'erreur) — seulement à garder le cache chaud en continu sur
|
||||
# le service réel.
|
||||
asyncio.create_task(_status_cache_refresher())
|
||||
|
||||
@app.get("/api/v1/profiles/status")
|
||||
async def get_status(_claims=Depends(require_jwt)):
|
||||
root = _root()
|
||||
mod_dir, _prof_dir, _pins_file, _active_file = _cli._paths(root)
|
||||
manifests = _load_manifests_or_500(mod_dir)
|
||||
routes = load_routes()
|
||||
actuals = _cli._observe_all(manifests, routes)
|
||||
|
||||
modules = []
|
||||
totals = {"count": 0, "on": 0, "off": 0, "unknown": 0, "rss_kb": 0, "exposed_public": 0}
|
||||
by_category: dict = {}
|
||||
by_runtime: dict = {}
|
||||
by_exposure: dict = {}
|
||||
|
||||
def bump(bucket: dict, key: str, state: str, rss: Optional[int]) -> None:
|
||||
b = bucket.setdefault(key, {"count": 0, "on": 0, "off": 0, "unknown": 0, "rss_kb": 0})
|
||||
b["count"] += 1
|
||||
b[state] += 1
|
||||
if rss:
|
||||
b["rss_kb"] += rss
|
||||
|
||||
for mid, m in sorted(manifests.items()):
|
||||
a = actuals.get(mid, Actual())
|
||||
st = _tri_state(a)
|
||||
modules.append({
|
||||
"id": mid, "category": m.category, "runtime": m.runtime,
|
||||
"exposure": m.exposure, "priority": m.priority,
|
||||
"protected": m.protected, "on": st, "rss_kb": a.rss_kb,
|
||||
})
|
||||
totals["count"] += 1
|
||||
totals[st] += 1
|
||||
if a.rss_kb:
|
||||
totals["rss_kb"] += a.rss_kb
|
||||
if m.exposure == "public":
|
||||
totals["exposed_public"] += 1
|
||||
bump(by_category, m.category, st, a.rss_kb)
|
||||
bump(by_runtime, m.runtime, st, a.rss_kb)
|
||||
bump(by_exposure, m.exposure, st, a.rss_kb)
|
||||
|
||||
return {
|
||||
"modules": modules,
|
||||
"totals": totals,
|
||||
"by_category": by_category,
|
||||
"by_runtime": by_runtime,
|
||||
"by_exposure": by_exposure,
|
||||
}
|
||||
return await _get_status_cached(_root())
|
||||
|
||||
@app.get("/api/v1/profiles/profiles")
|
||||
async def list_profiles(_claims=Depends(require_jwt)):
|
||||
|
|
|
|||
|
|
@ -1,3 +1,19 @@
|
|||
secubox-profiles (0.2.1-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* perf : /status batché — une seule commande `systemctl show` pour toutes
|
||||
les units natives + une seule énumération `lxc-ls -f`, au lieu de 3
|
||||
sous-process par module (observe.observe_all, mesuré 46s -> <1s sur 187
|
||||
modules). Cache TTL 60s hors-loop (asyncio.to_thread) côté API, avec
|
||||
rafraîchissement en arrière-plan ; cache froid = calcul direct, jamais
|
||||
d'erreur. /status expose désormais `cached_at`/`age_s`.
|
||||
* observe() module par module reste la référence ; observe_all() est le
|
||||
chemin rapide, pas un remplacement — unit/conteneur absent du batch
|
||||
reste enabled/active/lxc_running/lxc_autostart=None (jamais un False
|
||||
fabriqué), y compris pour les units *@.service (templates) qui feraient
|
||||
échouer tout le batch si elles y entraient.
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Fri, 17 Jul 2026 12:00:00 +0200
|
||||
|
||||
secubox-profiles (0.2.0-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* webui : API web (socket dédié /run/secubox/profiles.sock, service propre,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import json
|
|||
from pathlib import Path
|
||||
|
||||
from api.manifest import Manifest
|
||||
from api.observe import Actual, is_on, load_routes, observe
|
||||
from api.observe import Actual, is_on, load_routes, observe, observe_all
|
||||
|
||||
|
||||
def fake_run(table):
|
||||
|
|
@ -166,6 +166,184 @@ def test_observe_portal_routed_none_when_routes_undeterminable(monkeypatch):
|
|||
assert a.portal_routed is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# observe_all() — chemin batché (une commande systemctl show, une lxc-ls)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _show_block(unit: str, *, enabled="enabled", active="active", pid="0",
|
||||
order=("Id", "UnitFileState", "ActiveState", "MainPID")) -> str:
|
||||
values = {"Id": unit, "UnitFileState": enabled, "ActiveState": active, "MainPID": pid}
|
||||
return "\n".join(f"{k}={values[k]}" for k in order)
|
||||
|
||||
|
||||
def _lxc_ls_fancy(rows: list[tuple[str, str, str]]) -> str:
|
||||
"""Rebuild `lxc-ls -f` fixed-width column output (NAME/STATE/AUTOSTART/…)
|
||||
with the SAME column widths as the real board output (captured via ssh:
|
||||
`NAME STATE AUTOSTART GROUPS IPV4 IPV6 UNPRIVILEGED`),
|
||||
so the parser is exercised against realistic alignment instead of
|
||||
hand-typed spacing that may not match header offsets."""
|
||||
header = (f"{'NAME':<20}{'STATE':<8}{'AUTOSTART':<10}{'GROUPS':<7}"
|
||||
f"{'IPV4':<13}{'IPV6':<5}UNPRIVILEGED")
|
||||
lines = [header]
|
||||
for name, state, autostart in rows:
|
||||
lines.append(f"{name:<20}{state:<8}{autostart:<10}{'-':<7}{'-':<13}{'-':<5}true")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def test_observe_all_batches_a_single_systemctl_call_for_all_units():
|
||||
# LE point de la bascule : une seule commande `systemctl show`, pas une
|
||||
# boucle observe() par module (560 sous-process sur 187 modules -> 46s).
|
||||
calls = []
|
||||
|
||||
def run(argv):
|
||||
calls.append(argv)
|
||||
if argv[:2] == ["systemctl", "show"]:
|
||||
blocks = [_show_block(u) for u in
|
||||
("secubox-a.service", "secubox-b.service", "secubox-c.service")]
|
||||
return 0, "\n\n".join(blocks)
|
||||
return 1, ""
|
||||
|
||||
manifests = {
|
||||
mid: Manifest(id=mid, category="media", runtime="native", exposure="lan",
|
||||
units=(f"secubox-{mid}.service",))
|
||||
for mid in ("a", "b", "c")
|
||||
}
|
||||
actuals = observe_all(manifests, run=run, routes=set())
|
||||
|
||||
show_calls = [c for c in calls if c[:2] == ["systemctl", "show"]]
|
||||
assert len(show_calls) == 1, "must be ONE batched systemctl show, not one per module"
|
||||
for u in ("secubox-a.service", "secubox-b.service", "secubox-c.service"):
|
||||
assert u in show_calls[0]
|
||||
for mid in ("a", "b", "c"):
|
||||
assert actuals[mid].enabled is True and actuals[mid].active is True
|
||||
|
||||
|
||||
def test_observe_all_unit_missing_from_batch_output_is_none_not_false():
|
||||
# Une unit absente du bloc renvoyé (unit inconnue de systemd, ou batch
|
||||
# qui n'a couvert qu'une partie) doit rester indéterminée — jamais un
|
||||
# False fabriqué, qui déclencherait une fausse décision "à éteindre".
|
||||
def run(argv):
|
||||
if argv[:2] == ["systemctl", "show"]:
|
||||
return 0, _show_block("secubox-lyrion.service")
|
||||
return 1, ""
|
||||
|
||||
present = Manifest(id="lyrion", category="media", runtime="native", exposure="lan",
|
||||
units=("secubox-lyrion.service",))
|
||||
missing = Manifest(id="ghost", category="media", runtime="native", exposure="lan",
|
||||
units=("secubox-ghost-unit.service",))
|
||||
actuals = observe_all({"lyrion": present, "ghost": missing}, run=run, routes=set())
|
||||
|
||||
assert actuals["lyrion"].enabled is True and actuals["lyrion"].active is True
|
||||
assert actuals["ghost"].enabled is None
|
||||
assert actuals["ghost"].active is None
|
||||
|
||||
|
||||
def test_observe_all_excludes_bare_template_units_from_the_batch():
|
||||
# secubox-toolbox-ng-worker@.service (bare template, no instance) crashes
|
||||
# `systemctl show` for the WHOLE batch when included (verified on the
|
||||
# board: aborts after the first bad unit, dropping everything after it in
|
||||
# argv). It must never reach argv, and its own state stays unknown.
|
||||
calls = []
|
||||
|
||||
def run(argv):
|
||||
calls.append(argv)
|
||||
if argv[:2] == ["systemctl", "show"]:
|
||||
return 0, _show_block("secubox-lyrion.service")
|
||||
return 1, ""
|
||||
|
||||
normal = Manifest(id="lyrion", category="media", runtime="native", exposure="lan",
|
||||
units=("secubox-lyrion.service",))
|
||||
template = Manifest(id="worker", category="infra", runtime="native", exposure="internal",
|
||||
units=("secubox-toolbox-ng-worker@.service",))
|
||||
actuals = observe_all({"lyrion": normal, "worker": template}, run=run, routes=set())
|
||||
|
||||
show_call = next(c for c in calls if c[:2] == ["systemctl", "show"])
|
||||
assert "secubox-toolbox-ng-worker@.service" not in show_call
|
||||
assert actuals["worker"].enabled is None
|
||||
assert actuals["worker"].active is None
|
||||
assert actuals["lyrion"].enabled is True # unaffected
|
||||
|
||||
|
||||
def test_observe_all_parses_block_regardless_of_property_order():
|
||||
# Observé sur la board : MainPID peut apparaître AVANT Id dans un bloc.
|
||||
# Le parsing doit être par clé, jamais par position.
|
||||
def run(argv):
|
||||
if argv[:2] == ["systemctl", "show"]:
|
||||
return 0, _show_block(
|
||||
"secubox-lyrion.service", pid="4242",
|
||||
order=("MainPID", "ActiveState", "Id", "UnitFileState"))
|
||||
return 1, ""
|
||||
|
||||
m = Manifest(id="lyrion", category="media", runtime="native", exposure="lan",
|
||||
units=("secubox-lyrion.service",))
|
||||
actuals = observe_all({"lyrion": m}, run=run, routes=set())
|
||||
assert actuals["lyrion"].enabled is True
|
||||
assert actuals["lyrion"].active is True
|
||||
|
||||
|
||||
def test_observe_all_static_unit_file_state_is_enabled_true():
|
||||
# `systemctl is-enabled` exits 0 (enabled) for UnitFileState=static — pas
|
||||
# seulement pour "enabled" littéralement (25 des 183 units réelles de la
|
||||
# board sont "static"). Un mapping naïf ufs=="enabled" mentirait ici.
|
||||
def run(argv):
|
||||
if argv[:2] == ["systemctl", "show"]:
|
||||
return 0, _show_block("secubox-adblock-sync.service", enabled="static")
|
||||
return 1, ""
|
||||
|
||||
m = Manifest(id="adblock-sync", category="network", runtime="native", exposure="lan",
|
||||
units=("secubox-adblock-sync.service",))
|
||||
actuals = observe_all({"adblock-sync": m}, run=run, routes=set())
|
||||
assert actuals["adblock-sync"].enabled is True
|
||||
|
||||
|
||||
def test_observe_all_lxc_is_a_single_enumeration_and_missing_container_is_none():
|
||||
calls = []
|
||||
|
||||
def run(argv):
|
||||
calls.append(argv)
|
||||
if argv[:2] == ["systemctl", "show"]:
|
||||
blocks = [_show_block(u) for u in
|
||||
("secubox-peertube.service", "secubox-ghost.service")]
|
||||
return 0, "\n\n".join(blocks)
|
||||
if argv == ["lxc-ls", "-f"]:
|
||||
return 0, _lxc_ls_fancy([("peertube", "RUNNING", "1")])
|
||||
return 1, ""
|
||||
|
||||
peertube = Manifest(id="peertube", category="media", runtime="lxc", exposure="public",
|
||||
units=("secubox-peertube.service",), lxc="peertube")
|
||||
ghost = Manifest(id="ghost", category="media", runtime="lxc", exposure="lan",
|
||||
units=("secubox-ghost.service",), lxc="ghost-container")
|
||||
actuals = observe_all({"peertube": peertube, "ghost": ghost}, run=run, routes=set())
|
||||
|
||||
lxc_calls = [c for c in calls if c[0] == "lxc-ls"]
|
||||
assert len(lxc_calls) == 1, "must be ONE lxc-ls enumeration, not one lxc-info per container"
|
||||
assert actuals["peertube"].lxc_running is True
|
||||
assert actuals["peertube"].lxc_autostart is True
|
||||
# ghost-container absent from lxc-ls output -> stays unknown, not False.
|
||||
assert actuals["ghost"].lxc_running is None
|
||||
assert actuals["ghost"].lxc_autostart is None
|
||||
|
||||
|
||||
def test_observe_all_matches_observe_for_a_single_module(monkeypatch):
|
||||
# observe_all() doit produire le même résultat que observe() pour un
|
||||
# module donné — seul le coût change, pas le contrat.
|
||||
run_single = fake_run({
|
||||
("systemctl", "is-enabled", "secubox-lyrion.service"): (0, "enabled\n"),
|
||||
("systemctl", "is-active", "secubox-lyrion.service"): (0, "active\n"),
|
||||
("systemctl", "show", "secubox-lyrion.service", "-p", "MainPID", "--value"): (0, "0\n"),
|
||||
})
|
||||
single = observe(NATIVE, run=run_single, routes=set())
|
||||
|
||||
def run_batch(argv):
|
||||
if argv[:2] == ["systemctl", "show"]:
|
||||
return 0, _show_block("secubox-lyrion.service")
|
||||
return 1, ""
|
||||
|
||||
batched = observe_all({"lyrion": NATIVE}, run=run_batch, routes=set())["lyrion"]
|
||||
assert batched.enabled == single.enabled
|
||||
assert batched.active == single.active
|
||||
|
||||
|
||||
def test_load_routes_untraversable_parent_is_none_not_crash(tmp_path):
|
||||
"""Cas RÉEL de la board : /etc/secubox/waf est 0750 root:root alors que
|
||||
haproxy-routes.json est 0644 — le fichier est lisible, le répertoire non
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user