Merge feature/498-asgi-migrate : Phase 7.D ASGI consolidation (ref #498)

103 → 41 uvicorn procs, 3499 → 1984 MB RSS, 153 MB → 2.7 GiB free RAM.
117 modules now mount in a single secubox-aggregator process. Includes :
  * new secubox-aggregator package with synthetic-package loader
  * migration helper that rewrites all 3 nginx config locations and
    preserves /api/v1/<name>/ prefix
  * follow-up fixes : magicmirror ships routers/, openclaw chowns to
    secubox, migration helper skips ghost secubox-* duplicate dirs

packages/secubox-aggregator/aggregator/main.py

@@ -74,14 +74,31 @@ SECUBOX_LIB = Path("/usr/lib/secubox")
 def _load_app_from_path(name: str) -> FastAPI | None:
     """Locate the module's FastAPI by walking /usr/lib/secubox/<name>/.
 
-    SecuBox modules ship their FastAPI as `api/main.py` (and the per-module
-    systemd unit runs `uvicorn api.main:app` from cwd=/usr/lib/secubox/<name>).
-    They are not installed as Python packages, so importlib.import_module
-    can't find them — we load by absolute path via spec_from_file_location.
+    SecuBox modules ship their FastAPI as `api/main.py`. They are not
+    installed as Python packages — the per-module systemd unit runs
+    `uvicorn api.main:app` from cwd=/usr/lib/secubox/<name>, so module
+    code typically uses one of these import styles :
 
-    The aggregator adds the module's directory to sys.path TEMPORARILY so
-    relative imports inside `api/main.py` work (e.g. `from .deps import ...`).
-    The path is popped after loading to avoid cross-module shadowing.
+        from .deps import X         # relative — needs __package__ set
+        from api.deps import X      # absolute — needs cwd on sys.path
+        from api import deps        # mixed
+        import api.deps             # bare absolute
+
+    To make all four work for many modules in the SAME interpreter we :
+
+    1. Synthesise a per-module parent package `sbx_pkg_<safe>` whose
+       __path__ is `<name>/api`. Loading api/main.py as `<pkg>.main`
+       inside that synthetic package makes `from .deps` resolve to
+       `sbx_pkg_<safe>.deps`, served from the right `api/` directory.
+    2. Pop any cached `sys.modules["api"]` / `sys.modules["api.*"]` left
+       behind by previously-loaded modules and put <name> on sys.path —
+       so absolute `from api.X` resolves freshly to THIS module's `api/`.
+    3. After exec, drop the freshly-loaded `api*` entries again so the
+       next module starts with a clean slate (and remove the sys.path
+       entry to prevent cross-module bleed).
+
+    Both styles now work per-module without one module's `api/`
+    shadowing another's — the bug that hit haproxy after auth had loaded.
     """
     import importlib.util as _util
 
@@ -91,21 +108,60 @@ def _load_app_from_path(name: str) -> FastAPI | None:
     ]
     target = next((p for p in candidates if p.exists()), None)
     if target is None:
-        _LOAD_ERRORS[name] = f"main.py not found under {SECUBOX_LIB}/{{{name},{name.replace('-','_')}}}/api/"
+        _LOAD_ERRORS[name] = (
+            f"main.py not found under "
+            f"{SECUBOX_LIB}/{{{name},{name.replace('-', '_')}}}/api/"
+        )
         return None
 
-    mod_dir = str(target.parent.parent)  # /usr/lib/secubox/<name>
-    spec = _util.spec_from_file_location(f"sbx_mod_{name.replace('-', '_')}", target)
+    api_dir = target.parent           # /usr/lib/secubox/<name>/api
+    mod_dir = str(api_dir.parent)     # /usr/lib/secubox/<name>
+    safe = name.replace("-", "_")
+    pkg_name = f"sbx_pkg_{safe}"
+    main_name = f"{pkg_name}.main"
+
+    # ── (1) Synthetic parent package so `from .deps import X` works ──
+    pkg_init = api_dir / "__init__.py"
+    if pkg_init.exists():
+        pkg_spec = _util.spec_from_file_location(
+            pkg_name, pkg_init,
+            submodule_search_locations=[str(api_dir)],
+        )
+    else:
+        pkg_spec = _util.spec_from_loader(pkg_name, loader=None, is_package=True)
+        pkg_spec.submodule_search_locations = [str(api_dir)]
+    if pkg_spec is None:
+        _LOAD_ERRORS[name] = "could not build synthetic parent package spec"
+        return None
+    pkg_mod = _util.module_from_spec(pkg_spec)
+    pkg_mod.__path__ = [str(api_dir)]
+
+    # ── (2) Isolate the `api` namespace from prior module loads ──
+    for k in list(sys.modules):
+        if k == "api" or k.startswith("api."):
+            del sys.modules[k]
+    sys.path.insert(0, mod_dir)
+    sys.modules[pkg_name] = pkg_mod
+    if pkg_init.exists() and pkg_spec.loader is not None:
+        try:
+            pkg_spec.loader.exec_module(pkg_mod)
+        except Exception as e:
+            _LOAD_ERRORS[name] = f"exec_module(parent): {type(e).__name__}: {e}"
+            sys.modules.pop(pkg_name, None)
+            try:
+                sys.path.remove(mod_dir)
+            except ValueError:
+                pass
+            return None
+
+    # ── (3) Load api/main.py as `<pkg>.main` so it inherits __package__ ──
+    spec = _util.spec_from_file_location(main_name, target)
     if spec is None or spec.loader is None:
         _LOAD_ERRORS[name] = f"spec_from_file_location returned None for {target}"
         return None
-
     mod = _util.module_from_spec(spec)
-    # Inject into sys.modules under a unique name so reloading/dependent
-    # imports inside the module find it.
-    sys.modules[spec.name] = mod
-    # Add the module's parent dir to sys.path so its relative imports work.
-    sys.path.insert(0, mod_dir)
+    mod.__package__ = pkg_name
+    sys.modules[main_name] = mod
     try:
         spec.loader.exec_module(mod)
     except Exception as e:
@@ -116,6 +172,11 @@ def _load_app_from_path(name: str) -> FastAPI | None:
             sys.path.remove(mod_dir)
         except ValueError:
             pass
+        # Drop any `api*` cache this load produced so the next module gets
+        # its own fresh `api` namespace.
+        for k in list(sys.modules):
+            if k == "api" or k.startswith("api."):
+                del sys.modules[k]
 
     sub_app = getattr(mod, "app", None)
     if not isinstance(sub_app, FastAPI):

packages/secubox-aggregator/debian/postinst

@@ -16,6 +16,10 @@ case "$1" in
     systemctl daemon-reload
     systemctl enable secubox-aggregator.service
     systemctl start secubox-aggregator.service || true
+
+    echo "secubox-aggregator: to migrate all installed SecuBox modules into"
+    echo "  the aggregator (replaces per-module uvicorn processes) run :"
+    echo "    sudo /usr/sbin/secubox-aggregator-migrate"
     ;;
 esac
 #DEBHELPER#

packages/secubox-aggregator/debian/rules

@@ -14,3 +14,6 @@ override_dh_auto_install:
 	install -d $(CURDIR)/debian/secubox-aggregator/lib/systemd/system
 	install -m 644 systemd/secubox-aggregator.service \
 	  $(CURDIR)/debian/secubox-aggregator/lib/systemd/system/
+	install -d $(CURDIR)/debian/secubox-aggregator/usr/sbin
+	install -m 755 sbin/secubox-aggregator-migrate \
+	  $(CURDIR)/debian/secubox-aggregator/usr/sbin/

packages/secubox-aggregator/sbin/secubox-aggregator-migrate

@@ -0,0 +1,224 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: LicenseRef-CMSD-1.0
+# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
+
+# SecuBox-Deb :: secubox-aggregator-migrate
+# Phase 7 (#496) — automate cutover from per-module uvicorn to aggregator.
+#
+# Steps :
+#   1. Discover all /usr/lib/secubox/<name>/api/main.py modules
+#   2. Generate /etc/secubox/aggregator.toml with the full list
+#   3. Restart aggregator and capture which modules mounted successfully
+#   4. For each mounted module :
+#        - Switch nginx proxy_pass to /run/secubox/aggregator.sock
+#        - Stop + disable secubox-<name>.service
+#   5. Reload nginx
+#   6. Report savings
+#
+# Idempotent. Operator can re-run after package upgrades that add modules.
+
+set -euo pipefail
+readonly MODULE="secubox-aggregator-migrate"
+readonly VERSION="0.1.0"
+
+log() { printf '[%s] %s\n' "$MODULE" "$*" >&2; }
+err() { printf '[%s] ERROR: %s\n' "$MODULE" "$*" >&2; exit 1; }
+
+[ -d /usr/lib/secubox ] || err "no /usr/lib/secubox — SecuBox modules not installed"
+[ -d /run/secubox ] || err "no /run/secubox runtime dir — secubox-core required"
+command -v curl >/dev/null || err "curl required"
+
+log "Phase 7 ASGI migration — v${VERSION}"
+
+# ── Step 1+2 : Discover modules + write aggregator.toml ──────────────────
+#
+# Older packaging variants installed under /usr/lib/secubox/secubox-<name>/
+# (with the package-name prefix) before being superseded by the canonical
+# unprefixed /usr/lib/secubox/<name>/. Both directories often coexist on
+# upgraded systems. We filter out the `secubox-*` prefixed siblings here :
+# their FastAPIs are byte-identical with the unprefixed ones, so mounting
+# both would just waste memory and create ghost openapi.json entries.
+TOML_TMP=$(mktemp)
+{
+    echo "# Phase 7 (#496) — auto-generated by ${MODULE}"
+    echo "# Run secubox-aggregator-migrate to refresh after installing new modules."
+    echo "modules = ["
+    for f in /usr/lib/secubox/*/api/main.py; do
+        [ -f "$f" ] || continue
+        m=$(basename "$(dirname "$(dirname "$f")")")
+        case "$m" in
+            secubox-*)
+                # legacy prefixed dir — canonical unprefixed twin wins
+                stripped="${m#secubox-}"
+                if [ -f "/usr/lib/secubox/${stripped}/api/main.py" ]; then
+                    log "skip ${m} (duplicate of ${stripped})"
+                    continue
+                fi
+                ;;
+        esac
+        printf '    "%s",\n' "$m"
+    done
+    echo "]"
+} > "$TOML_TMP"
+
+DISCOVERED=$(grep -c '^    "' "$TOML_TMP" || echo 0)
+log "discovered ${DISCOVERED} modules"
+
+install -d -m 0755 /etc/secubox
+install -m 0644 "$TOML_TMP" /etc/secubox/aggregator.toml
+rm -f "$TOML_TMP"
+
+# ── Step 3 : Restart aggregator + wait for mount completion ──────────────
+systemctl restart secubox-aggregator
+sleep 8
+
+# ── Step 4 : Grab mounted module list from /health ───────────────────────
+HEALTH=$(curl -s --unix-socket /run/secubox/aggregator.sock http://localhost/health 2>&1) || \
+    err "aggregator /health unreachable"
+
+MOUNTED=$(printf '%s' "$HEALTH" | python3 -c "
+import sys, json
+try:
+    d = json.loads(sys.stdin.read())
+    print(' '.join(d.get('mounted', [])))
+except Exception as e:
+    print('', end='')
+    sys.stderr.write(f'parse: {e}\n')
+")
+
+if [ -z "$MOUNTED" ]; then
+    err "no modules mounted in aggregator — check 'journalctl -u secubox-aggregator'"
+fi
+
+MOUNTED_COUNT=$(printf '%s' "$MOUNTED" | wc -w)
+log "aggregator mounted ${MOUNTED_COUNT} modules — proceeding with cutover"
+
+# ── Step 5 : nginx proxy_pass switch ─────────────────────────────────────
+#
+# Per-module FastAPIs used to listen on /run/secubox/<name>.sock and the
+# per-module nginx route did :
+#     proxy_pass http://unix:.../<name>.sock:/;
+# The trailing `:/` strips the matching location prefix, so the upstream
+# saw a bare `/foo` URI and matched its top-level routes.
+#
+# The aggregator mounts each sub-app under /api/v1/<name>, so it expects
+# the FULL prefix to arrive intact. Rewrite must :
+#   1. Point at /run/secubox/aggregator.sock
+#   2. Preserve the /api/v1/<name>/ prefix on the upstream URI
+#
+# Three locations need patching :
+#   /etc/nginx/secubox.d/<name>.conf         ← legacy module-side drop-ins
+#   /etc/nginx/secubox-routes.d/<name>.conf  ← newer per-module route files
+#   /etc/nginx/sites-enabled/webui.conf      ← inline per-module location blocks
+#                                              + the /api/ catch-all fallback
+# Plus one special case : hub used to listen on TCP 127.0.0.1:8001.
+SWITCHED=0
+NGINX_SKIPPED=0
+
+_rewrite_file() {
+    local f="$1" name="$2"
+    [ -f "$f" ] || return 0
+    case "$f" in *.bak*|*.dpkg*|*.disabled*) return 0 ;; esac
+    # First-run backup
+    [ -f "${f}.bak.phase7" ] || cp "$f" "${f}.bak.phase7"
+    # Every per-module unix socket → aggregator + /api/v1/<name>/ prefix
+    sed -i -E \
+      -e "s|proxy_pass http://unix:/run/secubox/${name}\.sock(:/[^;]*)?;|proxy_pass http://unix:/run/secubox/aggregator.sock:/api/v1/${name}/;|g" \
+      -e "s|proxy_pass http://unix:/run/secubox/aggregator\.sock:/;|proxy_pass http://unix:/run/secubox/aggregator.sock:/api/v1/${name}/;|g" \
+      "$f"
+}
+
+for name in $MOUNTED; do
+    found_any=0
+    for dir in /etc/nginx/secubox.d /etc/nginx/secubox-routes.d; do
+        cfg="${dir}/${name}.conf"
+        if [ -f "$cfg" ]; then
+            _rewrite_file "$cfg" "$name"
+            found_any=1
+        fi
+    done
+    if [ "$found_any" -eq 0 ]; then
+        NGINX_SKIPPED=$((NGINX_SKIPPED + 1))
+        continue
+    fi
+    SWITCHED=$((SWITCHED + 1))
+done
+
+# Hub special case : it used TCP 127.0.0.1:8001 (not a unix socket). Replace
+# every such line in secubox.d/hub.conf and in webui.conf inline blocks.
+HUB_CFG=/etc/nginx/secubox.d/hub.conf
+if [ -f "$HUB_CFG" ]; then
+    [ -f "${HUB_CFG}.bak.phase7" ] || cp "$HUB_CFG" "${HUB_CFG}.bak.phase7"
+    sed -i -E "s|proxy_pass http://127\.0\.0\.1:8001/?;|proxy_pass http://unix:/run/secubox/aggregator.sock:/api/v1/hub/;|g" "$HUB_CFG"
+fi
+
+# webui.conf : inline `location /api/v1/<name>/` blocks + `/api/` catch-all
+WEBUI=/etc/nginx/sites-enabled/webui.conf
+if [ -f "$WEBUI" ]; then
+    [ -f "/root/webui.conf.bak.phase7" ] || cp "$WEBUI" /root/webui.conf.bak.phase7
+    # NOTE: backup stored under /root, not next to $WEBUI, because nginx
+    # auto-includes anything matching sites-enabled/* and would crash on
+    # duplicate-default-server.
+    for name in $MOUNTED; do
+        sed -i -E \
+          -e "s|proxy_pass http://unix:/run/secubox/${name}\.sock(:/[^;]*)?;|proxy_pass http://unix:/run/secubox/aggregator.sock:/api/v1/${name}/;|g" \
+          "$WEBUI"
+    done
+    # Catch-all : keep the full URI by removing any rewrite on /api/
+    sed -i -E "/location \/api\/ \{/,/\}/ s|proxy_pass http://127\.0\.0\.1:8001;|proxy_pass http://unix:/run/secubox/aggregator.sock;|" "$WEBUI"
+fi
+
+log "nginx routes switched : ${SWITCHED} (skipped ${NGINX_SKIPPED} with no route file)"
+
+# Validate + reload nginx
+if ! nginx -t 2>/dev/null; then
+    err "nginx config check failed — review /etc/nginx/{secubox.d,secubox-routes.d,sites-enabled/webui.conf}"
+fi
+systemctl reload nginx
+log "nginx reloaded"
+
+# ── Step 6 : Disable migrated per-module systemd units ───────────────────
+STOPPED=0
+for name in $MOUNTED; do
+    svc=secubox-${name}.service
+    systemctl is-active "$svc" >/dev/null 2>&1 || continue
+    systemctl stop "$svc" 2>/dev/null || true
+    systemctl disable "$svc" 2>/dev/null || true
+    STOPPED=$((STOPPED + 1))
+done
+log "stopped + disabled : ${STOPPED} per-module services"
+
+# ── Step 7 : Final report ───────────────────────────────────────────────
+cat <<REPORT
+
+────────────────────────────────────────────────────────────────────
+Phase 7 ASGI migration complete
+
+  Modules discovered           : ${DISCOVERED}
+  Modules mounted (aggregator) : ${MOUNTED_COUNT}
+  Modules failed to mount      : $((DISCOVERED - MOUNTED_COUNT))
+  Nginx routes switched        : ${SWITCHED}
+  Per-module services stopped  : ${STOPPED}
+
+Tip : a reboot is recommended to recover glibc memory fragmentation
+left by the previous per-module processes :
+
+    sudo systemctl reboot
+
+Failed mount reasons (relative imports etc.) :
+
+    curl --unix-socket /run/secubox/aggregator.sock \\
+         http://localhost/health | python3 -m json.tool
+
+Rollback per module if needed :
+
+    # restore whichever of these exist
+    cp /etc/nginx/secubox.d/<name>.conf.bak.phase7 \\
+       /etc/nginx/secubox.d/<name>.conf 2>/dev/null
+    cp /etc/nginx/secubox-routes.d/<name>.conf.bak.phase7 \\
+       /etc/nginx/secubox-routes.d/<name>.conf 2>/dev/null
+    cp /root/webui.conf.bak.phase7 /etc/nginx/sites-enabled/webui.conf
+    systemctl enable --now secubox-<name>.service
+    systemctl reload nginx
+────────────────────────────────────────────────────────────────────
+REPORT

packages/secubox-magicmirror/debian/rules

@@ -5,7 +5,10 @@
 override_dh_auto_install:
 	install -d $(CURDIR)/debian/secubox-magicmirror/usr/lib/secubox/magicmirror/api
 	install -m 644 api/main.py $(CURDIR)/debian/secubox-magicmirror/usr/lib/secubox/magicmirror/api/
-	touch $(CURDIR)/debian/secubox-magicmirror/usr/lib/secubox/magicmirror/api/__init__.py
+	install -m 644 api/__init__.py $(CURDIR)/debian/secubox-magicmirror/usr/lib/secubox/magicmirror/api/
+	install -d $(CURDIR)/debian/secubox-magicmirror/usr/lib/secubox/magicmirror/api/routers
+	install -m 644 api/routers/__init__.py $(CURDIR)/debian/secubox-magicmirror/usr/lib/secubox/magicmirror/api/routers/
+	install -m 644 api/routers/mmpm.py $(CURDIR)/debian/secubox-magicmirror/usr/lib/secubox/magicmirror/api/routers/
 	install -d $(CURDIR)/debian/secubox-magicmirror/usr/share/secubox/www/magicmirror
 	install -m 644 www/magicmirror/index.html $(CURDIR)/debian/secubox-magicmirror/usr/share/secubox/www/magicmirror/
 	install -d $(CURDIR)/debian/secubox-magicmirror/etc/nginx/secubox.d

packages/secubox-openclaw/debian/postinst

@@ -1,11 +1,18 @@
 #!/bin/sh
 set -e
 if [ "$1" = "configure" ]; then
-    # Create data directories
+    # Create data directories. The openclaw FastAPI imports a module that
+    # touches /var/lib/secubox/openclaw/scans at import time, so the user
+    # that loads it MUST be able to traverse the parent dir. Under
+    # Phase 7.D (secubox-aggregator) the loader runs as `secubox`, not
+    # root — so the parent dir is 0755 and owned by `secubox:secubox`.
     mkdir -p /var/lib/secubox/openclaw/scans
     mkdir -p /var/cache/secubox/openclaw
-    chown -R root:root /var/lib/secubox/openclaw
-    chmod 750 /var/lib/secubox/openclaw
+    if getent passwd secubox >/dev/null; then
+        chown -R secubox:secubox /var/lib/secubox/openclaw /var/cache/secubox/openclaw
+    fi
+    chmod 0755 /var/lib/secubox/openclaw
+    chmod 0750 /var/lib/secubox/openclaw/scans
 
     # Enable and start service
     systemctl daemon-reload