Merge pull request #567 from CyberMind-FR/feature/566-r3-r4-silent-ad-banner-ghoster-economisa

Modular mitm filters + R3+/R4 silent ad/banner ghoster (#566)

packages/secubox-toolbox/debian/changelog

@@ -1,3 +1,26 @@
+secubox-toolbox (2.6.23-1~bookworm1) bookworm; urgency=medium
+
+  * Modular mitm filters + R3+/R4 silent ad/banner ghoster (#566).
+      - secubox_toolbox/filters.py : single filter config
+        (/etc/secubox/toolbox/filters.json), 5 s cached, toggled live from
+        the toolbox WebUI — banner / protective(off|alert|spoof) / ad_ghost /
+        ad_ghost_block / per-category cosmetics. No restart to flip.
+      - mitmproxy_addons/ad_ghost.py : for R3+/R4 (10.99.1.0/24) only —
+        204 known ad/tracker hosts (real bandwidth save) + inject ad-hiding
+        CSS (ads / consent-nag / newsletter / social-widget categories) into
+        HTML. Tallies blocked_requests + bytes_saved_est + pages_cleaned →
+        /run/secubox/ghost.json. Wired into the mitm-wg launcher.
+      - inject_banner : reads the filter (banner on/off), shows the ghost
+        quick-stats (shield N + ~X Ko), relabels status inspected→protected
+        on R3+/R4. protective_mode level now sourced from the filter config
+        (env fallback).
+      - api: GET/POST /admin/filters, GET /admin/ghost, GET /admin/filters/ui
+        (toggle panel). postinst seeds filters.json (preserved on upgrade).
+    Doctrine: opt-in via filter, logged, reversible ; cosmetic ghosting only
+    on the opted-in R3+/R4 tiers, 1st-party pages stay usable.
+
+ -- Gerald KERMA <devel@cybermind.fr>  Sat, 13 Jun 2026 18:00:00 +0200
+
 secubox-toolbox (2.6.22-1~bookworm1) bookworm; urgency=medium
 
   * detect_antibot: deployment vs challenge, response-level signals (#564,

packages/secubox-toolbox/debian/postinst

@@ -42,6 +42,23 @@ case "$1" in
       chown root:secubox-toolbox /etc/secubox/toolbox/ca/ca.pem /etc/secubox/toolbox/ca/key.pem 2>/dev/null || true
     fi
 
+    # #566 : modular filter config (toolbox WebUI toggles). Seed once;
+    # preserve operator edits on upgrade. Writable by the portal user.
+    install -d -m 0750 -o root -g secubox-toolbox /etc/secubox/toolbox 2>/dev/null || true
+    if [ ! -f /etc/secubox/toolbox/filters.json ]; then
+      cat > /etc/secubox/toolbox/filters.json <<'SBXFILTERS'
+{
+ "banner": true,
+ "protective": "spoof",
+ "ad_ghost": true,
+ "ad_ghost_block": true,
+ "ad_ghost_categories": {"ads": true, "consent_nag": true, "newsletter": true, "social_widgets": true}
+}
+SBXFILTERS
+      chown root:secubox-toolbox /etc/secubox/toolbox/filters.json
+      chmod 0664 /etc/secubox/toolbox/filters.json
+    fi
+
     # 4. Storage dir (SQLite + future PDF reports)
     install -d -m 0750 -o secubox-toolbox -g secubox-toolbox /var/lib/secubox/toolbox
     # #536 : Android APK serve dir + best-effort fetch of the latest

packages/secubox-toolbox/mitmproxy_addons/ad_ghost.py

@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: LicenseRef-CMSD-1.0
+# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
+#
+# #566 — R3+/R4 silent ad/banner/widget GHOSTER + economisable savings.
+#
+# For R3+/R4 tunnel clients (on 10.99.1.0/24) ONLY, and only when enabled
+# in the modular filter config (toolbox WebUI → filters.json):
+#   - cosmetic ghost: inject a <style> that hides ad / consent-nag /
+#     newsletter-popup / social-widget containers (1st-party page stays
+#     usable) ;
+#   - block: 204 known ad/tracker hosts to save real bandwidth.
+# Tallies ghosted requests + estimated bytes saved → /run/secubox/ghost.json,
+# which inject_banner surfaces as quick stats. Doctrine: opt-in (filter
+# toggle), logged, reversible.
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import time
+
+from mitmproxy import http
+
+# Shared modular filter config (best-effort import; safe defaults if absent).
+try:
+    if "/usr/lib/secubox/toolbox" not in sys.path:
+        sys.path.insert(0, "/usr/lib/secubox/toolbox")
+    from secubox_toolbox.filters import get_filters
+except Exception:
+    def get_filters(force: bool = False):
+        return {"ad_ghost": True, "ad_ghost_block": True,
+                "ad_ghost_categories": {"ads": True, "consent_nag": True,
+                                        "newsletter": True, "social_widgets": True}}
+
+_STATS = "/run/secubox/ghost.json"
+_EST_BYTES_PER_REQ = 45000  # honest estimate per blocked ad/tracker request
+
+# Ad / tracker hosts to 204 (bandwidth save). Conservative: ad/tracker only.
+_AD_HOST = re.compile(
+    r"(?:^|\.)(?:doubleclick|googlesyndication|googleadservices|"
+    r"googletagservices|adservice\.google|amazon-adsystem|adnxs|adsrvr|"
+    r"adform|criteo|rubiconproject|taboola|outbrain|smartadserver|moatads|"
+    r"scorecardresearch|2mdn|adroll|pubmatic|openx|casalemedia|"
+    r"yieldlove|sharethrough|teads|3lift|adsystem|adserver)",
+    re.IGNORECASE,
+)
+
+# Cosmetic hide selectors, grouped so the WebUI can toggle each category.
+_COSMETIC = {
+    "ads": (
+        '[id^="google_ads"]', '[id^="div-gpt-ad"]', 'ins.adsbygoogle',
+        'iframe[src*="doubleclick"]', 'iframe[src*="googlesyndication"]',
+        'iframe[src*="amazon-adsystem"]', '[class*="ad-banner"]',
+        '[class*="advert"]', '[id*="banner-ad"]', '[id*="ad-container"]',
+        '[class*="-ads"]', '[class*="sponsored"]', 'aside[aria-label*="publicit"]',
+    ),
+    "consent_nag": (
+        '#onetrust-banner-sdk', '#onetrust-consent-sdk', '#didomi-host',
+        '.qc-cmp2-container', '[id^="sp_message_container"]',
+        '[id*="cookie-consent"]', '[class*="cookie-banner"]',
+        '[class*="cookie-notice"]', '[aria-label*="cookie"]', '.cmpbox',
+    ),
+    "newsletter": (
+        '[class*="newsletter-popup"]', '[class*="signup-modal"]',
+        '[id*="newsletter-modal"]', '[class*="subscribe-overlay"]',
+    ),
+    "social_widgets": (
+        '.fb-like', '.twitter-share-button', '[class*="social-share"]',
+        'iframe[src*="facebook.com/plugins"]', 'iframe[src*="platform.twitter"]',
+    ),
+}
+
+_RE_HEAD = re.compile(rb"</head>", re.IGNORECASE)
+_MARK = b"sbx-ghost-style"
+
+_counts = {"blocked_requests": 0, "bytes_saved_est": 0, "pages_cleaned": 0,
+           "since": int(time.time())}
+_last_flush = 0.0
+
+
+def _is_r3plus(flow) -> bool:
+    try:
+        ip = flow.client_conn.peername[0]
+        return bool(ip) and ip.startswith("10.99.1.")
+    except Exception:
+        return False
+
+
+def _flush(force: bool = False) -> None:
+    global _last_flush
+    now = time.time()
+    if not force and (now - _last_flush) < 5:
+        return
+    _last_flush = now
+    try:
+        os.makedirs(os.path.dirname(_STATS), exist_ok=True)
+        with open(_STATS, "w", encoding="utf-8") as f:
+            json.dump({**_counts, "updated": int(now)}, f)
+    except Exception:
+        pass
+
+
+def _style_for(cats: dict) -> bytes:
+    sels = []
+    for cat, on in cats.items():
+        if on and cat in _COSMETIC:
+            sels.extend(_COSMETIC[cat])
+    if not sels:
+        return b""
+    rule = ",".join(sels) + "{display:none!important;visibility:hidden!important}"
+    return (b"<style id=\"sbx-ghost-style\">" + rule.encode("utf-8") + b"</style>")
+
+
+class AdGhost:
+    def requestheaders(self, flow: http.HTTPFlow) -> None:
+        f = get_filters()
+        if not (f.get("ad_ghost") and f.get("ad_ghost_block")):
+            return
+        if not _is_r3plus(flow):
+            return
+        host = flow.request.pretty_host or ""
+        if _AD_HOST.search(host):
+            flow.response = http.Response.make(
+                204, b"", {"X-SecuBox-Ghost": "blocked"})
+            _counts["blocked_requests"] += 1
+            _counts["bytes_saved_est"] += _EST_BYTES_PER_REQ
+            _flush()
+
+    def response(self, flow: http.HTTPFlow) -> None:
+        f = get_filters()
+        if not f.get("ad_ghost") or not _is_r3plus(flow):
+            return
+        if not flow.response or flow.response.status_code != 200:
+            return
+        ct = (flow.response.headers.get("content-type", "") or "").lower()
+        if "text/html" not in ct:
+            return
+        try:
+            body = flow.response.content or b""
+        except Exception:
+            return
+        if not body or _MARK in body:
+            return
+        style = _style_for(f.get("ad_ghost_categories", {}))
+        if not style:
+            return
+        if _RE_HEAD.search(body):
+            new = _RE_HEAD.sub(style + b"</head>", body, count=1)
+        else:
+            new = style + body
+        flow.response.content = new
+        _counts["pages_cleaned"] += 1
+        _flush()
+
+
+addons = [AdGhost()]

packages/secubox-toolbox/mitmproxy_addons/inject_banner.py

@@ -479,6 +479,12 @@ def _banner_html_dynamic(sha1: str, ctx: dict, csp_strict: bool,
     if utiq_n > 0:
         # 📡 N — operator-grade tracker active
         right_parts.append(f"📡 utiq:{utiq_n}")
+    # #566 — ghost quick-stats: ads/widgets ghosted + bandwidth saved.
+    g_blocked = ctx.get("ghost_blocked", 0)
+    g_kb = ctx.get("ghost_kb", 0)
+    if g_blocked > 0:
+        # 🛡 N · ~X Ko économisés
+        right_parts.append(f"🛡 {g_blocked} ✕ ~{g_kb} Ko")
     if ctx["asn"]:
         right_parts.append(_ncr(ctx["asn"]))
     right_text = " · ".join(right_parts)  # middle dot · = ·
@@ -644,6 +650,17 @@ class InjectBanner:
         # AND R3 (portable WG opt-in). R0/R1 stay banner-free.
         if _client_level(flow) not in ("r2", "r3"):
             return
+        # #566 — modular filter toggle (toolbox WebUI). Banner can be
+        # disabled without touching the addon list.
+        try:
+            import sys as _sys
+            if "/usr/lib/secubox/toolbox" not in _sys.path:
+                _sys.path.insert(0, "/usr/lib/secubox/toolbox")
+            from secubox_toolbox.filters import get_filters as _gf
+            if not _gf().get("banner", True):
+                return
+        except Exception:
+            pass
         # Phase 10 perf : cheap pre-flight check on Content-Length to avoid
         # reading multi-MB bodies into RAM just to discover we'd skip them.
         # `flow.response.content` would buffer the whole body before returning.
@@ -667,10 +684,25 @@ class InjectBanner:
         # for R2) + level label ("R2" vs "R3")
         try:
             ctx = _compute_site_context(flow)
+            # #566 — ghost quick-stats (ads/widgets ghosted + KB saved).
+            try:
+                import json as _json
+                with open("/run/secubox/ghost.json", "r", encoding="utf-8") as _gf2:
+                    _g = _json.load(_gf2)
+                ctx["ghost_blocked"] = int(_g.get("blocked_requests", 0))
+                ctx["ghost_kb"] = int(_g.get("bytes_saved_est", 0)) // 1024
+            except Exception:
+                ctx["ghost_blocked"] = 0
+                ctx["ghost_kb"] = 0
             csp_strict = _detect_csp_strict(flow)
             report_url = _report_url_for(flow)
             level_label = _level_label(flow)
             level = _client_level(flow)
+            # #566 — R3+/R4 is an active-protection tier (spoof + ghost),
+            # not mere inspection: relabel the banner status accordingly.
+            if level in ("r3", "r4") and ctx.get("status") == "inspected":
+                ctx["status"] = "protected"
+                ctx["status_icon"] = "\U0001F6E1"  # 🛡
             snippet = _banner_html_dynamic(_CA_SHA1, ctx, csp_strict,
                                             report_url, level_label, level)
         except Exception as e:

packages/secubox-toolbox/mitmproxy_addons/protective_mode.py

@@ -22,6 +22,7 @@ from __future__ import annotations
 
 import logging
 import os
+import sys
 import re
 import time
 
@@ -61,7 +62,15 @@ _last_flush = 0.0
 
 
 def _level() -> str:
-    v = (os.environ.get("SECUBOX_PROTECTIVE_MODE") or "off").strip().lower()
+    # Modular filter config (toolbox WebUI) is the source of truth (#566);
+    # fall back to the SECUBOX_PROTECTIVE_MODE env when the config is absent.
+    try:
+        if "/usr/lib/secubox/toolbox" not in sys.path:
+            sys.path.insert(0, "/usr/lib/secubox/toolbox")
+        from secubox_toolbox.filters import get_filters
+        v = (get_filters().get("protective") or "off").strip().lower()
+    except Exception:
+        v = (os.environ.get("SECUBOX_PROTECTIVE_MODE") or "off").strip().lower()
     return v if v in ("off", "alert", "spoof") else "off"
 
 

packages/secubox-toolbox/sbin/secubox-toolbox-mitm-wg-launch

@@ -107,7 +107,10 @@ fi
 # protective_mode (#560) runs right after utiq_defense — early, so spoof-level
 # header/cookie stripping happens before the logging addons record the flow.
 # Inert unless SECUBOX_PROTECTIVE_MODE=alert|spoof (default off).
-for addon in inject_xff utiq_defense protective_mode local_store social_graph inject_banner dpi cookies avatar ja4 soc_relay cert_pin_detect; do
+# ad_ghost (#566) runs right after protective_mode: for R3+/R4 it 204s known
+# ad/tracker hosts (bandwidth save) at request time and injects ad-hiding CSS
+# on HTML responses. Gated by the modular filter config (toolbox WebUI).
+for addon in inject_xff utiq_defense protective_mode ad_ghost local_store social_graph inject_banner dpi cookies avatar ja4 soc_relay cert_pin_detect; do
     ARGS+=(-s "$ADDON_DIR/${addon}.py")
 done
 

packages/secubox-toolbox/secubox_toolbox/api.py

@@ -2422,9 +2422,97 @@ async def admin_protective() -> dict:
             out.update(_json.loads(st.read_text()))
     except Exception:
         pass
+    try:
+        from .filters import get_filters as _gf
+        out["mode"] = _gf().get("protective", out["mode"])
+    except Exception:
+        pass
     return out
 
 
+@router.get("/admin/ghost")
+async def admin_ghost() -> dict:
+    """#566 — ad/banner ghoster savings (R3+/R4). Read-only counters."""
+    import json as _json
+    from pathlib import Path as _P
+    out: dict = {"blocked_requests": 0, "bytes_saved_est": 0,
+                 "pages_cleaned": 0, "since": None, "updated": None}
+    try:
+        st = _P("/run/secubox/ghost.json")
+        if st.exists():
+            out.update(_json.loads(st.read_text()))
+    except Exception:
+        pass
+    out["mb_saved_est"] = round(out.get("bytes_saved_est", 0) / 1048576, 1)
+    return out
+
+
+@router.get("/admin/filters")
+async def admin_filters_get() -> dict:
+    """#566 — modular mitm filter config (read)."""
+    from .filters import get_filters
+    return get_filters(force=True)
+
+
+@router.post("/admin/filters")
+async def admin_filters_set(request: Request) -> dict:
+    """#566 — toggle mitm filters from the toolbox WebUI (write)."""
+    from .filters import set_filters
+    try:
+        patch = await request.json()
+    except Exception:
+        patch = {}
+    if not isinstance(patch, dict):
+        raise HTTPException(status_code=400, detail="json object expected")
+    return set_filters(patch)
+
+
+@router.get("/admin/filters/ui", response_class=HTMLResponse)
+async def admin_filters_ui() -> HTMLResponse:
+    """#566 — minimal filter toggle panel for the toolbox WebUI."""
+    html = """<!doctype html><html lang=fr><meta charset=utf-8>
+<meta name=viewport content="width=device-width,initial-scale=1">
+<title>Filtres ToolBoX</title>
+<style>
+ body{background:#0a0a0f;color:#e8e6d9;font:14px system-ui,sans-serif;max-width:560px;margin:30px auto;padding:0 18px}
+ h1{color:#c9a84c;font-size:18px} h2{color:#6e40c9;font-size:13px;margin:18px 0 6px}
+ label{display:flex;align-items:center;gap:8px;padding:6px 0;border-bottom:1px solid #1a1a22}
+ select,input{accent-color:#6e40c9} .muted{color:#6b6b7a;font-size:12px}
+ .stat{display:inline-block;background:#12121a;border:1px solid #222;border-radius:6px;padding:6px 10px;margin:4px 4px 0 0}
+ #msg{color:#00ff41;min-height:18px;margin-top:10px}
+</style>
+<h1>🛡 Filtres ToolBoX</h1>
+<p class=muted>Activation/désactivation à chaud des filtres mitm (R2/R3+). Prend effet en quelques secondes, sans redémarrage.</p>
+<div id=stats></div>
+<h2>Bannière & inspection</h2>
+<label><input type=checkbox data-k=banner> Bannière de transparence (R2/R3)</label>
+<label>Mode protecteur (spoof traceurs)
+  <select data-k=protective><option value=off>off</option><option value=alert>alert</option><option value=spoof>spoof</option></select></label>
+<h2>Ghosting pub (R3+/R4)</h2>
+<label><input type=checkbox data-k=ad_ghost> Masquer pubs/bannières/widgets (cosmétique)</label>
+<label><input type=checkbox data-k=ad_ghost_block> Bloquer les hôtes pub/traceurs (économise la bande passante)</label>
+<label><input type=checkbox data-c=ads> · catégorie : publicités</label>
+<label><input type=checkbox data-c=consent_nag> · catégorie : bandeaux cookies/consentement</label>
+<label><input type=checkbox data-c=newsletter> · catégorie : pop-ups newsletter</label>
+<label><input type=checkbox data-c=social_widgets> · catégorie : widgets sociaux</label>
+<p id=msg></p>
+<script>
+const $=s=>document.querySelector(s);
+function post(p){fetch('/admin/filters',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify(p)}).then(r=>r.json()).then(load).then(()=>{$('#msg').textContent='Enregistré ✓';setTimeout(()=>$('#msg').textContent='',1200)});}
+function load(f){
+ document.querySelectorAll('[data-k]').forEach(el=>{const k=el.dataset.k;if(el.type==='checkbox')el.checked=!!f[k];else el.value=f[k];});
+ const c=f.ad_ghost_categories||{};document.querySelectorAll('[data-c]').forEach(el=>el.checked=!!c[el.dataset.c]);
+}
+function wire(){
+ document.querySelectorAll('[data-k]').forEach(el=>el.addEventListener('change',()=>{const v=el.type==='checkbox'?el.checked:el.value;post({[el.dataset.k]:v});}));
+ document.querySelectorAll('[data-c]').forEach(el=>el.addEventListener('change',()=>post({ad_ghost_categories:{[el.dataset.c]:el.checked}})));
+}
+function stats(){fetch('/admin/ghost').then(r=>r.json()).then(g=>{$('#stats').innerHTML=`<span class=stat>🛡 ${g.blocked_requests||0} bloqués</span><span class=stat>💾 ~${g.mb_saved_est||0} Mo économisés</span><span class=stat>🧹 ${g.pages_cleaned||0} pages nettoyées</span>`;});}
+fetch('/admin/filters').then(r=>r.json()).then(f=>{load(f);wire();stats();});
+</script></html>"""
+    return HTMLResponse(content=html)
+
+
 @router.get("/social/report/{token}.pdf")
 async def social_report_pdf(token: str) -> Response:
     """Phase 11.C (#508) — bilingual FR/EN evidence PDF for a peer.

packages/secubox-toolbox/secubox_toolbox/filters.py

@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: LicenseRef-CMSD-1.0
+# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
+#
+# SecuBox-Deb :: toolbox :: modular filter config (#566)
+#
+# Single source of truth for which mitm filters are active, toggled from
+# the toolbox WebUI (GET/POST /admin/filters). The mitm addons
+# (inject_banner, protective_mode, ad_ghost) read this at flow time with a
+# short cache so toggles take effect within seconds — no restart.
+from __future__ import annotations
+
+import json
+import os
+import time
+from typing import Dict
+
+FILTERS_PATH = "/etc/secubox/toolbox/filters.json"
+
+DEFAULTS: Dict = {
+    "banner": True,                 # inject the R2/R3 transparency banner
+    "protective": "spoof",          # off | alert | spoof  (tracker spoofer)
+    "ad_ghost": True,               # R3+/R4 silent ad/banner/widget ghosting
+    "ad_ghost_block": True,         # 204 known ad/tracker hosts (save bandwidth)
+    "ad_ghost_categories": {        # cosmetic ghost groups
+        "ads": True,
+        "consent_nag": True,
+        "newsletter": True,
+        "social_widgets": True,
+    },
+}
+
+_VALID_PROTECTIVE = ("off", "alert", "spoof")
+
+_cache: Dict = {}
+_cache_ts: float = 0.0
+
+
+def get_filters(force: bool = False) -> Dict:
+    """Merged filter config (defaults + on-disk overrides), 5 s cached."""
+    global _cache, _cache_ts
+    now = time.time()
+    if not force and _cache and (now - _cache_ts) < 5:
+        return _cache
+    out = json.loads(json.dumps(DEFAULTS))  # deep copy
+    try:
+        with open(FILTERS_PATH, "r", encoding="utf-8") as f:
+            disk = json.load(f)
+        if isinstance(disk, dict):
+            for k, v in disk.items():
+                if k == "ad_ghost_categories" and isinstance(v, dict):
+                    out["ad_ghost_categories"].update(v)
+                elif k in out:
+                    out[k] = v
+    except Exception:
+        pass
+    if out.get("protective") not in _VALID_PROTECTIVE:
+        out["protective"] = DEFAULTS["protective"]
+    _cache = out
+    _cache_ts = now
+    return out
+
+
+def set_filters(patch: Dict) -> Dict:
+    """Merge a partial patch into the on-disk config and return the result.
+    Only known keys are accepted (defensive)."""
+    global _cache_ts
+    cur = get_filters(force=True)
+    for k, v in (patch or {}).items():
+        if k == "ad_ghost_categories" and isinstance(v, dict):
+            cur["ad_ghost_categories"].update(
+                {ck: bool(cv) for ck, cv in v.items()
+                 if ck in DEFAULTS["ad_ghost_categories"]})
+        elif k == "protective" and v in _VALID_PROTECTIVE:
+            cur["protective"] = v
+        elif k in ("banner", "ad_ghost", "ad_ghost_block"):
+            cur[k] = bool(v)
+    try:
+        os.makedirs(os.path.dirname(FILTERS_PATH), exist_ok=True)
+        tmp = FILTERS_PATH + ".tmp"
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(cur, f, indent=1)
+        os.replace(tmp, FILTERS_PATH)
+    except Exception:
+        pass
+    _cache_ts = 0.0  # invalidate
+    return cur