From 675f6ae4580ef0fe60d09d3ae486ad40da064b43 Mon Sep 17 00:00:00 2001 From: CyberMind-FR Date: Sun, 14 Jun 2026 10:57:48 +0200 Subject: [PATCH] feat(toolbox): DPI media/content-type statistifier + donut (closes #570) mitmproxy_addons/media_stats.py buckets responses by content-type category (emoji-iconified) + provider (eTLD+1), summing Content-Length (header only, no body read). Rolling -> /run/secubox/media.json. api: /admin/media + /admin/media/ui (SVG donut + emoji legend + top-5 providers w/ favicons). Wired into the mitm-wg launcher. secubox-toolbox 2.6.25. Co-Authored-By: Claude Opus 4.8 --- packages/secubox-toolbox/debian/changelog | 14 +++ .../mitmproxy_addons/media_stats.py | 113 ++++++++++++++++++ .../sbin/secubox-toolbox-mitm-wg-launch | 2 +- .../secubox-toolbox/secubox_toolbox/api.py | 79 ++++++++++++ 4 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 packages/secubox-toolbox/mitmproxy_addons/media_stats.py diff --git a/packages/secubox-toolbox/debian/changelog b/packages/secubox-toolbox/debian/changelog index 3c35accd..a0000612 100644 --- a/packages/secubox-toolbox/debian/changelog +++ b/packages/secubox-toolbox/debian/changelog @@ -1,3 +1,17 @@ +secubox-toolbox (2.6.25-1~bookworm1) bookworm; urgency=medium + + * DPI media/content-type statistifier + donut (#570). + - mitmproxy_addons/media_stats.py : buckets every response by content- + type CATEGORY (page/image/video/audio/script/style/font/api/text/ + other, emoji-iconified) and by PROVIDER (eTLD+1), summing + Content-Length (header only — never reads the body, safe on video). + Rolling counters → /run/secubox/media.json. Wired into the launcher. + - api: GET /admin/media (categories %+emoji, top-5 providers, totals) ; + GET /admin/media/ui (SVG donut + emoji legend + top-5 providers with + favicons). + + -- Gerald KERMA Sat, 13 Jun 2026 19:00:00 +0200 + secubox-toolbox (2.6.24-1~bookworm1) bookworm; urgency=medium * webext: cap the popup top-tracker list to 5 items (#568); webext 0.1.3. diff --git a/packages/secubox-toolbox/mitmproxy_addons/media_stats.py b/packages/secubox-toolbox/mitmproxy_addons/media_stats.py new file mode 100644 index 00000000..e847979b --- /dev/null +++ b/packages/secubox-toolbox/mitmproxy_addons/media_stats.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: LicenseRef-CMSD-1.0 +# Copyright (c) 2026 CyberMind — Gérald Kerma +# +# #570 — DPI media/content-type statistifier. Buckets every mitm response +# by content-type CATEGORY and by PROVIDER (eTLD+1), summing Content-Length +# (header only — never reads the body, safe on video/large media). Rolling +# in-memory counters flushed to /run/secubox/media.json for the donut UI. +from __future__ import annotations + +import json +import os +import re +import time + +from mitmproxy import http + +_STATS = "/run/secubox/media.json" + +# content-type → (category, emoji). Order matters (first match wins). +_CATS = ( + ("page", "\U0001F4C4", ("text/html", "application/xhtml")), # 📄 + ("image", "\U0001F5BC", ("image/",)), # 🖼 + ("video", "\U0001F3AC", ("video/", "application/vnd.apple.mpegurl", + "application/x-mpegurl", "application/dash+xml")), # 🎬 + ("audio", "\U0001F3B5", ("audio/",)), # 🎵 + ("script", "\U0001F9E9", ("javascript", "ecmascript")), # 🧩 + ("style", "\U0001F3A8", ("text/css",)), # 🎨 + ("font", "\U0001F524", ("font/", "application/font", "application/vnd.ms-fontobject", + "application/x-font")), # 🔤 + ("api", "\U0001F4E6", ("application/json", "+json", "application/xml", + "text/xml", "application/grpc")), # 📦 + ("text", "\U0001F4DD", ("text/plain", "text/")), # 📝 +) +_OTHER = ("other", "❓") # ❓ +EMOJI = {c: e for c, e, _ in _CATS} +EMOJI[_OTHER[0]] = _OTHER[1] + +_MAX_PROVIDERS = 250 +_2L_TLD = ("co.uk", "com.au", "co.jp", "co.nz", "com.br", "co.za", "gouv.fr") + +_cats: dict = {} +_providers: dict = {} +_total = {"bytes": 0, "count": 0, "since": int(time.time())} +_last_flush = 0.0 + + +def _category(ct: str) -> tuple: + ct = (ct or "").split(";", 1)[0].strip().lower() + if not ct: + return _OTHER + for cat, emoji, frags in _CATS: + if any(f in ct for f in frags): + return (cat, emoji) + return _OTHER + + +def _registrable(host: str) -> str: + host = (host or "").split(":", 1)[0].lower().strip(".") + if not host or host.replace(".", "").isdigit(): + return host or "?" + parts = host.split(".") + if len(parts) <= 2: + return host + last2 = ".".join(parts[-2:]) + if last2 in _2L_TLD and len(parts) >= 3: + return ".".join(parts[-3:]) + return last2 + + +def _flush(force: bool = False) -> None: + global _last_flush + now = time.time() + if not force and (now - _last_flush) < 5: + return + _last_flush = now + # cap providers to the heaviest _MAX_PROVIDERS by bytes + global _providers + if len(_providers) > _MAX_PROVIDERS: + _providers = dict(sorted(_providers.items(), + key=lambda kv: -kv[1]["bytes"])[:_MAX_PROVIDERS]) + try: + os.makedirs(os.path.dirname(_STATS), exist_ok=True) + with open(_STATS, "w", encoding="utf-8") as f: + json.dump({"categories": _cats, "providers": _providers, + "total": _total, "updated": int(now)}, f) + except Exception: + pass + + +class MediaStats: + def response(self, flow: http.HTTPFlow) -> None: + if not flow.response: + return + h = flow.response.headers + cat, _ = _category(h.get("content-type", "")) + try: + size = int(h.get("content-length", "0") or "0") + except (TypeError, ValueError): + size = 0 + prov = _registrable(flow.request.pretty_host or "") + + c = _cats.setdefault(cat, {"bytes": 0, "count": 0}) + c["bytes"] += size + c["count"] += 1 + p = _providers.setdefault(prov, {"bytes": 0, "count": 0}) + p["bytes"] += size + p["count"] += 1 + _total["bytes"] += size + _total["count"] += 1 + _flush() + + +addons = [MediaStats()] diff --git a/packages/secubox-toolbox/sbin/secubox-toolbox-mitm-wg-launch b/packages/secubox-toolbox/sbin/secubox-toolbox-mitm-wg-launch index 96dfbd6e..c54851f1 100755 --- a/packages/secubox-toolbox/sbin/secubox-toolbox-mitm-wg-launch +++ b/packages/secubox-toolbox/sbin/secubox-toolbox-mitm-wg-launch @@ -110,7 +110,7 @@ fi # ad_ghost (#566) runs right after protective_mode: for R3+/R4 it 204s known # ad/tracker hosts (bandwidth save) at request time and injects ad-hiding CSS # on HTML responses. Gated by the modular filter config (toolbox WebUI). -for addon in inject_xff utiq_defense protective_mode ad_ghost local_store social_graph inject_banner dpi cookies avatar ja4 soc_relay cert_pin_detect; do +for addon in inject_xff utiq_defense protective_mode ad_ghost local_store social_graph inject_banner dpi cookies avatar ja4 soc_relay cert_pin_detect media_stats; do ARGS+=(-s "$ADDON_DIR/${addon}.py") done diff --git a/packages/secubox-toolbox/secubox_toolbox/api.py b/packages/secubox-toolbox/secubox_toolbox/api.py index ee9d5ce0..406176f6 100644 --- a/packages/secubox-toolbox/secubox_toolbox/api.py +++ b/packages/secubox-toolbox/secubox_toolbox/api.py @@ -2447,6 +2447,85 @@ async def admin_ghost() -> dict: return out +_MEDIA_EMOJI = { + "page": "\U0001F4C4", "image": "\U0001F5BC", "video": "\U0001F3AC", + "audio": "\U0001F3B5", "script": "\U0001F9E9", "style": "\U0001F3A8", + "font": "\U0001F524", "api": "\U0001F4E6", "text": "\U0001F4DD", + "other": "❓", +} + + +@router.get("/admin/media") +async def admin_media() -> dict: + """#570 — DPI media/content-type statistics for the donut UI.""" + import json as _json + from pathlib import Path as _P + raw: dict = {"categories": {}, "providers": {}, "total": {"bytes": 0, "count": 0}} + try: + st = _P("/run/secubox/media.json") + if st.exists(): + raw.update(_json.loads(st.read_text())) + except Exception: + pass + tot_b = max(int(raw.get("total", {}).get("bytes", 0)), 1) + cats = [] + for cat, v in (raw.get("categories") or {}).items(): + b = int(v.get("bytes", 0)) + cats.append({"cat": cat, "emoji": _MEDIA_EMOJI.get(cat, "❓"), + "bytes": b, "count": int(v.get("count", 0)), + "mb": round(b / 1048576, 1), "pct": round(100 * b / tot_b, 1)}) + cats.sort(key=lambda x: -x["bytes"]) + provs = sorted( + ({"provider": p, "bytes": int(v.get("bytes", 0)), + "count": int(v.get("count", 0)), "mb": round(int(v.get("bytes", 0)) / 1048576, 1)} + for p, v in (raw.get("providers") or {}).items()), + key=lambda x: -x["bytes"])[:5] + return {"categories": cats, "top_providers": provs, + "total_mb": round(raw.get("total", {}).get("bytes", 0) / 1048576, 1), + "total_count": raw.get("total", {}).get("count", 0), + "updated": raw.get("updated")} + + +@router.get("/admin/media/ui", response_class=HTMLResponse) +async def admin_media_ui() -> HTMLResponse: + """#570 — donut + emoji legend + top-5 providers.""" + html = """ + +DPI Médias — ToolBoX + +

📊 DPI — types de contenus

+

…

+ +
+

Top 5 fournisseurs

+
+""" + return HTMLResponse(content=html) + + @router.get("/admin/filters") async def admin_filters_get() -> dict: """#566 — modular mitm filter config (read)."""