mirror of
https://github.com/CyberMind-FR/secubox-deb.git
synced 2026-07-30 00:19:30 +00:00
feat(secubox-core+toolbox): Phase 3 transparency layer (whitelist + analysis-status + quality scoring) (ref #492)
## Goal Make the cabine HONEST : tell the user not just what we inspected, but what we deliberately BYPASSED and WHY (cert-pinning vendor, banking policy, E2E opaque by design). Score each destination on passive observable signals. This is the foundation for #493 (later) full reverse-WAF enforcement — architecture intentionally aligned so action='block'/'redirect'/'strip' can be added without refactoring. ## New shared code (common/secubox_core/) - whitelist.py : YAML loader with hot-reload. fnmatch glob patterns. Reads /usr/share/secubox/toolbox/whitelist-baseline.yaml + /etc/secubox/toolbox/whitelist.yaml (operator override). Returns (match dict | None) for any host. - classifiers/security_quality.py : grade A+/A/B/C/D/F per destination. Two modes : passive : TLS version + JA4 + SNI + ALPN + is_e2e_messaging active : adds HSTS + CSP + X-Content-Type-Options + cookies attrs + mixed-content detection Returns {grade, score, mode, reasons[]} with each reason's weight. ## Curated whitelist baseline (47 patterns) - Apple ecosystem (push, iCloud, CDN, mzstatic, cloudkit, smoot) - Google/Android (googleapis, gstatic, fcm, android, play) - Messaging E2E (Signal, Threema, SimpleX, Matrix.org, Proton, Tutanota) - Banking FR (BanquePostale, SocGen, BNP, CA, CM, LCL, Boursorama, Revolut) - Government FR (service-public, impots.gouv, ameli, france-identite, francetravail, caf) - Health (Doctolib, Qare, MonEspaceSante) - Privacy tools (DDG, Qwant, Startpage, Proton, Tutanota, Mullvad, Tailscale) - Tor Project - OS updates (Windows Update, Microsoft, Debian, Ubuntu) ## local_store.py wiring - New _analysis_status(host, sni, decrypted) returns (status, reason). Order : decrypted > whitelist > E2E regex > pinned-failed fallback. - DPI events now carry analysis_status + analysis_reason fields. - New tls_failed_clienthello hook captures the SNI of bypassed/pinned flows so the report can be honest about them. ## Aggregator (secubox_toolbox/api.py) - New _build_transparency() builds the dict : breakdown : raw counts {inspected: N, bypassed-whitelist: N, ...} breakdown_pct : same as percentages total_events : sum per_host[] : worst-first grade table for the report UI whitelist_stats : how many patterns are loaded - Exposed under 'transparency' key in /report and /report/me/html. - Backfill : legacy events (no analysis_status) get post-hoc tagged via whitelist match check so older sessions also show meaningful breakdowns. ## Reports - HTML live (report-live.html.j2) : new card 'INSPECTION : CE QUI A ETE REGARDE' with breakdown bar + 'QUALITE SECURITE PAR DESTINATION' sortable table. Inserts before the Support card. - PDF (reports.py) : matching sections rendered ASCII-safe via _ascii_safe. Top 10 worst-quality hosts. ## Verified on gk2 (2026-06-05) - whitelist loaded : 47 patterns, baseline OK - aggregator returns transparency.has_transparency=True - per_host populated with quality grades - PDF still generates clean ## Out of scope (deferred to #493 enforcement phase) - Action types : block/redirect/strip-cookies/downgrade - nftables sinkhole for known-bad destinations - dnsmasq hijack for whitelisted FQDN auto-direction - PhishTank + threat-intel feeds active blocking - Per-flow TLS version + headers capture (currently we only grade default 'inspected' as C because no TLS metadata in the event yet — Phase 4 will extend local_store.response() to capture response headers + TLS info). ## Closes #492 (Phase 3 transparency foundation)
This commit is contained in:
parent
9641cb8f91
commit
27eacbcd24
237
common/secubox_core/classifiers/security_quality.py
Normal file
237
common/secubox_core/classifiers/security_quality.py
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||
|
||||
"""Security quality scorer (#492) : grade a destination on observable signals.
|
||||
|
||||
Two modes :
|
||||
- PASSIVE : TLS version + JA4 + SNI presence + ALPN (no decryption needed)
|
||||
- ACTIVE : adds HTTPS headers (HSTS, CSP) + cookies attrs (Secure/HttpOnly/SameSite)
|
||||
|
||||
Output : A+/A/B/C/D/F grade + per-criterion breakdown.
|
||||
|
||||
Phase 4 will add live cert chain analysis + CT log lookup + HSTS preload check.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
# Cipher categories — TLS 1.3 only uses AEAD ciphers (good). TLS 1.2 mix.
|
||||
_WEAK_CIPHERS = {
|
||||
# CBC mode = vulnerable to padding oracle attacks
|
||||
"RSA_WITH_AES_128_CBC", "RSA_WITH_AES_256_CBC",
|
||||
"RSA_WITH_3DES_EDE_CBC",
|
||||
# RC4 deprecated
|
||||
"RSA_WITH_RC4_128",
|
||||
# Export-grade — should never see in 2025+
|
||||
"RSA_EXPORT",
|
||||
}
|
||||
|
||||
|
||||
def _grade_to_score(grade: str) -> int:
|
||||
return {"A+": 100, "A": 90, "B": 75, "C": 60, "D": 40, "F": 0}.get(grade, 50)
|
||||
|
||||
|
||||
def _score_to_grade(score: int) -> str:
|
||||
if score >= 95:
|
||||
return "A+"
|
||||
if score >= 85:
|
||||
return "A"
|
||||
if score >= 70:
|
||||
return "B"
|
||||
if score >= 55:
|
||||
return "C"
|
||||
if score >= 35:
|
||||
return "D"
|
||||
return "F"
|
||||
|
||||
|
||||
def grade_passive(
|
||||
*,
|
||||
tls_version: str | None = None,
|
||||
alpn_protocols: list | None = None,
|
||||
sni: str | None = None,
|
||||
ja4_known_client: bool = False,
|
||||
is_e2e_messaging: bool = False,
|
||||
) -> dict:
|
||||
"""Grade a flow we can only observe transport-side (whitelisted/pinned/E2E)."""
|
||||
score = 50
|
||||
reasons: list[dict] = []
|
||||
|
||||
# TLS version
|
||||
if tls_version in ("13", "1.3", "TLSv1.3"):
|
||||
score += 30
|
||||
reasons.append({"+": 30, "why": "TLS 1.3 (modern AEAD ciphers, no downgrade)"})
|
||||
elif tls_version in ("12", "1.2", "TLSv1.2"):
|
||||
score += 15
|
||||
reasons.append({"+": 15, "why": "TLS 1.2 (acceptable but legacy)"})
|
||||
elif tls_version in ("11", "1.1", "TLSv1.1", "10", "1.0", "TLSv1.0"):
|
||||
score -= 30
|
||||
reasons.append({"-": 30, "why": "TLS legacy (deprecated, vulnerable)"})
|
||||
elif tls_version is None:
|
||||
score += 5
|
||||
reasons.append({"~": 0, "why": "TLS version unknown (HTTP or pre-handshake)"})
|
||||
|
||||
# ALPN
|
||||
if alpn_protocols:
|
||||
if "h3" in alpn_protocols or "h3-29" in alpn_protocols:
|
||||
score += 10
|
||||
reasons.append({"+": 10, "why": "HTTP/3 (QUIC, modern transport)"})
|
||||
elif "h2" in alpn_protocols:
|
||||
score += 5
|
||||
reasons.append({"+": 5, "why": "HTTP/2"})
|
||||
|
||||
# SNI present (encrypted SNI / ECH not yet detectable here)
|
||||
if sni:
|
||||
score += 5
|
||||
reasons.append({"+": 5, "why": "SNI present (standard practice)"})
|
||||
|
||||
# JA4 known-good
|
||||
if ja4_known_client:
|
||||
score += 10
|
||||
reasons.append({"+": 10, "why": "JA4 fingerprint matches known-good client"})
|
||||
|
||||
# E2E bonus (Signal/Threema/etc. — even though opaque, the design is strong)
|
||||
if is_e2e_messaging:
|
||||
score += 5
|
||||
reasons.append({"+": 5, "why": "E2E messaging design (Signal protocol or similar)"})
|
||||
|
||||
score = max(0, min(100, score))
|
||||
return {
|
||||
"grade": _score_to_grade(score),
|
||||
"score": score,
|
||||
"mode": "passive",
|
||||
"reasons": reasons,
|
||||
}
|
||||
|
||||
|
||||
def grade_active(
|
||||
*,
|
||||
tls_version: str | None = None,
|
||||
alpn_protocols: list | None = None,
|
||||
sni: str | None = None,
|
||||
headers: dict | None = None,
|
||||
cookies_attrs: list[dict] | None = None,
|
||||
mixed_content_detected: bool = False,
|
||||
) -> dict:
|
||||
"""Grade a flow we fully inspected via MITM."""
|
||||
base = grade_passive(tls_version=tls_version, alpn_protocols=alpn_protocols, sni=sni)
|
||||
score = base["score"]
|
||||
reasons = list(base["reasons"])
|
||||
headers = {k.lower(): v for k, v in (headers or {}).items()}
|
||||
|
||||
# HSTS
|
||||
hsts = headers.get("strict-transport-security", "")
|
||||
if hsts:
|
||||
if "max-age=31536000" in hsts or "max-age=63072000" in hsts:
|
||||
score += 10
|
||||
reasons.append({"+": 10, "why": "HSTS strong (1y+)"})
|
||||
if "preload" in hsts.lower():
|
||||
score += 5
|
||||
reasons.append({"+": 5, "why": "HSTS preload eligible"})
|
||||
else:
|
||||
score += 3
|
||||
reasons.append({"+": 3, "why": "HSTS present but weak max-age"})
|
||||
else:
|
||||
score -= 10
|
||||
reasons.append({"-": 10, "why": "HSTS missing (downgrade attack possible)"})
|
||||
|
||||
# CSP
|
||||
csp = headers.get("content-security-policy", "")
|
||||
if csp:
|
||||
if "default-src 'none'" in csp or "default-src 'self'" in csp:
|
||||
score += 8
|
||||
reasons.append({"+": 8, "why": "CSP strict"})
|
||||
elif "unsafe-inline" not in csp and "unsafe-eval" not in csp:
|
||||
score += 4
|
||||
reasons.append({"+": 4, "why": "CSP present (moderate)"})
|
||||
else:
|
||||
score -= 5
|
||||
reasons.append({"-": 5, "why": "CSP missing"})
|
||||
|
||||
# X-Frame-Options / X-Content-Type-Options (basic hardening)
|
||||
if "x-content-type-options" in headers:
|
||||
score += 2
|
||||
reasons.append({"+": 2, "why": "X-Content-Type-Options: nosniff"})
|
||||
|
||||
# Cookies
|
||||
if cookies_attrs:
|
||||
weak_cookies = 0
|
||||
strong_cookies = 0
|
||||
for c in cookies_attrs:
|
||||
if c.get("secure") and c.get("httponly") and c.get("samesite") in ("strict", "lax"):
|
||||
strong_cookies += 1
|
||||
else:
|
||||
weak_cookies += 1
|
||||
if strong_cookies and not weak_cookies:
|
||||
score += 5
|
||||
reasons.append({"+": 5, "why": f"All {strong_cookies} cookies have Secure+HttpOnly+SameSite"})
|
||||
elif weak_cookies:
|
||||
score -= 5
|
||||
reasons.append({"-": 5, "why": f"{weak_cookies} cookies missing Secure/HttpOnly/SameSite"})
|
||||
|
||||
# Mixed content = critical demerit
|
||||
if mixed_content_detected:
|
||||
score -= 20
|
||||
reasons.append({"-": 20, "why": "Mixed HTTP+HTTPS content (man-in-the-middle window)"})
|
||||
|
||||
score = max(0, min(100, score))
|
||||
return {
|
||||
"grade": _score_to_grade(score),
|
||||
"score": score,
|
||||
"mode": "active",
|
||||
"reasons": reasons,
|
||||
}
|
||||
|
||||
|
||||
def aggregate_per_host(events: list[dict]) -> dict:
|
||||
"""Compute per-host quality aggregate from a list of mitm events.
|
||||
|
||||
Each event payload should optionally have :
|
||||
- tls_version, alpn_protocols, sni (from JA4 / handshake)
|
||||
- headers, set_cookie_attrs (from MITM-inspected response)
|
||||
- analysis_status ('inspected' | 'bypassed-*' | 'pinned-*' | 'e2e-opaque')
|
||||
|
||||
Returns {host: {grade, score, mode, samples_count, last_seen_ms}}.
|
||||
"""
|
||||
by_host: dict[str, dict] = {}
|
||||
for ev in events:
|
||||
payload = ev.get("payload") or ev
|
||||
host = payload.get("host") or payload.get("sni") or "?"
|
||||
if host == "?" or not host:
|
||||
continue
|
||||
if host in by_host:
|
||||
by_host[host]["samples_count"] += 1
|
||||
by_host[host]["last_seen_ms"] = max(
|
||||
by_host[host]["last_seen_ms"], payload.get("ts_ms", 0)
|
||||
)
|
||||
continue
|
||||
analysis_status = payload.get("analysis_status", "")
|
||||
if analysis_status.startswith("bypassed") or analysis_status.startswith("pinned"):
|
||||
g = grade_passive(
|
||||
tls_version=payload.get("tls_version"),
|
||||
alpn_protocols=payload.get("alpn_protocols"),
|
||||
sni=payload.get("sni"),
|
||||
ja4_known_client=bool(payload.get("ja4_known_client")),
|
||||
)
|
||||
elif analysis_status == "e2e-opaque":
|
||||
g = grade_passive(
|
||||
tls_version=payload.get("tls_version") or "13",
|
||||
sni=payload.get("sni"),
|
||||
is_e2e_messaging=True,
|
||||
)
|
||||
else:
|
||||
g = grade_active(
|
||||
tls_version=payload.get("tls_version"),
|
||||
alpn_protocols=payload.get("alpn_protocols"),
|
||||
sni=payload.get("sni"),
|
||||
headers=payload.get("response_headers"),
|
||||
cookies_attrs=payload.get("set_cookie_attrs"),
|
||||
mixed_content_detected=bool(payload.get("mixed_content")),
|
||||
)
|
||||
by_host[host] = {
|
||||
**g,
|
||||
"host": host,
|
||||
"samples_count": 1,
|
||||
"last_seen_ms": payload.get("ts_ms", 0),
|
||||
"analysis_status": analysis_status or "inspected",
|
||||
}
|
||||
return by_host
|
||||
97
common/secubox_core/whitelist.py
Normal file
97
common/secubox_core/whitelist.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||
|
||||
"""ToolBoX whitelist (#492) : map host/SNI -> bypass-reason + category.
|
||||
|
||||
YAML format (operator can override at /etc/secubox/toolbox/whitelist.yaml) :
|
||||
|
||||
whitelist:
|
||||
- pattern: "*.push.apple.com"
|
||||
reason: cert-pinning, no MITM possible
|
||||
category: vendor-os
|
||||
quality_expected: A+
|
||||
- pattern: "signal.org"
|
||||
reason: E2E messenger, transport-only observation
|
||||
category: messaging-e2e
|
||||
quality_expected: A+
|
||||
|
||||
Patterns are fnmatch-style globs (Unix shell). Phase 4 will add regex/ASN/JA4 match.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
log = logging.getLogger("secubox.whitelist")
|
||||
|
||||
BASELINE_PATH = Path("/usr/share/secubox/toolbox/whitelist-baseline.yaml")
|
||||
OVERRIDE_PATH = Path("/etc/secubox/toolbox/whitelist.yaml")
|
||||
|
||||
_CACHE: dict | None = None
|
||||
_CACHE_MTIME: float = 0.0
|
||||
|
||||
|
||||
def _load_yaml(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
import yaml as _yaml
|
||||
except ImportError:
|
||||
log.warning("PyYAML unavailable, whitelist disabled")
|
||||
return []
|
||||
try:
|
||||
data = _yaml.safe_load(path.read_text()) or {}
|
||||
return data.get("whitelist", []) or []
|
||||
except Exception as e:
|
||||
log.warning("whitelist load failed (%s): %s", path, e)
|
||||
return []
|
||||
|
||||
|
||||
def _refresh_cache() -> dict:
|
||||
"""Hot-reload : combine baseline + override, re-read if either mtime changed."""
|
||||
global _CACHE, _CACHE_MTIME
|
||||
mtime = 0.0
|
||||
for p in (BASELINE_PATH, OVERRIDE_PATH):
|
||||
if p.exists():
|
||||
mtime = max(mtime, p.stat().st_mtime)
|
||||
if _CACHE is not None and mtime <= _CACHE_MTIME:
|
||||
return _CACHE
|
||||
entries = _load_yaml(BASELINE_PATH) + _load_yaml(OVERRIDE_PATH)
|
||||
# Compile : map pattern -> entry. Override wins on dup (last in list).
|
||||
compiled: dict[str, dict] = {}
|
||||
for e in entries:
|
||||
p = e.get("pattern", "").lower().strip()
|
||||
if p:
|
||||
compiled[p] = e
|
||||
_CACHE = {"entries": compiled, "count": len(compiled)}
|
||||
_CACHE_MTIME = mtime
|
||||
log.info("whitelist reloaded : %d entries", len(compiled))
|
||||
return _CACHE
|
||||
|
||||
|
||||
def match(host: str) -> Optional[dict]:
|
||||
"""Return the matching whitelist entry for a host, or None."""
|
||||
if not host:
|
||||
return None
|
||||
h = host.lower().strip()
|
||||
cache = _refresh_cache()
|
||||
for pattern, entry in cache["entries"].items():
|
||||
if fnmatch.fnmatch(h, pattern):
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def is_whitelisted(host: str) -> bool:
|
||||
return match(host) is not None
|
||||
|
||||
|
||||
def stats() -> dict:
|
||||
"""Returns {count, baseline_loaded, override_loaded}."""
|
||||
cache = _refresh_cache()
|
||||
return {
|
||||
"count": cache["count"],
|
||||
"baseline_loaded": BASELINE_PATH.exists(),
|
||||
"override_loaded": OVERRIDE_PATH.exists(),
|
||||
}
|
||||
|
|
@ -333,6 +333,69 @@ li::before{content:"▸ ";color:var(--phos);text-shadow:0 0 4px var(--phos)}
|
|||
<a href="/status">📊 Statut JSON</a>
|
||||
</div>
|
||||
|
||||
{# Phase 3 (#492) : transparency layer — inspection breakdown + per-host quality #}
|
||||
{% set t = transparency|default({}) %}
|
||||
{% if t and t.get('total_events', 0) > 0 %}
|
||||
<div class="card">
|
||||
<h2>🔎 INSPECTION : CE QU'ON A REGARDÉ (et ce qu'on n'a PAS regardé)</h2>
|
||||
<p style="font-size:0.78rem;color:var(--dim);margin-bottom:0.6rem;font-style:italic">
|
||||
Honnêteté avant magie : la cabine te dit ce qu'elle a inspecté, ce qu'elle a sciemment bypassé, et pourquoi.
|
||||
</p>
|
||||
<div class="kv">
|
||||
{% set b = t.get('breakdown_pct', {}) %}
|
||||
{% if b.get('inspected') %}
|
||||
<div class="k">🔍 Inspecté (HTTPS via notre CA)</div>
|
||||
<div class="v">{{ b.get('inspected', 0) }}% — contenu visible</div>
|
||||
{% endif %}
|
||||
{% if b.get('bypassed-whitelist') %}
|
||||
<div class="k">🛡 Bypass whitelist</div>
|
||||
<div class="v">{{ b.get('bypassed-whitelist', 0) }}% — décision policy (cert-pinning vendor)</div>
|
||||
{% endif %}
|
||||
{% if b.get('pinned-failed-mitm') %}
|
||||
<div class="k">🔒 Cert-pinning détecté</div>
|
||||
<div class="v">{{ b.get('pinned-failed-mitm', 0) }}% — l'app refuse notre CA, normal+bon signe</div>
|
||||
{% endif %}
|
||||
{% if b.get('e2e-opaque') %}
|
||||
<div class="k">🔐 E2E messaging</div>
|
||||
<div class="v">{{ b.get('e2e-opaque', 0) }}% — opaque par design, ton chiffrement marche</div>
|
||||
{% endif %}
|
||||
<div class="k">📊 Total events analysés</div>
|
||||
<div class="v">{{ t.get('total_events', 0) }}</div>
|
||||
{% if t.get('whitelist_stats') %}
|
||||
<div class="k">📜 Patterns whitelist actifs</div>
|
||||
<div class="v">{{ t.get('whitelist_stats', {}).get('count', 0) }} (baseline + override)</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if t.get('per_host') %}
|
||||
<div class="card">
|
||||
<h2>🎯 QUALITÉ SÉCURITÉ PAR DESTINATION</h2>
|
||||
<p style="font-size:0.78rem;color:var(--dim);margin-bottom:0.6rem;font-style:italic">
|
||||
Grade A+/A/B/C/D/F basé sur TLS version + JA4 + headers + cookies. Worst-first :
|
||||
</p>
|
||||
<table style="width:100%;font-size:0.78rem;border-collapse:collapse">
|
||||
<thead>
|
||||
<tr style="color:var(--phos);text-align:left;border-bottom:1px solid var(--dim)">
|
||||
<th style="padding:0.3rem">Grade</th>
|
||||
<th style="padding:0.3rem">Destination</th>
|
||||
<th style="padding:0.3rem">Statut analyse</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for h in t.get('per_host', [])[:15] %}
|
||||
<tr style="border-bottom:1px solid rgba(0,221,68,0.1)">
|
||||
<td style="padding:0.25rem;font-weight:bold;color:{% if h.grade in ['A+','A'] %}var(--phos-hot){% elif h.grade == 'B' %}var(--phos){% elif h.grade == 'C' %}var(--amber){% else %}var(--red){% endif %}">{{ h.grade }}</td>
|
||||
<td style="padding:0.25rem;font-family:monospace;color:var(--text)">{{ h.host[:60] }}</td>
|
||||
<td style="padding:0.25rem;color:var(--dim);font-size:0.72rem">{{ h.status }} — {{ h.reason[:70] }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
<p class="refresh">↻ Auto-refresh toutes les 15 secondes</p>
|
||||
|
||||
<div class="card">
|
||||
|
|
|
|||
226
packages/secubox-toolbox/conf/whitelist-baseline.yaml
Normal file
226
packages/secubox-toolbox/conf/whitelist-baseline.yaml
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||
# CyberMind ToolBoX Whitelist baseline (#492)
|
||||
#
|
||||
# This file is installed to /usr/share/secubox/toolbox/whitelist-baseline.yaml
|
||||
# and reloaded automatically when modified. To override or extend without
|
||||
# touching this file, edit /etc/secubox/toolbox/whitelist.yaml (operator-owned,
|
||||
# survives upgrades).
|
||||
#
|
||||
# Each entry causes the matching FQDN to BYPASS mitmproxy inspection. The
|
||||
# cabine reports the bypass in the user-facing /report : honesty over
|
||||
# magic-numbers. Quality scoring continues passively (TLS version, JA4).
|
||||
|
||||
whitelist:
|
||||
|
||||
# ───────────── Apple ecosystem (cert-pinning ubiquitous) ─────────────
|
||||
- pattern: "*.push.apple.com"
|
||||
reason: "Apple Push Notifications cert-pinned, no MITM possible"
|
||||
category: vendor-os
|
||||
quality_expected: A+
|
||||
- pattern: "*.icloud.com"
|
||||
reason: "Apple iCloud cert-pinned"
|
||||
category: vendor-os
|
||||
quality_expected: A+
|
||||
- pattern: "*.apple.com"
|
||||
reason: "Apple infrastructure (incl. itunes, mzstatic, appstore)"
|
||||
category: vendor-os
|
||||
quality_expected: A+
|
||||
- pattern: "*.aaplimg.com"
|
||||
reason: "Apple CDN"
|
||||
category: vendor-os
|
||||
quality_expected: A
|
||||
- pattern: "*.cdn-apple.com"
|
||||
reason: "Apple CDN"
|
||||
category: vendor-os
|
||||
quality_expected: A
|
||||
- pattern: "*.mzstatic.com"
|
||||
reason: "Apple iTunes/Store CDN"
|
||||
category: vendor-os
|
||||
quality_expected: A
|
||||
- pattern: "smoot.apple.com"
|
||||
reason: "Apple Search/Siri"
|
||||
category: vendor-os
|
||||
quality_expected: A+
|
||||
- pattern: "*.apple-cloudkit.com"
|
||||
reason: "Apple CloudKit"
|
||||
category: vendor-os
|
||||
quality_expected: A+
|
||||
|
||||
# ───────────── Google / Android OS ─────────────
|
||||
- pattern: "*.googleapis.com"
|
||||
reason: "Google APIs, cert-pinned for FCM/auth"
|
||||
category: vendor-os
|
||||
quality_expected: A+
|
||||
- pattern: "*.gstatic.com"
|
||||
reason: "Google CDN"
|
||||
category: vendor-os
|
||||
quality_expected: A
|
||||
- pattern: "fcm.googleapis.com"
|
||||
reason: "Firebase Cloud Messaging, cert-pinned"
|
||||
category: vendor-os
|
||||
quality_expected: A+
|
||||
- pattern: "android.googleapis.com"
|
||||
reason: "Android OS services, cert-pinned"
|
||||
category: vendor-os
|
||||
quality_expected: A+
|
||||
- pattern: "play.googleapis.com"
|
||||
reason: "Google Play Services"
|
||||
category: vendor-os
|
||||
quality_expected: A+
|
||||
|
||||
# ───────────── Messaging E2E (transport-only observation OK) ─────────────
|
||||
- pattern: "*.signal.org"
|
||||
reason: "Signal E2E messenger, content opaque by design"
|
||||
category: messaging-e2e
|
||||
quality_expected: A+
|
||||
- pattern: "*.whispersystems.org"
|
||||
reason: "Signal infrastructure"
|
||||
category: messaging-e2e
|
||||
quality_expected: A+
|
||||
- pattern: "*.threema.ch"
|
||||
reason: "Threema E2E"
|
||||
category: messaging-e2e
|
||||
quality_expected: A+
|
||||
- pattern: "*.simplex.chat"
|
||||
reason: "SimpleX E2E"
|
||||
category: messaging-e2e
|
||||
quality_expected: A+
|
||||
|
||||
# ───────────── Banking FR (CSPN policy : ne JAMAIS MITM la banque) ─────────────
|
||||
- pattern: "*.banque-postale.fr"
|
||||
reason: "Banking, MITM forbidden by ToolBoX policy"
|
||||
category: financial
|
||||
quality_expected: A
|
||||
- pattern: "*.societegenerale.fr"
|
||||
reason: "Banking, MITM forbidden by ToolBoX policy"
|
||||
category: financial
|
||||
quality_expected: A
|
||||
- pattern: "*.bnpparibas.com"
|
||||
reason: "Banking, MITM forbidden by ToolBoX policy"
|
||||
category: financial
|
||||
quality_expected: A
|
||||
- pattern: "*.credit-agricole.fr"
|
||||
reason: "Banking, MITM forbidden by ToolBoX policy"
|
||||
category: financial
|
||||
quality_expected: A
|
||||
- pattern: "*.creditmutuel.fr"
|
||||
reason: "Banking, MITM forbidden by ToolBoX policy"
|
||||
category: financial
|
||||
quality_expected: A
|
||||
- pattern: "*.lcl.fr"
|
||||
reason: "Banking, MITM forbidden by ToolBoX policy"
|
||||
category: financial
|
||||
quality_expected: A
|
||||
- pattern: "*.boursorama.com"
|
||||
reason: "Banking, MITM forbidden by ToolBoX policy"
|
||||
category: financial
|
||||
quality_expected: A
|
||||
- pattern: "*.revolut.com"
|
||||
reason: "Banking (neobank), MITM forbidden"
|
||||
category: financial
|
||||
quality_expected: A+
|
||||
|
||||
# ───────────── Government FR (sensitive citizen data) ─────────────
|
||||
- pattern: "*.service-public.fr"
|
||||
reason: "French government services, MITM forbidden"
|
||||
category: government
|
||||
quality_expected: A
|
||||
- pattern: "*.impots.gouv.fr"
|
||||
reason: "French tax service, MITM forbidden"
|
||||
category: government
|
||||
quality_expected: A
|
||||
- pattern: "*.ameli.fr"
|
||||
reason: "French healthcare, MITM forbidden"
|
||||
category: government
|
||||
quality_expected: A
|
||||
- pattern: "*.france-identite.gouv.fr"
|
||||
reason: "French digital identity, MITM forbidden"
|
||||
category: government
|
||||
quality_expected: A+
|
||||
- pattern: "*.francetravail.fr"
|
||||
reason: "French employment service"
|
||||
category: government
|
||||
quality_expected: A
|
||||
- pattern: "*.caf.fr"
|
||||
reason: "French family allowance"
|
||||
category: government
|
||||
quality_expected: A
|
||||
|
||||
# ───────────── Health (medical confidentiality) ─────────────
|
||||
- pattern: "*.doctolib.fr"
|
||||
reason: "Healthcare booking, medical confidentiality"
|
||||
category: health
|
||||
quality_expected: A
|
||||
- pattern: "*.qare.fr"
|
||||
reason: "Telemedicine, MITM forbidden"
|
||||
category: health
|
||||
quality_expected: A
|
||||
- pattern: "monespacesante.fr"
|
||||
reason: "French shared health record"
|
||||
category: health
|
||||
quality_expected: A+
|
||||
|
||||
# ───────────── Privacy-respecting search / DNS / VPN ─────────────
|
||||
- pattern: "*.duckduckgo.com"
|
||||
reason: "Privacy-respecting search"
|
||||
category: privacy-tools
|
||||
quality_expected: A+
|
||||
- pattern: "*.qwant.com"
|
||||
reason: "Privacy-respecting search FR"
|
||||
category: privacy-tools
|
||||
quality_expected: A
|
||||
- pattern: "*.startpage.com"
|
||||
reason: "Privacy-respecting search"
|
||||
category: privacy-tools
|
||||
quality_expected: A+
|
||||
- pattern: "*.proton.me"
|
||||
reason: "Proton (mail/VPN/drive), MITM forbidden"
|
||||
category: privacy-tools
|
||||
quality_expected: A+
|
||||
- pattern: "*.protonmail.com"
|
||||
reason: "Proton Mail legacy domain"
|
||||
category: privacy-tools
|
||||
quality_expected: A+
|
||||
- pattern: "*.tutanota.com"
|
||||
reason: "Tutanota E2E mail"
|
||||
category: privacy-tools
|
||||
quality_expected: A+
|
||||
- pattern: "*.mullvad.net"
|
||||
reason: "Mullvad VPN, MITM forbidden"
|
||||
category: privacy-tools
|
||||
quality_expected: A+
|
||||
- pattern: "*.tailscale.com"
|
||||
reason: "Tailscale mesh, control plane"
|
||||
category: privacy-tools
|
||||
quality_expected: A+
|
||||
|
||||
# ───────────── Tor (opaque by design + MITM impossible) ─────────────
|
||||
- pattern: "*.torproject.org"
|
||||
reason: "Tor Project (downloads + bridges)"
|
||||
category: anon-network
|
||||
quality_expected: A+
|
||||
|
||||
# ───────────── OS critical updates ─────────────
|
||||
- pattern: "*.windowsupdate.com"
|
||||
reason: "Windows Update infrastructure"
|
||||
category: vendor-os
|
||||
quality_expected: A
|
||||
- pattern: "*.microsoft.com"
|
||||
reason: "Microsoft services (Office365/Auth)"
|
||||
category: vendor-os
|
||||
quality_expected: A
|
||||
- pattern: "*.debian.org"
|
||||
reason: "Debian package repositories"
|
||||
category: vendor-os
|
||||
quality_expected: A
|
||||
- pattern: "*.ubuntu.com"
|
||||
reason: "Ubuntu package repositories"
|
||||
category: vendor-os
|
||||
quality_expected: A
|
||||
|
||||
# ───────────── Captive portal probes (must reach the portal itself, not internet) ─────────────
|
||||
# NOTE : these probes are HANDLED locally by the captive splash, never reach
|
||||
# this whitelist. Listed here for documentation.
|
||||
# - pattern: "captive.apple.com"
|
||||
# - pattern: "connectivitycheck.gstatic.com"
|
||||
# - pattern: "www.msftconnecttest.com"
|
||||
|
|
@ -33,6 +33,41 @@ SALT_FILE = Path("/etc/secubox/secrets/toolbox-mac-salt")
|
|||
_RE_MAC = re.compile(r"lladdr\s+([0-9a-f:]{17})", re.I)
|
||||
_SALT_CACHE: str | None = None
|
||||
|
||||
# Phase 3 (#492) : analysis_status tagging on every event
|
||||
# Soft import : whitelist may not be installed yet (older boards)
|
||||
try:
|
||||
from secubox_core import whitelist as _whitelist
|
||||
except Exception:
|
||||
_whitelist = None
|
||||
|
||||
# Known E2E SNI suffixes — content opaque by design
|
||||
_E2E_SNI_PATTERNS = re.compile(
|
||||
r"(\.signal\.org|\.whispersystems\.org|\.threema\.|\.simplex\.|"
|
||||
r"\.matrix\.org|\.proton\.me|\.protonmail\.|\.tutanota\.)$",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def _analysis_status(host: str | None, sni: str | None, decrypted: bool) -> tuple[str, str]:
|
||||
"""Return (status, reason) for an observed flow.
|
||||
|
||||
Order :
|
||||
1. decrypted -> inspected
|
||||
2. whitelist match -> bypassed-whitelist + reason from yaml
|
||||
3. E2E SNI pattern -> e2e-opaque
|
||||
4. fallback -> pinned-failed-mitm
|
||||
"""
|
||||
if decrypted:
|
||||
return ("inspected", "MITM decryption succeeded via ToolBoX CA")
|
||||
target = (sni or host or "").lower()
|
||||
if _whitelist is not None and target:
|
||||
m = _whitelist.match(target)
|
||||
if m:
|
||||
return ("bypassed-whitelist", m.get("reason", "whitelisted"))
|
||||
if target and _E2E_SNI_PATTERNS.search(target):
|
||||
return ("e2e-opaque", "E2E messenger : content opaque by design")
|
||||
return ("pinned-failed-mitm", "TLS handshake failed (cert-pinning detected)")
|
||||
|
||||
|
||||
def _salt() -> str:
|
||||
global _SALT_CACHE
|
||||
|
|
@ -103,6 +138,9 @@ class LocalStore:
|
|||
if not mac_hash:
|
||||
return
|
||||
host = flow.request.host
|
||||
sni = getattr(flow.client_conn, "sni", None)
|
||||
# If we got here with a parsed request, decryption succeeded.
|
||||
status, reason = _analysis_status(host, sni, decrypted=True)
|
||||
_insert(mac_hash, "dpi", {
|
||||
"client_ip": ip,
|
||||
"host": host,
|
||||
|
|
@ -110,6 +148,8 @@ class LocalStore:
|
|||
"method": flow.request.method,
|
||||
"path": flow.request.path[:200],
|
||||
"user_agent": flow.request.headers.get("user-agent"),
|
||||
"analysis_status": status,
|
||||
"analysis_reason": reason,
|
||||
})
|
||||
|
||||
def response(self, flow: http.HTTPFlow) -> None:
|
||||
|
|
@ -160,6 +200,27 @@ class LocalStore:
|
|||
"weight": 15,
|
||||
})
|
||||
|
||||
def tls_failed_clienthello(self, data) -> None:
|
||||
"""Phase 3 (#492) : record cert-pinning / whitelisted bypasses.
|
||||
|
||||
Fires when TLS handshake fails (cert-pinning) OR when we deliberately
|
||||
skipped MITM (whitelist). We log the SNI + status so the report can
|
||||
be honest about what we didn't inspect.
|
||||
"""
|
||||
ip = data.context.client.peername[0] if data.context.client.peername else None
|
||||
mac_hash = _hash_mac(_mac_of(ip))
|
||||
if not mac_hash:
|
||||
return
|
||||
sni = getattr(data.client_hello, "sni", None) if hasattr(data, "client_hello") else None
|
||||
status, reason = _analysis_status(host=None, sni=sni, decrypted=False)
|
||||
_insert(mac_hash, "dpi", {
|
||||
"client_ip": ip,
|
||||
"host": sni or "?",
|
||||
"sni": sni,
|
||||
"analysis_status": status,
|
||||
"analysis_reason": reason,
|
||||
})
|
||||
|
||||
def tls_clienthello(self, data) -> None:
|
||||
ip = data.context.client.peername[0] if data.context.client.peername else None
|
||||
mac_hash = _hash_mac(_mac_of(ip))
|
||||
|
|
|
|||
|
|
@ -27,6 +27,13 @@ from . import (
|
|||
store,
|
||||
threat_intel,
|
||||
)
|
||||
# Phase 3 (#492) : transparency layer
|
||||
try:
|
||||
from secubox_core import whitelist as _whitelist_mod
|
||||
from secubox_core.classifiers import security_quality as _sec_quality
|
||||
_HAS_TRANSPARENCY = True
|
||||
except ImportError:
|
||||
_HAS_TRANSPARENCY = False
|
||||
from .config import load_config, resolve_secret
|
||||
from .models import AcceptResp, ClientRow, Config, StatusResp
|
||||
|
||||
|
|
@ -258,6 +265,9 @@ def _aggregate_session(mac_hash: str) -> dict:
|
|||
soc_indicators: list[dict] = []
|
||||
ja4_snis: set[str] = set()
|
||||
ja4_alpns: set[str] = set()
|
||||
# Phase 3 (#492) : transparency layer state
|
||||
analysis_breakdown: dict[str, int] = {}
|
||||
host_analysis: dict[str, dict] = {}
|
||||
try:
|
||||
with _sq3.connect("/var/lib/secubox/toolbox/toolbox.db", timeout=2) as c:
|
||||
cur = c.execute(
|
||||
|
|
@ -279,6 +289,15 @@ def _aggregate_session(mac_hash: str) -> dict:
|
|||
ua = p.get("user_agent")
|
||||
if ua:
|
||||
user_agents.add(ua[:80])
|
||||
# Phase 3 (#492) : analysis_status breakdown
|
||||
status = p.get("analysis_status")
|
||||
if status:
|
||||
analysis_breakdown[status] = analysis_breakdown.get(status, 0) + 1
|
||||
# Keep a per-host status for the quality table
|
||||
host_analysis[host] = {
|
||||
"status": status,
|
||||
"reason": p.get("analysis_reason", ""),
|
||||
}
|
||||
elif source == "cookies":
|
||||
cookies_total_set += p.get("set_cookie_count", 0)
|
||||
cookies_total_sent += p.get("cookie_count", 0)
|
||||
|
|
@ -417,6 +436,78 @@ def _aggregate_session(mac_hash: str) -> dict:
|
|||
"cookies_providers": cookie_analysis.top_providers(cookies_urls, limit=10),
|
||||
# ── Phase 2b (#488) : pull events from receiving modules ──
|
||||
"mitm_modules": _pull_mitm_module_events(mac_hash),
|
||||
# ── Phase 3 (#492) : transparency layer ──
|
||||
"transparency": _build_transparency(
|
||||
analysis_breakdown, host_analysis, dpi_hosts, ja4_snis,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _build_transparency(
|
||||
breakdown: dict[str, int],
|
||||
host_analysis: dict[str, dict],
|
||||
dpi_hosts: dict[str, int],
|
||||
ja4_snis: set,
|
||||
) -> dict:
|
||||
"""Phase 3 (#492) : pack the inspection breakdown + per-host quality table.
|
||||
|
||||
The breakdown shows what % of traffic was inspected / bypassed / pinned /
|
||||
e2e. The per-host quality table grades each destination via passive or
|
||||
active signals (currently passive, since header capture is not yet wired).
|
||||
"""
|
||||
total = sum(breakdown.values()) or 1
|
||||
pct = {k: round(100 * v / total, 1) for k, v in breakdown.items()}
|
||||
|
||||
# Backfill any host without explicit analysis_status (legacy events)
|
||||
for h in dpi_hosts:
|
||||
if h not in host_analysis:
|
||||
# Best-effort post-hoc tag : whitelist match → bypass, else inspected
|
||||
target = h.lower()
|
||||
if _HAS_TRANSPARENCY and _whitelist_mod.is_whitelisted(target):
|
||||
host_analysis[h] = {
|
||||
"status": "bypassed-whitelist",
|
||||
"reason": (_whitelist_mod.match(target) or {}).get("reason", ""),
|
||||
}
|
||||
else:
|
||||
host_analysis[h] = {
|
||||
"status": "inspected",
|
||||
"reason": "MITM decryption (post-hoc tag)",
|
||||
}
|
||||
|
||||
# Build per-host quality (passive grading from what we have)
|
||||
per_host: list[dict] = []
|
||||
if _HAS_TRANSPARENCY:
|
||||
for h, info in host_analysis.items():
|
||||
status = info.get("status", "inspected")
|
||||
is_e2e = status == "e2e-opaque"
|
||||
# We don't know tls_version per host without further capture ; assume
|
||||
# 13 for whitelisted (cert-pinned apps use modern TLS) and unknown otherwise
|
||||
tls_v = "13" if status in ("bypassed-whitelist", "pinned-failed-mitm", "e2e-opaque") else None
|
||||
g = _sec_quality.grade_passive(
|
||||
tls_version=tls_v,
|
||||
sni=h if h in ja4_snis else None,
|
||||
is_e2e_messaging=is_e2e,
|
||||
)
|
||||
per_host.append({
|
||||
"host": h,
|
||||
"grade": g["grade"],
|
||||
"score": g["score"],
|
||||
"status": status,
|
||||
"reason": info.get("reason", ""),
|
||||
})
|
||||
# Sort : worst quality first (catches user attention)
|
||||
per_host.sort(key=lambda x: (x["score"], x["host"]))
|
||||
|
||||
# Whitelist sanity stats
|
||||
wl_stats = _whitelist_mod.stats() if _HAS_TRANSPARENCY else {"count": 0}
|
||||
|
||||
return {
|
||||
"breakdown": breakdown,
|
||||
"breakdown_pct": pct,
|
||||
"total_events": total,
|
||||
"per_host": per_host[:50],
|
||||
"whitelist_stats": wl_stats,
|
||||
"has_transparency": _HAS_TRANSPARENCY,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -249,6 +249,36 @@ def render_pdf(report: dict) -> bytes:
|
|||
_bullet(pdf, rec)
|
||||
pdf.ln(2)
|
||||
|
||||
# Phase 3 (#492) : Transparency — inspection breakdown + per-host quality
|
||||
t = report.get("transparency") or {}
|
||||
if t.get("total_events"):
|
||||
_section(pdf, "INSPECTION : CE QUI A ETE REGARDE (et pas regarde)")
|
||||
b = t.get("breakdown_pct") or {}
|
||||
if b.get("inspected"):
|
||||
_bullet(pdf, f"Inspecte (MITM via notre CA) : {b['inspected']}% - contenu visible")
|
||||
if b.get("bypassed-whitelist"):
|
||||
_bullet(pdf, f"Bypass whitelist : {b['bypassed-whitelist']}% - decision policy (vendor cert-pinning)")
|
||||
if b.get("pinned-failed-mitm"):
|
||||
_bullet(pdf, f"Cert-pinning : {b['pinned-failed-mitm']}% - app refuse notre CA, normal+bon signe")
|
||||
if b.get("e2e-opaque"):
|
||||
_bullet(pdf, f"E2E messaging : {b['e2e-opaque']}% - opaque par design, ton chiffrement marche")
|
||||
_bullet(pdf, f"Total events analyses : {t.get('total_events', 0)}")
|
||||
wl = (t.get("whitelist_stats") or {}).get("count", 0)
|
||||
if wl:
|
||||
_bullet(pdf, f"Patterns whitelist actifs : {wl} (baseline + override operateur)")
|
||||
pdf.ln(2)
|
||||
|
||||
# Per-host quality table — worst first, capped 10
|
||||
per_host = t.get("per_host") or []
|
||||
if per_host:
|
||||
_section(pdf, "QUALITE SECURITE PAR DESTINATION (worst-first)")
|
||||
for h in per_host[:10]:
|
||||
grade = h.get("grade", "?")
|
||||
host = h.get("host", "?")[:50]
|
||||
status = h.get("status", "?")
|
||||
_bullet(pdf, f"[{grade}] {host} - {status}", font_size=8)
|
||||
pdf.ln(2)
|
||||
|
||||
# Retention
|
||||
_section(pdf, "RETENTION DES DONNEES")
|
||||
_bullet(pdf, "Hash MAC anonyme : 24h (sel rotatif quotidien)")
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user