feat(#662 Phase 3): port toolbox BLOCK/SPLICE logic into Go core

Add cmd/sbxmitm/policy.go: a LoadPolicy() layer that reads the SAME on-disk
config the Python addons use (ad-allowlist.txt, learned-trackers.txt,
tls-splice-seed.conf, splice-learned.txt, pure-trackers.txt) with the same
env overrides, plus a unified Decide(host, sni) -> {allow,block,splice,mitm}.

Ports, byte-for-byte against the Python source of truth:
  - _AD_HOST regex (RE2-safe → Go (?i) inline flag, no fallback needed)
  - _registrable incl. the _2L two-level-TLD list
  - splice.host_matches / should_splice (never wins; then seed∪learned)
  - ad_ghost._allowed (own-infra + allowlist ALWAYS win first)

Loader nuance preserved: ad_ghost._learned_set does NOT comment-strip
(machine-generated file), unlike the splice/allowlist loaders — mirrored via
loadLinesRaw vs loadLines so a '#' in learned-trackers is kept verbatim.

Decide precedence: allow > splice (never-set excludes trackers) > block > mitm.

Wire the loaded policy into the PoC CONNECT proxy (replacing the hardcoded
AdHosts/SpliceHosts); action() keeps the legacy 3-verb surface (allow→mitm).
Old TestActionDecision removed (drove the removed hardcoded fields); coverage
moves to the parity harness.
This commit is contained in:
CyberMind-FR 2026-06-18 17:32:20 +02:00
parent 84f0a37fdf
commit 25a3afaff1
3 changed files with 391 additions and 53 deletions

View File

@ -129,38 +129,14 @@ func (c *CA) forge(host string) (*tls.Certificate, error) {
return tc, nil
}
// ── Pure handler logic (the ported addon decisions) ─────────────────────────
type Policy struct {
AdHosts []string // ad_ghost: 204 these (suffix match)
SpliceHosts []string // tls_splice: passthrough, no MITM (suffix match)
Inject []byte // banner / ad-CSS marker injected before </head> or </body>
}
func suffixMatch(host string, pats []string) bool {
h := strings.ToLower(strings.TrimSpace(host))
for _, p := range pats {
p = strings.ToLower(p)
if h == p || strings.HasSuffix(h, "."+p) {
return true
}
}
return false
}
// action: "block" (204), "splice" (passthrough), or "mitm".
func (p Policy) action(host string) string {
if suffixMatch(host, p.SpliceHosts) {
return "splice"
}
if suffixMatch(host, p.AdHosts) {
return "block"
}
return "mitm"
}
// ── Pure handler logic ───────────────────────────────────────────────────────
//
// The decision surface (Decide / action / registrable / splice helpers) lives
// in policy.go, ported from the Python addons and proven at parity by the
// cross-engine harness. The body-inject helper is kept here next to the wiring.
// injectMarker inserts p.Inject before </head> (else </body>, else prepends).
func (p Policy) injectMarker(body []byte) []byte {
func (p *Policy) injectMarker(body []byte) []byte {
if len(p.Inject) == 0 || bytes.Contains(body, p.Inject) {
return body
}
@ -201,7 +177,7 @@ func ja4ish(h *tls.ClientHelloInfo) string {
type Proxy struct {
ca *CA
pol Policy
pol *Policy
jaSink func(string) // JA4 observations (logged; a sidecar in prod)
}
@ -290,13 +266,17 @@ func main() {
if err != nil {
log.Fatalf("CA load: %v", err)
}
// Load the BLOCK/SPLICE policy from the SAME on-disk config the Python
// addons read (defaults + env overrides). Missing files are tolerated
// (best-effort, like the addons): the engine then simply MITMs everything.
pol, err := LoadPolicy(PolicyOpts{})
if err != nil {
log.Fatalf("policy load: %v", err)
}
pol.Inject = []byte("<!-- sbx-ng banner -->")
px := &Proxy{
ca: ca,
pol: Policy{
AdHosts: []string{"doubleclick.net", "googlesyndication.com"},
SpliceHosts: []string{"googlevideo.com", "fbcdn.net"},
Inject: []byte("<!-- sbx-ng banner -->"),
},
ca: ca,
pol: pol,
jaSink: func(s string) { log.Printf("ja4 %s", s) },
}
srv := &http.Server{Addr: *addr, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

View File

@ -72,24 +72,13 @@ func TestForgeChainsToCA(t *testing.T) {
}
}
func TestActionDecision(t *testing.T) {
p := Policy{AdHosts: []string{"doubleclick.net"}, SpliceHosts: []string{"googlevideo.com"}}
cases := map[string]string{
"ads.doubleclick.net": "block",
"doubleclick.net": "block",
"r1.googlevideo.com": "splice",
"news.example.com": "mitm",
"notdoubleclick.net": "mitm",
}
for host, want := range cases {
if got := p.action(host); got != want {
t.Errorf("action(%q)=%q want %q", host, got, want)
}
}
}
// NOTE (#662 Phase 3): the old TestActionDecision drove the removed hardcoded
// Policy{AdHosts, SpliceHosts} fields. The decision surface now loads from
// disk (LoadPolicy) and mirrors the Python addons; coverage moved to
// TestParityDecide / TestPolicyActionVerbs in policy_test.go.
func TestInjectMarker(t *testing.T) {
p := Policy{Inject: []byte("<!--SBX-->")}
p := &Policy{Inject: []byte("<!--SBX-->")}
out := string(p.injectMarker([]byte("<html><head></head><body>hi</body></html>")))
if !contains(out, "<!--SBX--></head>") {
t.Fatalf("marker not injected before </head>: %s", out)

View File

@ -0,0 +1,369 @@
// SPDX-License-Identifier: LicenseRef-CMSD-1.0
// Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
//
// SecuBox-Deb :: toolbox-ng :: policy layer (#662 Phase 3)
//
// Ports the toolbox BLOCK (ad_ghost) and SPLICE (tls_splice) decision logic
// into the Go core, reading the SAME on-disk config files the Python addons
// use. Python is the source of truth; this mirrors it byte-for-byte on the
// decision surface, proven by the cross-engine parity harness
// (testdata/parity-fixtures.json + policy_test.go ↔ tests/test_engine_parity.py).
//
// Pure standard library — no external modules, no go.sum.
package main
import (
"bufio"
"os"
"regexp"
"strings"
)
// ── ad_ghost: static ad/tracker host pattern (port of _AD_HOST) ──────────────
//
// Python (mitmproxy_addons/ad_ghost.py):
//
// _AD_HOST = re.compile(
// r"(?:^|\.)(?:doubleclick|googlesyndication|googleadservices|"
// r"googletagservices|adservice\.google|amazon-adsystem|adnxs|adsrvr|"
// r"adform|criteo|rubiconproject|taboola|outbrain|smartadserver|moatads|"
// r"scorecardresearch|2mdn|adroll|pubmatic|openx|casalemedia|"
// r"yieldlove|sharethrough|teads|3lift|adsystem|adserver)",
// re.IGNORECASE)
//
// Every construct here — non-capturing groups, `^`, `\.`, alternation, the
// case-insensitive flag — is RE2-safe, so it translates 1:1 to Go regexp via
// the `(?i)` inline flag. No fallback substring split was needed.
const adHostPattern = `(?i)(?:^|\.)(?:doubleclick|googlesyndication|googleadservices|` +
`googletagservices|adservice\.google|amazon-adsystem|adnxs|adsrvr|` +
`adform|criteo|rubiconproject|taboola|outbrain|smartadserver|moatads|` +
`scorecardresearch|2mdn|adroll|pubmatic|openx|casalemedia|` +
`yieldlove|sharethrough|teads|3lift|adsystem|adserver)`
// _2L_TLD: two-level public suffixes (port of ad_ghost._2L_TLD).
var twoLevelTLD = map[string]bool{
"co.uk": true, "com.au": true, "co.jp": true, "co.nz": true,
"com.br": true, "co.za": true, "gouv.fr": true,
}
// ── PolicyOpts: configurable file paths (env-overridable, like Python) ───────
// PolicyOpts holds the on-disk paths the loaders read. Empty fields fall back
// to the real production defaults (or the env override) in LoadPolicy.
type PolicyOpts struct {
AllowPath string // ad-allowlist.txt (_ALLOW_PATH)
LearnedPath string // learned-trackers.txt (_LEARNED_PATH)
SpliceSeedPath string // conf/tls-splice-seed.conf (SEED_PATH)
SpliceLearnPath string // splice-learned.txt (LEARNED_PATH)
PureTrackersPath string // pure-trackers.txt (PURE_PATH)
FortknoxSites []string // filters.json fortknox_sites
SelfDomains []string // _SELF_REGS (default {secubox.in}, env SECUBOX_SELF_DOMAINS)
}
// defaultPolicyOpts returns the production defaults, honoring the same env vars
// the Python addons read.
func defaultPolicyOpts() PolicyOpts {
o := PolicyOpts{
AllowPath: "/var/lib/secubox/toolbox/ad-allowlist.txt",
LearnedPath: "/var/lib/secubox/toolbox/learned-trackers.txt",
SpliceSeedPath: envOr("SECUBOX_SPLICE_SEED", "/usr/lib/secubox/toolbox/conf/tls-splice-seed.conf"),
SpliceLearnPath: envOr("SECUBOX_SPLICE_LEARNED", "/var/lib/secubox/toolbox/splice-learned.txt"),
PureTrackersPath: envOr("SECUBOX_PURE_TRACKERS", "/var/lib/secubox/toolbox/pure-trackers.txt"),
}
// _SELF_REGS: env SECUBOX_SELF_DOMAINS (comma-split), default {secubox.in}.
self := os.Getenv("SECUBOX_SELF_DOMAINS")
if strings.TrimSpace(self) == "" {
self = "secubox.in"
}
for _, d := range strings.Split(self, ",") {
if d = strings.TrimSpace(strings.ToLower(d)); d != "" {
o.SelfDomains = append(o.SelfDomains, d)
}
}
return o
}
func envOr(key, def string) string {
if v := os.Getenv(key); v != "" {
return v
}
return def
}
// ── Policy: the loaded decision state ────────────────────────────────────────
// Policy carries the loaded sets/regex and decides per-host actions. It also
// keeps the legacy PoC fields (Inject) so the existing wiring/tests still work.
type Policy struct {
adHost *regexp.Regexp
learned map[string]bool // learned-trackers (host or registrable, lowercased)
allow map[string]bool // ad-allowlist (host or registrable, lowercased)
spliceSeed map[string]bool // splice seed patterns
spliceLearn map[string]bool // splice learned patterns
never map[string]bool // pure-trackers fortknox (splice never-set)
selfRegs map[string]bool // own-infra registrable domains
selfDomains []string // own-infra (for the host==d || host endswith .d guard)
// Legacy PoC fields kept so non-policy behaviour is unchanged.
Inject []byte // banner / ad-CSS marker injected before </head> or </body>
}
// loadLines mirrors the comment-stripping Python loaders (splice._load_lines,
// ad_ghost._allowed's allowlist read): split on first '#', trim, lowercase,
// skip blanks. Missing/unreadable file → empty set (best-effort).
func loadLines(path string) map[string]bool {
return scanLines(path, true)
}
// loadLinesRaw mirrors ad_ghost._learned_set, which does NOT comment-strip —
// learned-trackers.txt is a machine-generated one-host-per-line file. It does
// `{ln.strip().lower() for ln in f if ln.strip()}`. Matching this exactly is
// load-bearing for parity (a '#' in this file would be kept verbatim, not a
// comment), so the Go core must mirror the divergent behaviour, not normalise it.
func loadLinesRaw(path string) map[string]bool {
return scanLines(path, false)
}
func scanLines(path string, stripComments bool) map[string]bool {
out := map[string]bool{}
f, err := os.Open(path)
if err != nil {
return out
}
defer f.Close()
sc := bufio.NewScanner(f)
sc.Buffer(make([]byte, 0, 64*1024), 1<<20)
for sc.Scan() {
ln := sc.Text()
if stripComments {
if i := strings.IndexByte(ln, '#'); i >= 0 {
ln = ln[:i]
}
}
ln = strings.ToLower(strings.TrimSpace(ln))
if ln != "" {
out[ln] = true
}
}
return out
}
// LoadPolicy loads all backing files from opts (defaults applied for empty
// fields) and compiles the ad-host regex. It never returns an error for missing
// files (best-effort, like the Python addons), only for a regex-compile bug.
func LoadPolicy(opts PolicyOpts) (*Policy, error) {
def := defaultPolicyOpts()
if opts.AllowPath == "" {
opts.AllowPath = def.AllowPath
}
if opts.LearnedPath == "" {
opts.LearnedPath = def.LearnedPath
}
if opts.SpliceSeedPath == "" {
opts.SpliceSeedPath = def.SpliceSeedPath
}
if opts.SpliceLearnPath == "" {
opts.SpliceLearnPath = def.SpliceLearnPath
}
if opts.PureTrackersPath == "" {
opts.PureTrackersPath = def.PureTrackersPath
}
if len(opts.SelfDomains) == 0 {
opts.SelfDomains = def.SelfDomains
}
re, err := regexp.Compile(adHostPattern)
if err != nil {
return nil, err
}
// never-set = pure-trackers fortknox_sites (mirrors TlsSplice._refresh_sets).
never := loadLines(opts.PureTrackersPath)
for _, s := range opts.FortknoxSites {
if s = strings.Trim(strings.ToLower(strings.TrimSpace(s)), "."); s != "" {
never[s] = true
}
}
selfRegs := map[string]bool{}
selfDomains := make([]string, 0, len(opts.SelfDomains))
for _, d := range opts.SelfDomains {
d = strings.ToLower(strings.TrimSpace(d))
if d == "" {
continue
}
selfRegs[d] = true
selfDomains = append(selfDomains, d)
}
return &Policy{
adHost: re,
learned: loadLinesRaw(opts.LearnedPath), // mirrors _learned_set (no comment-strip)
allow: loadLines(opts.AllowPath),
spliceSeed: loadLines(opts.SpliceSeedPath),
spliceLearn: loadLines(opts.SpliceLearnPath),
never: never,
selfRegs: selfRegs,
selfDomains: selfDomains,
}, nil
}
// ── registrable: port of ad_ghost._registrable ───────────────────────────────
//
// host = host.split(":")[0].lower().strip(".")
// if not host or host.replace(".","").isdigit() or ":" in host: return None
// p = host.split(".")
// if len(p) <= 2: return host
// last2 = ".".join(p[-2:])
// return ".".join(p[-3:]) if (last2 in _2L_TLD and len(p) >= 3) else last2
func registrable(host string) string {
host = strings.ToLower(host)
if i := strings.IndexByte(host, ':'); i >= 0 {
host = host[:i]
}
host = strings.Trim(host, ".")
if host == "" {
return ""
}
// host.replace(".","").isdigit() → all-digit IPv4-ish → no registrable.
if isAllDigits(strings.ReplaceAll(host, ".", "")) {
return ""
}
// The Python checks ":" in host AFTER stripping the port; a residual colon
// (e.g. an IPv6 literal) yields None. We already split on the first colon,
// so re-check the remainder for any colon to mirror exactly.
if strings.IndexByte(host, ':') >= 0 {
return ""
}
p := strings.Split(host, ".")
if len(p) <= 2 {
return host
}
last2 := strings.Join(p[len(p)-2:], ".")
if twoLevelTLD[last2] && len(p) >= 3 {
return strings.Join(p[len(p)-3:], ".")
}
return last2
}
func isAllDigits(s string) bool {
if s == "" {
return false // Python "".isdigit() is False
}
for _, r := range s {
if r < '0' || r > '9' {
return false
}
}
return true
}
// ── splice helpers: port of splice.host_matches / should_splice ──────────────
// hostMatches: True if host == pattern OR host is a dotted-suffix subdomain.
func hostMatches(host string, patterns map[string]bool) bool {
h := strings.Trim(strings.ToLower(host), ".")
if h == "" || len(patterns) == 0 {
return false
}
if patterns[h] {
return true
}
for p := range patterns {
if strings.HasSuffix(h, "."+p) {
return true
}
}
return false
}
// allowed: port of ad_ghost._allowed. Own-infra ALWAYS wins (reflash-safe),
// then the operator allowlist (host or registrable).
func (p *Policy) allowed(host string) bool {
h := strings.ToLower(host)
reg := registrable(h)
if reg == "" {
reg = h
}
// own infra: registrable in selfRegs, OR host == d || host endswith "."+d.
if p.selfRegs[reg] {
return true
}
for _, d := range p.selfDomains {
if h == d || strings.HasSuffix(h, "."+d) {
return true
}
}
return p.allow[h] || p.allow[reg]
}
// shouldSplice: port of splice.should_splice (never wins; then seed learned).
func (p *Policy) shouldSplice(sni string) bool {
s := strings.Trim(strings.ToLower(sni), ".")
if s == "" {
return false
}
if hostMatches(s, p.never) {
return false
}
return hostMatches(s, p.spliceSeed) || hostMatches(s, p.spliceLearn)
}
// blockedByAd: port of the ad_ghost requestheaders block decision (sans the
// allowlist guard, which Decide applies first): _AD_HOST match OR
// registrable/host in learned-trackers.
func (p *Policy) blockedByAd(host string) bool {
if p.adHost.MatchString(host) {
return true
}
reg := registrable(host)
if reg != "" && p.learned[reg] {
return true
}
return p.learned[strings.ToLower(host)]
}
// ── Decide: the unified cross-engine decision ────────────────────────────────
//
// action ∈ {"allow","block","splice","mitm"}. Precedence (mirrors the Python
// across the two addons, documented in the harness):
//
// 1. own-infra / allowlist → "allow" (ad_ghost._allowed; never block/splice)
// 2. splice never-set check, then seed/learned → "splice"
// (tls_splice runs FIRST at the TLS layer; should_splice already excludes
// the never-set = pure-trackers fortknox, so a tracker that is also a
// splice candidate fails should_splice here and falls through to block)
// 3. _AD_HOST / learned → "block" (ad_ghost requestheaders, request layer)
// 4. otherwise → "mitm"
//
// sni defaults to host when empty (the live engine splices on SNI == the TLS
// host; for the parity harness host and sni are the same value).
func (p *Policy) Decide(host, sni string) string {
if sni == "" {
sni = host
}
if p.allowed(host) {
return "allow"
}
if p.shouldSplice(sni) {
return "splice"
}
if p.blockedByAd(host) {
return "block"
}
return "mitm"
}
// action keeps the legacy 3-verb surface (block/splice/mitm) for the PoC
// CONNECT wiring, derived from Decide: "allow" collapses to "mitm" (an
// allowlisted host is intercepted normally, just never short-circuited).
func (p *Policy) action(host string) string {
switch p.Decide(host, host) {
case "splice":
return "splice"
case "block":
return "block"
default: // "allow" and "mitm" both → normal interception
return "mitm"
}
}