mirror of
https://github.com/CyberMind-FR/secubox-deb.git
synced 2026-07-28 21:17:36 +00:00
The #662 Go cutover blocked from STATIC lists but never (1) emitted learning candidates nor (2) live-reloaded the lists, so new adwares slipped through forever and even autolearn promotions needed a worker restart. Restore the full loop: feeders/outsiders -> lock to blocklist -> silence (204) -> smog (poison) -> statistify, fed by BOTH the ad-path heuristic AND cross-site cookie reuse (the social graph). Go (packages/secubox-toolbox-ng): - policy.go: mtime-based live-reload (Part 1, linchpin). Policy now holds the backing file paths + per-file last-mtime; maybeReload() (throttled ~15s) re-stats each file and atomically swaps the changed map under an RWMutex. Decide/shouldPoison take the read lock; allowedSafe() is the lock-taking entry for the candidate feed. Covers learned-trackers + ad-allowlist + splice seed/learned + pure-trackers. Promotions/edits now take effect with NO worker restart. - adstats.go: ad-candidate learning feed (Part 2). Ports ad_ghost._AD_PATH (RE2) + a (host,site)->hits aggregator (cap 20k), drained into the existing ad-event payload's new "candidates" list by the same 10s flusher. - main.go: maybeRecordAdCandidate() on the allow/mitm branch — 3rd-party (registrable(host) != registrable(site)) AND _AD_PATH match, gated behind the analysis relay flag, O(1) fire-and-forget. Python (packages/secubox-toolbox): - api.py: /__toolbox/ad-event now ingests "candidates" -> store.record_ad_candidates(); capped, try/except, never 500s. - secubox-toolbox-autolearn: new _social_feed() promotes any cross-site cookie-reuse tracker (>= SECUBOX_SOCIAL_MIN_SITES distinct src_site in a recent window) from social_edges into learned-trackers.txt, reusing the _ad_feed allowlist/self guard and merge/de-dup. Smog: confirmed isTracker() already consults the live-reloaded learned set (blockedByAd), so a promoted cross-site tracker is poisoned automatically once the policy reloads it — no new poison code. TDD: reload_test.go (incl. -race concurrency), adcand_test.go, test_ad_event_candidates.py, test_autolearn_socialfeed.py. Go build (offline arm64 + darwin), vet, go test -race all green.
195 lines
5.8 KiB
Go
195 lines
5.8 KiB
Go
// SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
|
// Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
|
//
|
|
// SecuBox-Deb :: toolbox-ng :: ad-block metrics aggregator tests (#662)
|
|
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
)
|
|
|
|
func TestRecordAdBlockAggregatesGlobal(t *testing.T) {
|
|
a := newAdStats()
|
|
a.recordAdBlock("ads.example.com", "news.example", "")
|
|
a.recordAdBlock("ads.example.com", "news.example", "")
|
|
a.recordAdBlock("ads.example.com", "blog.example", "")
|
|
|
|
if got := len(a.blocks); got != 2 {
|
|
t.Fatalf("want 2 (adHost,site) keys, got %d", got)
|
|
}
|
|
c := a.blocks[adKey{"ads.example.com", "news.example"}]
|
|
if c == nil || c.hits != 2 || c.bytes != 2*adEstBytesPerBlock {
|
|
t.Fatalf("news.example counter wrong: %+v", c)
|
|
}
|
|
c = a.blocks[adKey{"ads.example.com", "blog.example"}]
|
|
if c == nil || c.hits != 1 || c.bytes != adEstBytesPerBlock {
|
|
t.Fatalf("blog.example counter wrong: %+v", c)
|
|
}
|
|
}
|
|
|
|
func TestRecordAdBlockEmptyHostIgnored(t *testing.T) {
|
|
a := newAdStats()
|
|
a.recordAdBlock("", "site", "mac")
|
|
if len(a.blocks) != 0 || len(a.clients) != 0 {
|
|
t.Fatalf("empty adHost must be ignored: blocks=%d clients=%d", len(a.blocks), len(a.clients))
|
|
}
|
|
}
|
|
|
|
func TestRecordAdBlockPerClientOnlyWhenMacSet(t *testing.T) {
|
|
a := newAdStats()
|
|
a.recordAdBlock("ads.example.com", "site", "") // no mac → no client row
|
|
a.recordAdBlock("ads.example.com", "site", "mac1") // mac → client row
|
|
a.recordAdBlock("ads.example.com", "site", "mac1")
|
|
|
|
if len(a.clients) != 1 {
|
|
t.Fatalf("want 1 client key, got %d", len(a.clients))
|
|
}
|
|
c := a.clients[cliKey{"mac1", "ads.example.com"}]
|
|
if c == nil || c.hits != 2 || c.bytes != 2*adEstBytesPerBlock {
|
|
t.Fatalf("client counter wrong: %+v", c)
|
|
}
|
|
// Global counter accumulated all three hits regardless of mac.
|
|
if g := a.blocks[adKey{"ads.example.com", "site"}]; g == nil || g.hits != 3 {
|
|
t.Fatalf("global counter should be 3: %+v", g)
|
|
}
|
|
}
|
|
|
|
func TestSnapshotClearsMaps(t *testing.T) {
|
|
a := newAdStats()
|
|
a.recordAdBlock("ads.example.com", "site", "mac1")
|
|
p := a.snapshot()
|
|
if len(p.Blocks) != 1 || len(p.Clients) != 1 {
|
|
t.Fatalf("snapshot payload wrong: %d blocks %d clients", len(p.Blocks), len(p.Clients))
|
|
}
|
|
if len(a.blocks) != 0 || len(a.clients) != 0 {
|
|
t.Fatalf("snapshot must clear the maps: blocks=%d clients=%d", len(a.blocks), len(a.clients))
|
|
}
|
|
// A second snapshot with nothing recorded is empty.
|
|
if p2 := a.snapshot(); !p2.empty() {
|
|
t.Fatalf("empty snapshot must report empty: %+v", p2)
|
|
}
|
|
}
|
|
|
|
func TestSizeCapDropsNewKeysGracefully(t *testing.T) {
|
|
a := newAdStats()
|
|
// Fill past the cap with distinct keys.
|
|
for i := 0; i < adMapCap+50; i++ {
|
|
host := "ads" + itoa(i) + ".example.com"
|
|
a.recordAdBlock(host, "site", "mac"+itoa(i))
|
|
}
|
|
if len(a.blocks) > adMapCap {
|
|
t.Fatalf("blocks map exceeded cap: %d > %d", len(a.blocks), adMapCap)
|
|
}
|
|
if len(a.clients) > adMapCap {
|
|
t.Fatalf("clients map exceeded cap: %d > %d", len(a.clients), adMapCap)
|
|
}
|
|
// An EXISTING key still accumulates even when the map is at cap.
|
|
existing := "ads0.example.com"
|
|
before := a.blocks[adKey{existing, "site"}].hits
|
|
a.recordAdBlock(existing, "site", "mac0")
|
|
after := a.blocks[adKey{existing, "site"}].hits
|
|
if after != before+1 {
|
|
t.Fatalf("existing key must keep accumulating at cap: %d -> %d", before, after)
|
|
}
|
|
}
|
|
|
|
func TestFlushOncePayloadShapeMatchesContract(t *testing.T) {
|
|
a := newAdStats()
|
|
a.recordAdBlock("ads.example.com", "news.example", "mac1")
|
|
|
|
var got adEventPayload
|
|
var ct string
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
ct = r.Header.Get("Content-Type")
|
|
body, _ := io.ReadAll(r.Body)
|
|
_ = json.Unmarshal(body, &got)
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}))
|
|
defer srv.Close()
|
|
|
|
a.flushOnce(srv.URL, nil)
|
|
|
|
if ct != "application/json" {
|
|
t.Fatalf("Content-Type = %q, want application/json", ct)
|
|
}
|
|
if len(got.Blocks) != 1 {
|
|
t.Fatalf("blocks payload: %+v", got.Blocks)
|
|
}
|
|
b := got.Blocks[0]
|
|
if b.AdHost != "ads.example.com" || b.Site != "news.example" || b.Hits != 1 || b.Bytes != adEstBytesPerBlock {
|
|
t.Fatalf("block row shape wrong: %+v", b)
|
|
}
|
|
if len(got.Clients) != 1 {
|
|
t.Fatalf("clients payload: %+v", got.Clients)
|
|
}
|
|
c := got.Clients[0]
|
|
if c.MacHash != "mac1" || c.AdHost != "ads.example.com" || c.Hits != 1 || c.Bytes != adEstBytesPerBlock {
|
|
t.Fatalf("client row shape wrong: %+v", c)
|
|
}
|
|
|
|
// flushOnce cleared the maps.
|
|
if len(a.blocks) != 0 || len(a.clients) != 0 {
|
|
t.Fatalf("flushOnce must clear maps")
|
|
}
|
|
}
|
|
|
|
func TestFlushOnceEmptySkipsPost(t *testing.T) {
|
|
a := newAdStats()
|
|
posted := false
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
posted = true
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}))
|
|
defer srv.Close()
|
|
a.flushOnce(srv.URL, nil)
|
|
if posted {
|
|
t.Fatalf("flushOnce on empty aggregator must not POST")
|
|
}
|
|
}
|
|
|
|
func TestFlushOnceSwallowsPortalError(t *testing.T) {
|
|
a := newAdStats()
|
|
a.recordAdBlock("ads.example.com", "site", "")
|
|
// Unreachable portal → must not panic, must still clear the maps (snapshot
|
|
// happens before the POST).
|
|
a.flushOnce("http://127.0.0.1:1", nil)
|
|
if len(a.blocks) != 0 {
|
|
t.Fatalf("flushOnce must clear maps even on POST failure")
|
|
}
|
|
}
|
|
|
|
func TestRefererSite(t *testing.T) {
|
|
cases := []struct{ ref, want string }{
|
|
{"", ""},
|
|
{"https://news.example.com/article/123", "example.com"},
|
|
{"https://www.bbc.co.uk/news", "bbc.co.uk"},
|
|
{"http://blog.example.com:8443/x", "example.com"},
|
|
{"not a url ::::", ""},
|
|
}
|
|
for _, c := range cases {
|
|
if got := refererSite(c.ref); got != c.want {
|
|
t.Errorf("refererSite(%q) = %q, want %q", c.ref, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// itoa is a tiny stdlib-free int→string for the cap test keys (avoids importing
|
|
// strconv just for the test).
|
|
func itoa(n int) string {
|
|
if n == 0 {
|
|
return "0"
|
|
}
|
|
var b [20]byte
|
|
i := len(b)
|
|
for n > 0 {
|
|
i--
|
|
b[i] = byte('0' + n%10)
|
|
n /= 10
|
|
}
|
|
return string(b[i:])
|
|
}
|