diff --git a/packages/secubox-toolbox-ng/cmd/sbxwaf/main.go b/packages/secubox-toolbox-ng/cmd/sbxwaf/main.go index c1a83e52..138a4a5e 100644 --- a/packages/secubox-toolbox-ng/cmd/sbxwaf/main.go +++ b/packages/secubox-toolbox-ng/cmd/sbxwaf/main.go @@ -195,6 +195,14 @@ type Server struct { // feature entirely: an unmapped host always gets 421, exactly as before. onDemand *OnDemand + // vhostSignals is the #896 Task 15 per-vhost last-request/active-conns + // emitter the profiles-side sleeper reads to decide idleness. Nil + // disables it entirely (--vhost-signals=""). Only ever Begin/End'd for + // on-demand vhosts (see the handler() call site) — recording every + // public vhost is unnecessary (the sleeper only cares about the + // on-demand set) and would defeat the "map stays tiny" property. + vhostSignals *VhostSignals + // wakerSocket overrides the unix socket path dialled by wakerProxy(). // Empty means the production default (see wakerproxy.go's // defaultWakerSocket). Tests inject a stub socket path here. @@ -308,6 +316,22 @@ func (s *Server) handler() http.Handler { return } + // #896 Task 15 — bracket this request in the per-vhost signal emitter. + // Reached ONLY when routeLookup found a live route (ok == true) and we + // did NOT take the waker branch above (that branch already returned) — + // so this is exactly "a real request is about to be proxied to a real + // backend". Gated to on-demand vhosts: those are the only ones the + // sleeper ever acts on, so recording anything else would just grow the + // map for no benefit. A single placement here (rather than one at each + // of the two proxy.ServeHTTP call sites below — the media-cache-miss + // path and the plain path) covers both: `defer` fires whichever one the + // function returns through, so one Begin always pairs with exactly one + // End regardless of which branch handles the request. + if s.vhostSignals != nil && s.onDemand != nil && s.onDemand.Contains(host) { + s.vhostSignals.Begin(host) + defer s.vhostSignals.End(host) + } + // Use the cached proxy from Routes when available (Task 1.2 perf goal: // no per-request *httputil.ReverseProxy allocation). var proxy *httputil.ReverseProxy @@ -693,6 +717,11 @@ func main() { // (the sleeper stopped them) are proxied to the waker splash instead of 421. onDemandVhosts := flag.String("on-demand-vhosts", "", "path to on-demand-vhosts.json (JSON array of vhosts; hot-reloaded); empty disables the waker branch") + // #896 Task 15: per-vhost last-request/active-conns signal the + // profiles-side sleeper (api/sleeper.py) reads to decide idleness. + // Only ever populated for on-demand vhosts (see handler()); empty disables. + vhostSignalsFile := flag.String("vhost-signals", "/var/cache/secubox/waf/vhost-signals.json", + "path for the per-vhost last-request/active-conns JSON snapshot (scale-to-zero signal source); empty disables") upstreamTimeout := flag.Duration("upstream-timeout", 10*time.Second, "per-request upstream timeout") threatLog := flag.String("threat-log", "/var/log/secubox/waf/waf-threats.log", "path for append-only WAF threat log (NDJSON, one record per hit)") @@ -779,6 +808,14 @@ func main() { log.Printf("sbxwaf: visit-stats enabled → %s (flush %s)", *visitsStats, visitFlushInterval) } + // #896 Task 15: per-vhost scale-to-zero signal emitter. Disabled when + // --vhost-signals is empty. + var vhostSignals *VhostSignals + if *vhostSignalsFile != "" { + vhostSignals = NewVhostSignals(*vhostSignalsFile) + log.Printf("sbxwaf: vhost-signals enabled → %s (flush %s)", *vhostSignalsFile, vhostSignalsFlushInterval) + } + srv := &Server{ upstreamTimeout: *upstreamTimeout, transport: sharedTransport, @@ -796,6 +833,8 @@ func main() { mediaCache: mediaCache, // #747: non-attacker visit statistics. visits: visits, + // #896 Task 15: per-vhost scale-to-zero signal emitter. + vhostSignals: vhostSignals, // #747: first-party host suffixes + Hub origin for the injected health banner. widgetHosts: splitCSV(*widgetHosts), bannerOrigin: strings.TrimSpace(*bannerOrigin), diff --git a/packages/secubox-toolbox-ng/cmd/sbxwaf/main_test.go b/packages/secubox-toolbox-ng/cmd/sbxwaf/main_test.go index d8754b75..64faea56 100644 --- a/packages/secubox-toolbox-ng/cmd/sbxwaf/main_test.go +++ b/packages/secubox-toolbox-ng/cmd/sbxwaf/main_test.go @@ -223,6 +223,97 @@ func TestOnDemandVisitsExcluded(t *testing.T) { } } +// TestVhostSignalsRecordedForRealOnDemandRequest verifies the handler() +// Begin/End hook (#896 Task 15): a request to an on-demand vhost that DOES +// have a live route is bracketed — active_conns is back to 0 (End ran) and +// last_request_ts is set once the request completes. +func TestVhostSignalsRecordedForRealOnDemandRequest(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer backend.Close() + backendAddr := strings.TrimPrefix(backend.URL, "http://") + + vs := NewVhostSignals("") // no on-disk flush, inspect state directly + srv := &Server{ + routeLookup: func(host string) (string, int, bool) { + h, p, err := splitHostPort(backendAddr) + if err != nil { + return "", 0, false + } + return h, p, true + }, + onDemand: &OnDemand{entries: map[string]bool{"sleepy.example.com": true}}, + vhostSignals: vs, + } + + handler := srv.handler() + req := httptest.NewRequest(http.MethodGet, "http://sleepy.example.com/", nil) + req.Host = "sleepy.example.com" + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected 200 from the real backend, got %d", rec.Code) + } + + snap := vs.snapshot() + entry, ok := snap["sleepy.example.com"] + if !ok { + t.Fatal("expected sleepy.example.com to be recorded in vhost signals") + } + if entry.ActiveConns != 0 { + t.Fatalf("expected active_conns=0 once the (synchronous) request completed, got %d", entry.ActiveConns) + } + if entry.LastRequestTS == 0 { + t.Fatal("expected a non-zero last_request_ts") + } +} + +// TestVhostSignalsExcludedForWakerBranch verifies that a request served by +// the waker splash (the vhost is asleep, no live route) is NOT recorded into +// vhost signals — recording it would make a sleeping vhost look like it just +// received a real hit, defeating the idle check (mirrors the analogous +// TestOnDemandVisitsExcluded for the #747 visit-stats aggregator). +func TestVhostSignalsExcludedForWakerBranch(t *testing.T) { + sockPath := filepath.Join(t.TempDir(), "waker-vhostsignals.sock") + ln, err := net.Listen("unix", sockPath) + if err != nil { + t.Fatalf("listen unix %s: %v", sockPath, err) + } + defer ln.Close() + + waker := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = io.WriteString(w, "waking up…") + }), + } + go waker.Serve(ln) //nolint:errcheck + defer waker.Close() + + vs := NewVhostSignals("") + srv := &Server{ + routeLookup: func(host string) (string, int, bool) { return "", 0, false }, + onDemand: &OnDemand{entries: map[string]bool{"sleepy.example.com": true}}, + wakerSocket: sockPath, + vhostSignals: vs, + } + + handler := srv.handler() + req := httptest.NewRequest(http.MethodGet, "http://sleepy.example.com/", nil) + req.Host = "sleepy.example.com" + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("expected 503 from waker splash, got %d", rec.Code) + } + if snap := vs.snapshot(); len(snap) != 0 { + t.Fatalf("waker splash must not be recorded in vhost signals; got %+v", snap) + } +} + // TestTrustedHostSkipsWAF verifies that a request to a trusted host is NOT // blocked even when the payload would normally trigger the WAF. // Mirrors Python check_request whitelist (secubox_waf.py:761-763). diff --git a/packages/secubox-toolbox-ng/cmd/sbxwaf/vhostsignals.go b/packages/secubox-toolbox-ng/cmd/sbxwaf/vhostsignals.go new file mode 100644 index 00000000..146c2ab7 --- /dev/null +++ b/packages/secubox-toolbox-ng/cmd/sbxwaf/vhostsignals.go @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: LicenseRef-CMSD-1.0 +// Copyright (c) 2026 CyberMind — Gérald Kerma +// Source-Disclosed License — All rights reserved except as expressly granted. +// See LICENCE-CMSD-1.0.md for terms. + +// SecuBox-Deb :: toolbox-ng :: sbxwaf — per-vhost signal emitter (#896 Task 15) +// +// SecuBox scale-to-zero: the profiles-side sleeper (secubox-profiles, +// api/sleeper.py) decides whether an on-demand public vhost is idle from two +// per-vhost numbers — "how long since its last request" and "how many +// requests are in flight right now". Nothing in sbxwaf tracked either: the +// #747 visit-stats aggregator (visitstats.go) only keeps cumulative COUNTS, +// no timestamps, no in-flight tracking. This file adds that missing signal. +// +// Shape mirrors VisitStats deliberately (lock-guarded maps + a background +// flusher doing an atomic temp-write + rename — see writeSnapshot in +// visitstats.go): the hot path only ever touches an in-memory counter, disk +// I/O happens on a timer, never on the request path. +// +// Two differences from VisitStats, both intentional: +// - The flush interval is 5s, not 30s. The sleeper (api/sleeper_daemon.py) +// polls every ~30s by default; a snapshot that is stale by up to 30s +// itself would double the effective latency of the idle decision. 5s +// keeps that skew small relative to the sleeper's own poll interval. +// - No top-N cap on the vhost map. VisitStats caps at visitMapCap because +// it tracks EVERY vhost the WAF ever sees (thousands of public sites); +// dropping the least-busy is fine because the panel only shows a +// top-N chart. Here the sleeper needs EVERY on-demand vhost's signal — +// dropping one under memory pressure would silently disable auto-sleep +// for that vhost. A cap is unnecessary anyway: recording is gated to +// on-demand vhosts only (see the Begin/End call site in main.go), and +// that set is operator-curated and expected to stay small (it is the +// handful of rarely-used services worth scaling to zero, not the whole +// public vhost fleet). +package main + +import ( + "encoding/json" + "os" + "sync" + "time" +) + +// vhostSignalsFlushInterval is how often the in-memory Begin/End state is +// snapshotted to disk. Shorter than visitFlushInterval (30s) — see the +// package doc comment above for why. +const vhostSignalsFlushInterval = 5 * time.Second + +// VhostSignals tracks, per on-demand vhost, the wall-clock time of its most +// recent request (lastSeen) and how many requests are currently in flight +// (active). All maps are guarded by mu; Begin/End are the only hot-path +// operations and take the lock only for an O(1) map update. +type VhostSignals struct { + mu sync.Mutex + lastSeen map[string]int64 // vhost -> unix seconds of the most recent Begin + active map[string]int64 // vhost -> count of requests currently in flight + path string +} + +// NewVhostSignals builds the aggregator and starts the background flusher +// writing to path every vhostSignalsFlushInterval. path == "" disables +// persistence (the aggregator still counts, useful for tests). +func NewVhostSignals(path string) *VhostSignals { + v := &VhostSignals{ + lastSeen: map[string]int64{}, + active: map[string]int64{}, + path: path, + } + if path != "" { + go v.runFlusher() + } + return v +} + +// Begin records the start of a request to vhost: bumps lastSeen to now and +// increments the in-flight counter. Pair with a deferred End(vhost) at the +// call site so a panic mid-request still decrements (see main.go handler()). +func (v *VhostSignals) Begin(vhost string) { + if v == nil { + return + } + v.mu.Lock() + defer v.mu.Unlock() + v.lastSeen[vhost] = time.Now().Unix() + v.active[vhost]++ +} + +// End records the end of a request to vhost: decrements the in-flight +// counter, pruning the map entry once it reaches zero (so an idle vhost's +// active-conns snapshot is simply "key absent" == 0, never a stale zero +// left lying around). lastSeen is NEVER touched here — the idle-age math on +// the profiles side needs the timestamp of the LAST request to survive long +// after active_conns has dropped back to zero. +func (v *VhostSignals) End(vhost string) { + if v == nil { + return + } + v.mu.Lock() + defer v.mu.Unlock() + v.active[vhost]-- + if v.active[vhost] <= 0 { + delete(v.active, vhost) + } +} + +// vhostSignalEntry is the per-vhost on-disk JSON shape the profiles-side +// sleeper reads (api/front_signals.py::vhost_signals). last_request_ts is a +// UNIX WALL-CLOCK timestamp (time.Now().Unix()) — the Python reader MUST +// compute its age against a wall-clock `now` (time.time), not a monotonic +// one, or the age math is meaningless. +type vhostSignalEntry struct { + LastRequestTS int64 `json:"last_request_ts"` + ActiveConns int64 `json:"active_conns"` +} + +// snapshot copies the counters under the lock into the on-disk shape. +func (v *VhostSignals) snapshot() map[string]vhostSignalEntry { + v.mu.Lock() + defer v.mu.Unlock() + out := make(map[string]vhostSignalEntry, len(v.lastSeen)) + for vhost, ts := range v.lastSeen { + out[vhost] = vhostSignalEntry{ + LastRequestTS: ts, + ActiveConns: v.active[vhost], // 0 (map default) when no key — pruned by End + } + } + return out +} + +// writeSnapshot atomically writes the snapshot JSON to path (temp + rename) +// so the reader never sees a half-written file. Best-effort: errors are +// swallowed, mirroring VisitStats.writeSnapshot. +func (v *VhostSignals) writeSnapshot() { + if v.path == "" { + return + } + snap := v.snapshot() + buf, err := json.Marshal(snap) + if err != nil { + return + } + tmp := v.path + ".tmp" + if err := os.WriteFile(tmp, buf, 0o640); err != nil { + return + } + _ = os.Rename(tmp, v.path) +} + +// runFlusher overwrites the signals file every vhostSignalsFlushInterval for +// the process lifetime. Started once from NewVhostSignals. +func (v *VhostSignals) runFlusher() { + t := time.NewTicker(vhostSignalsFlushInterval) + defer t.Stop() + for range t.C { + v.writeSnapshot() + } +} diff --git a/packages/secubox-toolbox-ng/cmd/sbxwaf/vhostsignals_test.go b/packages/secubox-toolbox-ng/cmd/sbxwaf/vhostsignals_test.go new file mode 100644 index 00000000..822582c7 --- /dev/null +++ b/packages/secubox-toolbox-ng/cmd/sbxwaf/vhostsignals_test.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: LicenseRef-CMSD-1.0 +// Copyright (c) 2026 CyberMind — Gérald Kerma +// +// SecuBox-Deb :: toolbox-ng :: sbxwaf — per-vhost signal emitter tests (#896) +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// TestVhostSignalsBeginEnd verifies the core Begin/End contract: Begin sets a +// non-zero last-seen timestamp and bumps the in-flight counter; End +// decrements it and prunes the map entry once it reaches zero, while +// preserving lastSeen (the idle-age math on the profiles side needs the +// timestamp of the last request to survive long after active_conns drops +// back to zero). +func TestVhostSignalsBeginEnd(t *testing.T) { + v := NewVhostSignals("") // no on-disk flush needed for this test + + v.Begin("sleepy.example.com") + v.mu.Lock() + ts := v.lastSeen["sleepy.example.com"] + active := v.active["sleepy.example.com"] + v.mu.Unlock() + if ts == 0 { + t.Fatal("expected Begin to set a non-zero last-seen timestamp") + } + if active != 1 { + t.Fatalf("expected active=1 after one Begin, got %d", active) + } + + // A second concurrent request to the same vhost. + v.Begin("sleepy.example.com") + v.mu.Lock() + active = v.active["sleepy.example.com"] + v.mu.Unlock() + if active != 2 { + t.Fatalf("expected active=2 after two Begins, got %d", active) + } + + v.End("sleepy.example.com") + v.mu.Lock() + active = v.active["sleepy.example.com"] + v.mu.Unlock() + if active != 1 { + t.Fatalf("expected active=1 after one End, got %d", active) + } + + v.End("sleepy.example.com") + v.mu.Lock() + _, stillPresent := v.active["sleepy.example.com"] + lastSeenAfter := v.lastSeen["sleepy.example.com"] + v.mu.Unlock() + if stillPresent { + t.Fatal("expected the active-conns entry to be pruned once the count reaches 0") + } + if lastSeenAfter != ts { + t.Fatal("expected lastSeen to be preserved after active count drops to 0 (idle-age math needs it)") + } +} + +// TestVhostSignalsEndWithoutBeginNeverGoesPositive guards against a +// mismatched End (defer fired without a matching Begin, or double-fired) +// making active_conns look positive forever — that would permanently block +// auto-sleep for the vhost. +func TestVhostSignalsEndWithoutBeginNeverGoesPositive(t *testing.T) { + v := NewVhostSignals("") + v.End("never-begun.example.com") + v.mu.Lock() + _, present := v.active["never-begun.example.com"] + v.mu.Unlock() + if present { + t.Fatal("an End with no matching Begin must not leave a positive/zero active entry behind") + } +} + +// TestVhostSignalsFlushWritesJSON verifies the on-disk snapshot shape the +// profiles-side reader (api/sleeper_daemon.py::_signal_reader) consumes: +// {"": {"last_request_ts": , "active_conns": }}. +func TestVhostSignalsFlushWritesJSON(t *testing.T) { + path := filepath.Join(t.TempDir(), "vhost-signals.json") + v := NewVhostSignals(path) + + v.Begin("sleepy.example.com") + v.writeSnapshot() + + buf, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read snapshot: %v", err) + } + var got map[string]vhostSignalEntry + if err := json.Unmarshal(buf, &got); err != nil { + t.Fatalf("unmarshal snapshot: %v", err) + } + entry, ok := got["sleepy.example.com"] + if !ok { + t.Fatal("expected sleepy.example.com in the snapshot") + } + if entry.ActiveConns != 1 { + t.Fatalf("expected active_conns=1, got %d", entry.ActiveConns) + } + if entry.LastRequestTS == 0 { + t.Fatal("expected a non-zero last_request_ts") + } +} + +// TestVhostSignalsFlushReflectsEndedRequest verifies that once a request +// completes (End called), the NEXT flush reports active_conns=0 while still +// keeping the vhost's last_request_ts — exactly what should_sleep() needs to +// decide "idle" (active_conns == 0 AND last_request_age >= threshold). +func TestVhostSignalsFlushReflectsEndedRequest(t *testing.T) { + path := filepath.Join(t.TempDir(), "vhost-signals.json") + v := NewVhostSignals(path) + + v.Begin("sleepy.example.com") + v.End("sleepy.example.com") + v.writeSnapshot() + + buf, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read snapshot: %v", err) + } + var got map[string]vhostSignalEntry + if err := json.Unmarshal(buf, &got); err != nil { + t.Fatalf("unmarshal snapshot: %v", err) + } + entry, ok := got["sleepy.example.com"] + if !ok { + t.Fatal("expected sleepy.example.com to remain in the snapshot after End") + } + if entry.ActiveConns != 0 { + t.Fatalf("expected active_conns=0 after End, got %d", entry.ActiveConns) + } + if entry.LastRequestTS == 0 { + t.Fatal("expected last_request_ts to survive after active_conns drops to 0") + } +} + +// TestVhostSignalsEmptyPathDisablesFlush verifies path=="" (the convention +// used elsewhere — NewVisitStats("")) never attempts a disk write. +func TestVhostSignalsEmptyPathDisablesFlush(t *testing.T) { + v := NewVhostSignals("") + v.Begin("sleepy.example.com") + v.writeSnapshot() // must be a safe no-op, not a panic on an empty path +} diff --git a/packages/secubox-waf-ng/systemd/secubox-waf-ng-worker@.service b/packages/secubox-waf-ng/systemd/secubox-waf-ng-worker@.service index 37cac0fb..06c9613f 100644 --- a/packages/secubox-waf-ng/systemd/secubox-waf-ng-worker@.service +++ b/packages/secubox-waf-ng/systemd/secubox-waf-ng-worker@.service @@ -36,6 +36,7 @@ ExecStart=/usr/sbin/sbxwaf \ --routes /etc/secubox/waf/haproxy-routes.json \ --rules /etc/secubox/waf/waf-rules.json \ --on-demand-vhosts /etc/secubox/waf/on-demand-vhosts.json \ + --vhost-signals /var/cache/secubox/waf/vhost-signals.json \ --threat-log /var/log/secubox/waf/waf-threats.log \ --cookie-audit-log /var/log/secubox/cookie-audit/server.jsonl \ --media-cache-dir /var/cache/secubox/waf/media \