From 8094a7507718d3069fb1fe40fb1a10f43a4ca511 Mon Sep 17 00:00:00 2001 From: CyberMind-FR Date: Mon, 22 Jun 2026 10:46:02 +0200 Subject: [PATCH] feat(dpi): fill all dashboard lists from the exfil engine (closes #695) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four list cards (Top Applications, Top Protocols, Bandwidth by Device, Active Flows) showed "No data" — they queried the inactive netifyd backend. Now driven by the real R3 DPI engine: - collector emits global rollups in /exfil: top_apps (by service/host), top_protocols (by nDPI proto), per-device up+down bytes, active_flows (top flows incl. uncategorised dests). - dashboard renders all four lists + the repointed stat cards from a single /exfil fetch; netifyd loaders dropped from the refresh loop. secubox-dpi 1.1.2. Live on gk2: admin.gk2/dpi/ lists populated (top_apps 10, top_protocols 10, active_flows 13). --- packages/secubox-dpi/collector/main.go | 84 +++++++++++++++++++++---- packages/secubox-dpi/debian/changelog | 10 +++ packages/secubox-dpi/www/dpi/index.html | 31 +++++++-- 3 files changed, 109 insertions(+), 16 deletions(-) diff --git a/packages/secubox-dpi/collector/main.go b/packages/secubox-dpi/collector/main.go index 4d6aaf1f..ec7adadf 100644 --- a/packages/secubox-dpi/collector/main.go +++ b/packages/secubox-dpi/collector/main.go @@ -378,15 +378,24 @@ func saveSeen(m map[string]bool) { func writeState(aggs map[string]*agg, alerts []alert, now int64) { // per-device rollup type devstat struct { - Device string `json:"device"` - Flows int `json:"flows"` - UpBytes int64 `json:"up_bytes"` - Services []*agg `json:"services"` // all classified egress (any category) - Clouds []*agg `json:"clouds"` // back-compat: exfil-relevant subset - ByCat map[string]int `json:"by_category"` // category → flow count - Alerts []alert `json:"alerts"` + Device string `json:"device"` + Flows int `json:"flows"` + UpBytes int64 `json:"up_bytes"` + DownBytes int64 `json:"down_bytes"` + Services []*agg `json:"services"` // all classified egress (any category) + Clouds []*agg `json:"clouds"` // back-compat: exfil-relevant subset + ByCat map[string]int `json:"by_category"` // category → flow count + Alerts []alert `json:"alerts"` } devs := map[string]*devstat{} + // global rollups (incl. uncategorised dests) for the dashboard list cards + type rollup struct { + Name string `json:"name"` + Bytes int64 `json:"bytes"` + Flows int `json:"flows"` + } + apps := map[string]*rollup{} // by service||dst + protos := map[string]*rollup{} // by nDPI proto for _, a := range aggs { d := devs[a.Device] if d == nil { @@ -395,6 +404,7 @@ func writeState(aggs map[string]*agg, alerts []alert, now int64) { } d.Flows += a.Flows d.UpBytes += a.Up + d.DownBytes += a.Down if a.Category != "" { d.Services = append(d.Services, a) d.ByCat[a.Category] += a.Flows @@ -402,6 +412,32 @@ func writeState(aggs map[string]*agg, alerts []alert, now int64) { if a.Cloud != "" { d.Clouds = append(d.Clouds, a) } + // global app rollup (service name if classified, else the dst host) + an := a.Service + if an == "" { + an = a.Dst + } + if an != "" { + r := apps[an] + if r == nil { + r = &rollup{Name: an} + apps[an] = r + } + r.Bytes += a.Up + a.Down + r.Flows += a.Flows + } + // global protocol rollup + pn := a.Proto + if pn == "" { + pn = "unknown" + } + r := protos[pn] + if r == nil { + r = &rollup{Name: pn} + protos[pn] = r + } + r.Bytes += a.Up + a.Down + r.Flows += a.Flows } for _, al := range alerts { if d := devs[al.Device]; d != nil { @@ -421,11 +457,37 @@ func writeState(aggs map[string]*agg, alerts []alert, now int64) { list = append(list, d) } sort.Slice(list, func(i, j int) bool { return list[i].UpBytes > list[j].UpBytes }) + + // rank the global rollups, cap at topN + rank := func(m map[string]*rollup) []*rollup { + out := make([]*rollup, 0, len(m)) + for _, r := range m { + out = append(out, r) + } + sort.Slice(out, func(i, j int) bool { return out[i].Bytes > out[j].Bytes }) + if len(out) > topN { + out = out[:topN] + } + return out + } + // active flows = the individual aggs (incl. uncategorised), top by total bytes + flows := make([]*agg, 0, len(aggs)) + for _, a := range aggs { + flows = append(flows, a) + } + sort.Slice(flows, func(i, j int) bool { return (flows[i].Up + flows[i].Down) > (flows[j].Up + flows[j].Down) }) + if len(flows) > 20 { + flows = flows[:20] + } + out := map[string]any{ - "generated_at": now, - "devices": list, - "alerts": alerts, - "alert_count": len(alerts), + "generated_at": now, + "devices": list, + "alerts": alerts, + "alert_count": len(alerts), + "top_apps": rank(apps), + "top_protocols": rank(protos), + "active_flows": flows, } writeJSON(statePath, out) } diff --git a/packages/secubox-dpi/debian/changelog b/packages/secubox-dpi/debian/changelog index 6e449667..6df5864b 100644 --- a/packages/secubox-dpi/debian/changelog +++ b/packages/secubox-dpi/debian/changelog @@ -1,3 +1,13 @@ +secubox-dpi (1.1.2-1~bookworm1) bookworm; urgency=low + + * #695 dashboard: fill the four list cards (Top Applications, Top Protocols, + Bandwidth by Device, Active Flows) from the real R3 exfil engine instead of + the inactive netifyd backend. Collector now emits global rollups in /exfil + (top_apps, top_protocols by nDPI proto, per-device up+down, active_flows + incl. uncategorised dests). + + -- Gerald KERMA Mon, 22 Jun 2026 10:45:00 +0000 + secubox-dpi (1.1.1-1~bookworm1) bookworm; urgency=low * #692 collector: beaconing scenario now requires a C2-plausible cadence diff --git a/packages/secubox-dpi/www/dpi/index.html b/packages/secubox-dpi/www/dpi/index.html index c5ceb5b9..d73195dd 100644 --- a/packages/secubox-dpi/www/dpi/index.html +++ b/packages/secubox-dpi/www/dpi/index.html @@ -609,6 +609,31 @@ foot.textContent = data.generated_at ? `Last capture: ${agoStr(data.generated_at)} · ${devices.length} device(s) · ${alerts.length} alert(s)` : (data.note || 'no capture window completed yet'); + + // #695 fill the list cards from the same /exfil rollups (real DPI engine) + const fillList = (id, rows, render, empty) => { + const el = document.getElementById(id); if (!el) return; + el.innerHTML = (rows && rows.length) ? rows.map(render).join('') + : `

${empty}

`; + }; + fillList('top-apps', data.top_apps, a => ` +
${a.name} + ${formatBytes(a.bytes)} · ${a.flows}f
`, + 'No data'); + fillList('top-protocols', data.top_protocols, p => ` +
${p.name} + ${formatBytes(p.bytes)} · ${p.flows}f
`, + 'No data'); + fillList('bandwidth-devices', devices, d => ` +
📟 ${shortDev(d.device)} + ↑${formatBytes(d.up_bytes)} ↓${formatBytes(d.down_bytes||0)}
`, + 'No data'); + fillList('active-flows', data.active_flows, f => { + const m = catMeta(f.category); const name = f.service || f.dst; + return `
${m.icon} ${name} + ${f.proto||''} + ↑${formatBytes(f.up_bytes)} ↓${formatBytes(f.down_bytes)}
`; + }, 'No active flows'); } async function setupMirred() { const r = await api('/setup_mirred', 'POST'); alert(r.error ? 'Error: ' + r.error : 'Mirred configured'); loadStatus(); } @@ -631,11 +656,7 @@ function refreshAll() { loadStatus(); - loadExfil(); - loadTopApps(); - loadTopProtocols(); - loadBandwidthDevices(); - loadActiveFlows(); + loadExfil(); // #695: also fills Top Apps/Protocols/Bandwidth/Active-Flows from the exfil engine loadBlockRules(); }