From 9c02d0c996170c33c33e81f8fa59ac5e343f4781 Mon Sep 17 00:00:00 2001 From: CyberMind-FR Date: Mon, 22 Jun 2026 13:19:38 +0200 Subject: [PATCH] feat(dpi): Phase 3 per-device daily history + timeline (closes #720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a time dimension to the DPI engine: - collector: updateHistory() adds each window's per-device increments to today's daily bucket in history.json (14d retention, exact β€” no double counting). JSON- backed (the static collector has no CGO SQLite driver); richer SQL can layer later. - api: GET /api/v1/dpi/history (?device=… per-device, else board-wide daily totals). - dashboard: "πŸ“ˆ Timeline β€” flux/jour (14 j)" panel (daily bars, red = alert day). secubox-dpi 1.1.4. Live on gk2: history.json accumulates today's bucket (e6b6ca13… 114 flux); /history serves the timeline. --- packages/secubox-dpi/api/main.py | 30 +++++++++++ packages/secubox-dpi/collector/main.go | 71 +++++++++++++++++++++++++ packages/secubox-dpi/debian/changelog | 8 +++ packages/secubox-dpi/www/dpi/index.html | 26 +++++++++ 4 files changed, 135 insertions(+) diff --git a/packages/secubox-dpi/api/main.py b/packages/secubox-dpi/api/main.py index 5123ae03..e4ffbb3d 100644 --- a/packages/secubox-dpi/api/main.py +++ b/packages/secubox-dpi/api/main.py @@ -81,6 +81,36 @@ async def exfil_state(): return {"generated_at": 0, "devices": [], "alerts": [], "alert_count": 0, "note": "no capture window completed yet (or wg-toolbox idle)"} + +@app.get("/history") +async def exfil_history(device: str = "", days: int = 14): + """#720 β€” per-device DAILY timeline from the collector history.json. Without + ?device, returns board-wide daily totals. Fail-empty.""" + import json as _json + from pathlib import Path as _P + p = _P("/var/lib/secubox/dpi/history.json") + try: + rows = _json.loads(p.read_text()) if p.exists() else [] + except Exception as e: # pragma: no cover + return {"device": device, "days": [], "error": str(e)} + if device: + per = [r for r in rows if r.get("device") == device] + return {"device": device, "days": per[-days:]} + # board-wide: sum per day + by_day: dict = {} + for r in rows: + d = by_day.setdefault(r.get("day"), { + "day": r.get("day"), "flows": 0, "up_bytes": 0, "down_bytes": 0, + "alerts": 0, "devices": 0}) + d["flows"] += int(r.get("flows", 0) or 0) + d["up_bytes"] += int(r.get("up_bytes", 0) or 0) + d["down_bytes"] += int(r.get("down_bytes", 0) or 0) + d["alerts"] += int(r.get("alerts", 0) or 0) + d["devices"] += 1 + days_sorted = sorted(by_day.values(), key=lambda x: x["day"] or "") + return {"device": "", "days": days_sorted[-days:]} + + app.include_router(auth_router, prefix="/auth") router = APIRouter() log = get_logger("dpi") diff --git a/packages/secubox-dpi/collector/main.go b/packages/secubox-dpi/collector/main.go index df48c68d..a74c91a1 100644 --- a/packages/secubox-dpi/collector/main.go +++ b/packages/secubox-dpi/collector/main.go @@ -52,8 +52,12 @@ var ( cumulPath = env("SECUBOX_DPI_CUMUL", "/var/lib/secubox/dpi/cumulative.json") // #718 β€” ASN DB: classify SNI-less flows (IP-only/QUIC) by destination ASN. asnDBPath = env("SECUBOX_DPI_ASN", "/var/lib/GeoIP/GeoLite2-ASN.mmdb") + // #720 β€” per-device DAILY history buckets for the timeline view. + historyPath = env("SECUBOX_DPI_HISTORY", "/var/lib/secubox/dpi/history.json") ) +const historyDays = 14 // keep two weeks of daily buckets + // #718 β€” ASN-org name fragments that mark a destination as cloud/hosting egress // (lowercase substring match). Used only when there is no SNI category, so a big // upload to an IP-only AWS/Google/etc host is still caught as cloud exfil. @@ -396,9 +400,76 @@ func main() { saveSeen(newseen) writeState(aggs, alerts, now) updateCumulative(aggs, alerts, now) + updateHistory(aggs, alerts, now) fmt.Printf("collector: %d flows-agg, %d alerts @ %d\n", len(aggs), len(alerts), now) } +// #720 β€” per-device DAILY history. Adds THIS window's increments to today's +// bucket (exact, no double counting), prunes to historyDays. JSON-backed (the +// static collector has no CGO SQLite driver); rich SQL can be layered later. +func updateHistory(aggs map[string]*agg, alerts []alert, now int64) { + type bucket struct { + Day string `json:"day"` + Device string `json:"device"` + Flows int `json:"flows"` + UpBytes int64 `json:"up_bytes"` + DownBytes int64 `json:"down_bytes"` + Alerts int `json:"alerts"` + ByCat map[string]int `json:"by_category"` + TS int64 `json:"ts"` + } + day := time.Unix(now, 0).UTC().Format("2006-01-02") + // load existing β†’ index by day|device + idx := map[string]*bucket{} + if b, err := os.ReadFile(historyPath); err == nil { + var rows []*bucket + if json.Unmarshal(b, &rows) == nil { + for _, r := range rows { + if r.ByCat == nil { + r.ByCat = map[string]int{} + } + idx[r.Day+"|"+r.Device] = r + } + } + } + // add this window's per-device increments + for _, a := range aggs { + k := day + "|" + a.Device + bk := idx[k] + if bk == nil { + bk = &bucket{Day: day, Device: a.Device, ByCat: map[string]int{}} + idx[k] = bk + } + bk.Flows += a.Flows + bk.UpBytes += a.Up + bk.DownBytes += a.Down + if a.Category != "" { + bk.ByCat[a.Category] += a.Flows + } + bk.TS = now + } + for _, al := range alerts { + if bk := idx[day+"|"+al.Device]; bk != nil { + bk.Alerts++ + } + } + // prune to historyDays, sort by day then device + cutoff := time.Unix(now, 0).UTC().AddDate(0, 0, -historyDays).Format("2006-01-02") + out := make([]*bucket, 0, len(idx)) + for _, bk := range idx { + if bk.Day >= cutoff { + out = append(out, bk) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Day != out[j].Day { + return out[i].Day < out[j].Day + } + return out[i].Device < out[j].Device + }) + writeJSON(historyPath, out) +} + // cumDev is the persisted per-device cumulative rollup. Superset of the state // device schema (adds last_seen/first_seen) so the report's _dpi_stats reads it // unchanged. diff --git a/packages/secubox-dpi/debian/changelog b/packages/secubox-dpi/debian/changelog index 1bdf66c4..15371bab 100644 --- a/packages/secubox-dpi/debian/changelog +++ b/packages/secubox-dpi/debian/changelog @@ -1,3 +1,11 @@ +secubox-dpi (1.1.4-1~bookworm1) bookworm; urgency=low + + * #720 Phase 3 β€” per-device DAILY history buckets (collector history.json, 14d) + + GET /api/v1/dpi/history (per-device or board-wide timeline) + dashboard + "Timeline" panel (flux/jour, red bar = alert day). + + -- Gerald KERMA Mon, 22 Jun 2026 13:45:00 +0000 + secubox-dpi (1.1.3-1~bookworm1) bookworm; urgency=low * #718 Phase 3 β€” ASN enrichment: classify SNI-less flows (IP-only / QUIC, diff --git a/packages/secubox-dpi/www/dpi/index.html b/packages/secubox-dpi/www/dpi/index.html index d73195dd..6a1e5f33 100644 --- a/packages/secubox-dpi/www/dpi/index.html +++ b/packages/secubox-dpi/www/dpi/index.html @@ -360,6 +360,14 @@

+ +
+

πŸ“ˆ Timeline β€” flux/jour (14 j)

+
+

Loading…

+
+
+

πŸ“Š Top Applications

@@ -654,9 +662,27 @@ loadBlockRules(); } + async function loadTimeline() { + const data = await api('/history?days=14'); + const box = document.getElementById('dpi-timeline'); + const days = (data && data.days) || []; + if (!days.length) { box.innerHTML = '

Pas encore d\'historique (β‰₯1 jour de capture).

'; return; } + const max = Math.max(1, ...days.map(d => d.flows || 0)); + box.innerHTML = days.map(d => { + const h = Math.round(6 + 78 * (d.flows || 0) / max); + const dd = (d.day || '').slice(5); // MM-DD + return `
+ ${d.flows||0} + + ${dd} +
`; + }).join(''); + } + function refreshAll() { loadStatus(); loadExfil(); // #695: also fills Top Apps/Protocols/Bandwidth/Active-Flows from the exfil engine + loadTimeline(); loadBlockRules(); }