From fdfc404818b117733fa32926e31f0c77f269741d Mon Sep 17 00:00:00 2001 From: CyberMind-FR Date: Wed, 24 Jun 2026 13:59:59 +0200 Subject: [PATCH] =?UTF-8?q?feat(threatmesh):=20sovereign=20threat-intel=20?= =?UTF-8?q?=E2=80=94=20feeds=20+=20mesh=20+=20API,=20drop=20CrowdSec=20CAP?= =?UTF-8?q?I=20(#728)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 0 (board): CrowdSec online_client/CAPI disabled, LAPI kept. Phase 1: secubox-threatfeed timer pulls 8 free public blocklists (feodo/sslbl/firehol/spamhaus-drop/blocklist.de/cins/et/dshield) -> shared threat_intel -> secubox-blacklist-sync -> nft. ~45k IOCs live. Phase 2: secubox-threatmesh service gossips locally-detected CrowdSec decisions to SecuBox P2P peers over WireGuard + ingests peer decisions (mesh:, consensus-counted); :8780 locked to wg*/lo by an nft drop-in. Phase 3: /status /peers /decisions (bouncer-compatible aggregate) /mesh/ingest + C3BOX dashboard (sovereign mode). No CAPI, no account, no paywall. Co-Authored-By: Claude Opus 4.8 --- packages/secubox-threatmesh/api/__init__.py | 0 packages/secubox-threatmesh/api/main.py | 267 ++++++++++++++++++ packages/secubox-threatmesh/debian/changelog | 13 + packages/secubox-threatmesh/debian/control | 18 ++ packages/secubox-threatmesh/debian/postinst | 23 ++ packages/secubox-threatmesh/debian/prerm | 11 + packages/secubox-threatmesh/debian/rules | 23 ++ .../secubox-threatmesh/debian/source/format | 1 + .../menu.d/37-threatmesh.json | 9 + .../nftables.d/secubox-threatmesh.nft | 10 + .../secubox-threatmesh/nginx/threatmesh.conf | 10 + .../sbin/secubox-threatfeed | 121 ++++++++ .../systemd/secubox-threatfeed.service | 12 + .../systemd/secubox-threatfeed.timer | 11 + .../systemd/secubox-threatmesh.service | 22 ++ .../www/threatmesh/index.html | 87 ++++++ 16 files changed, 638 insertions(+) create mode 100644 packages/secubox-threatmesh/api/__init__.py create mode 100644 packages/secubox-threatmesh/api/main.py create mode 100644 packages/secubox-threatmesh/debian/changelog create mode 100644 packages/secubox-threatmesh/debian/control create mode 100644 packages/secubox-threatmesh/debian/postinst create mode 100644 packages/secubox-threatmesh/debian/prerm create mode 100755 packages/secubox-threatmesh/debian/rules create mode 100644 packages/secubox-threatmesh/debian/source/format create mode 100644 packages/secubox-threatmesh/menu.d/37-threatmesh.json create mode 100644 packages/secubox-threatmesh/nftables.d/secubox-threatmesh.nft create mode 100644 packages/secubox-threatmesh/nginx/threatmesh.conf create mode 100644 packages/secubox-threatmesh/sbin/secubox-threatfeed create mode 100644 packages/secubox-threatmesh/systemd/secubox-threatfeed.service create mode 100644 packages/secubox-threatmesh/systemd/secubox-threatfeed.timer create mode 100644 packages/secubox-threatmesh/systemd/secubox-threatmesh.service create mode 100644 packages/secubox-threatmesh/www/threatmesh/index.html diff --git a/packages/secubox-threatmesh/api/__init__.py b/packages/secubox-threatmesh/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/packages/secubox-threatmesh/api/main.py b/packages/secubox-threatmesh/api/main.py new file mode 100644 index 00000000..c684dbb9 --- /dev/null +++ b/packages/secubox-threatmesh/api/main.py @@ -0,0 +1,267 @@ +# SPDX-License-Identifier: LicenseRef-CMSD-1.0 +# Copyright (c) 2026 CyberMind — Gérald Kerma +# Source-Disclosed License — All rights reserved except as expressly granted. +# See LICENCE-CMSD-1.0.md for terms. +""" +SecuBox-Deb :: secubox-threatmesh (#728) +CyberMind — https://cybermind.fr + +Sovereign threat-intel control plane that replaces CrowdSec CAPI: + + Phase 1 free public feeds (secubox-threatfeed timer) -> threat_intel + Phase 2 MESH distribution of LOCAL decisions over the SecuBox P2P mesh + (gossip to peers from secubox-p2p; ingest peer decisions) -> threat_intel + Phase 3 status / aggregate / CrowdSec-bouncer-compatible decisions API + UI + +All IOCs land in the shared `threat_intel` table (toolbox.db); secubox-blacklist-sync +drains it to the nft blacklist. No central account, no CAPI, no paywall. +""" +import asyncio +import json +import re +import sqlite3 +import subprocess +import time +import urllib.request +from pathlib import Path +from typing import Optional + +from fastapi import FastAPI, APIRouter, Depends, Request +from fastapi.responses import JSONResponse + +from secubox_core.auth import require_jwt +from secubox_core.logger import get_logger + +log = get_logger("threatmesh") + +TI_DB = Path("/var/lib/secubox/toolbox/toolbox.db") +P2P_PEERS = Path("/var/lib/secubox/p2p/peers.json") +P2P_NODE = Path("/var/lib/secubox/p2p/node_id") +MESH_INTERVAL = 600 # gossip every 10 min +LOCAL_ORIGINS = {"crowdsec", "cscli", "manual", "secubox-waf", "secubox"} +IPV4 = re.compile(r"^\d{1,3}(\.\d{1,3}){3}(/\d{1,2})?$") + +app = FastAPI(title="secubox-threatmesh", version="1.0.0", root_path="/api/v1/threatmesh") +router = APIRouter() +_node_id = None + + +# ── store ─────────────────────────────────────────────────────────── +def _conn(): + c = sqlite3.connect(TI_DB, timeout=20) + c.row_factory = sqlite3.Row + return c + + +def _ensure(c): + c.execute("""CREATE TABLE IF NOT EXISTS threat_intel ( + ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL, + weight INTEGER NOT NULL DEFAULT 50, label TEXT, + first_seen INTEGER, last_seen INTEGER, + PRIMARY KEY (ioc, ioc_type, source))""") + + +def node_id() -> str: + global _node_id + if _node_id: + return _node_id + try: + _node_id = P2P_NODE.read_text().strip() + except Exception: + try: + _node_id = json.loads(P2P_PEERS.read_text())["peers"][0]["id"] + except Exception: + _node_id = "sb-local" + return _node_id + + +def upsert_iocs(rows, source, weight=60, label=None): + """rows: iterable of (ioc, ioc_type). Returns count.""" + now = int(time.time()) + n = 0 + with _conn() as c: + _ensure(c) + for ioc, t in rows: + c.execute( + "INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) " + "VALUES(?,?,?,?,?,?,?) ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen", + (ioc, t, source, weight, label, now, now)) + n += 1 + c.commit() + return n + + +def counts_by_source(): + with _conn() as c: + _ensure(c) + return {r["source"]: r["n"] for r in c.execute( + "SELECT source, COUNT(*) n FROM threat_intel GROUP BY source ORDER BY n DESC")} + + +def aggregate_ips(limit=50000): + """Distinct bad IPs across ALL sources, with a consensus count.""" + with _conn() as c: + _ensure(c) + return [dict(r) for r in c.execute( + "SELECT ioc, COUNT(DISTINCT source) consensus, MAX(weight) weight, " + "GROUP_CONCAT(DISTINCT source) sources FROM threat_intel " + "WHERE ioc_type='ip' GROUP BY ioc ORDER BY consensus DESC LIMIT ?", (limit,))] + + +# ── mesh (Phase 2) ────────────────────────────────────────────────── +def mesh_peers(): + """Other SecuBox nodes from secubox-p2p (skip self).""" + me = node_id() + out = [] + try: + d = json.loads(P2P_PEERS.read_text()) + for p in d.get("peers", []): + if p.get("id") == me: + continue + addr = None + for a in p.get("addresses", []): + if a.get("type") in ("wg", "wireguard", "mesh"): + addr = a.get("address"); break + addr = addr or (p.get("wg_addresses") or [None])[0] or p.get("address") + if addr: + out.append({"id": p.get("id"), "name": p.get("name"), "address": addr}) + except Exception as e: + log.debug(f"peers read: {e}") + return out + + +def local_decisions(): + """Our OWN locally-detected bad IPs (CrowdSec LAPI bans of local origin).""" + try: + out = subprocess.run(["cscli", "decisions", "list", "-o", "json", "-a"], + capture_output=True, text=True, timeout=20) + data = json.loads(out.stdout or "[]") or [] + except Exception as e: + log.debug(f"cscli decisions: {e}") + return [] + ips = [] + for alert in data: + for dec in (alert.get("decisions") or []): + if dec.get("type") == "ban" and dec.get("scope", "").lower() == "ip": + origin = (dec.get("origin") or "").lower() + val = dec.get("value", "") + if IPV4.match(val) and not origin.startswith(("feed:", "mesh:", "lists:")): + ips.append(val) + return sorted(set(ips)) + + +async def _post_json(url, payload, timeout=15): + def _do(): + req = urllib.request.Request( + url, data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", "User-Agent": "SecuBox-ThreatMesh/1.0"}, + method="POST") + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status + return await asyncio.to_thread(_do) + + +async def mesh_gossip_once(): + peers = mesh_peers() + decisions = local_decisions() + if not peers or not decisions: + return {"peers": len(peers), "shared": len(decisions), "pushed": 0} + payload = {"node": node_id(), "ts": int(time.time()), "ips": decisions} + pushed = 0 + for p in peers: + url = f"http://{p['address']}:8780/api/v1/threatmesh/mesh/ingest" + try: + await _post_json(url, payload) + pushed += 1 + except Exception as e: + log.debug(f"gossip to {p['id']} failed: {e}") + return {"peers": len(peers), "shared": len(decisions), "pushed": pushed} + + +async def _mesh_loop(): + while True: + try: + r = await mesh_gossip_once() + log.info(f"mesh gossip: {r}") + except Exception as e: + log.error(f"mesh loop: {e}") + await asyncio.sleep(MESH_INTERVAL) + + +@app.on_event("startup") +async def _startup(): + asyncio.create_task(_mesh_loop()) + + +# ── endpoints ─────────────────────────────────────────────────────── +@app.get("/health") +async def health(): + return {"status": "ok", "module": "deb"} + + +@router.get("/status") +async def status(): + src = counts_by_source() + feeds = {k: v for k, v in src.items() if k.startswith("feed:")} + mesh = {k: v for k, v in src.items() if k.startswith("mesh:")} + return { + "mode": "sovereign", # CAPI dropped (#728) + "node": node_id(), + "capi": False, + "feeds": feeds, "feed_total": sum(feeds.values()), + "mesh": mesh, "mesh_total": sum(mesh.values()), + "peers": len(mesh_peers()), + "sources": src, + } + + +@router.get("/peers") +async def peers(): + return {"node": node_id(), "peers": mesh_peers()} + + +@router.get("/decisions") +async def decisions(min_consensus: int = 1, limit: int = 50000): + """Aggregated sovereign blocklist (feeds + mesh + local). CrowdSec-bouncer + friendly shape so external consumers can poll OUR server, not crowdsec.net.""" + rows = [r for r in aggregate_ips(limit) if r["consensus"] >= min_consensus] + return [{ + "id": i, "origin": "secubox-threatmesh", "type": "ban", "scope": "Ip", + "value": r["ioc"], "duration": "24h", + "consensus": r["consensus"], "sources": r["sources"], + } for i, r in enumerate(rows)] + + +@router.post("/mesh/ingest") +async def mesh_ingest(request: Request): + """Receive a peer's locally-detected decisions over the mesh -> threat_intel + tagged mesh:. Trust boundary = the WireGuard mesh transport.""" + try: + body = await request.json() + except Exception: + return JSONResponse({"ok": False, "error": "bad json"}, status_code=400) + node = str(body.get("node", "")).strip()[:64] or "unknown" + if not re.match(r"^[A-Za-z0-9._:-]+$", node): + return JSONResponse({"ok": False, "error": "bad node"}, status_code=400) + ips = [ip for ip in (body.get("ips") or []) if isinstance(ip, str) and IPV4.match(ip)] + n = upsert_iocs(((ip, "ip") for ip in ips[:20000]), f"mesh:{node}", weight=70, label="mesh") + log.info(f"mesh ingest from {node}: {n} ips") + return {"ok": True, "ingested": n} + + +@router.post("/feeds/refresh", dependencies=[Depends(require_jwt)]) +async def feeds_refresh(): + """Trigger the feed fetcher now (normally on a timer).""" + try: + subprocess.Popen(["/usr/sbin/secubox-threatfeed"]) + return {"ok": True, "started": True} + except Exception as e: + return JSONResponse({"ok": False, "error": str(e)}, status_code=500) + + +@router.post("/mesh/gossip", dependencies=[Depends(require_jwt)]) +async def mesh_gossip_now(): + return await mesh_gossip_once() + + +app.include_router(router) diff --git a/packages/secubox-threatmesh/debian/changelog b/packages/secubox-threatmesh/debian/changelog new file mode 100644 index 00000000..1cb9b3dc --- /dev/null +++ b/packages/secubox-threatmesh/debian/changelog @@ -0,0 +1,13 @@ +secubox-threatmesh (1.0.0-1~bookworm1) bookworm; urgency=medium + + * Initial release (#728) — sovereign threat-intel, CrowdSec CAPI replacement. + - Phase 1: secubox-threatfeed timer pulls free public blocklists + (feodo/sslbl/firehol/spamhaus-drop/blocklist.de/cins/et/dshield) into the + shared threat_intel table -> secubox-blacklist-sync -> nft. + - Phase 2: MESH distribution — gossip locally-detected CrowdSec decisions to + SecuBox P2P peers over WireGuard; ingest peer decisions (tagged mesh:), + consensus-counted. :8780 locked to wg*/loopback by an nft drop-in. + - Phase 3: /status, /peers, /decisions (bouncer-compatible aggregate), + /mesh/ingest, dashboard. CrowdSec LAPI kept; CAPI dropped. + + -- Gerald KERMA Wed, 24 Jun 2026 16:00:00 +0000 diff --git a/packages/secubox-threatmesh/debian/control b/packages/secubox-threatmesh/debian/control new file mode 100644 index 00000000..35dc9c4b --- /dev/null +++ b/packages/secubox-threatmesh/debian/control @@ -0,0 +1,18 @@ +Source: secubox-threatmesh +Section: net +Priority: optional +Maintainer: Gerald KERMA +Build-Depends: debhelper-compat (= 13) +Standards-Version: 4.6.2 + +Package: secubox-threatmesh +Architecture: all +Depends: ${misc:Depends}, secubox-core (>= 1.0), python3, python3-uvicorn | python3-pip, secubox-vortex-firewall | secubox-toolbox +Recommends: secubox-p2p, crowdsec +Description: Sovereign threat-intel mesh for SecuBox (CrowdSec CAPI replacement) + Replaces the CrowdSec Central API with self-sourced free public blocklists + (Phase 1) and peer-to-peer MESH distribution of locally-detected decisions over + the SecuBox WireGuard mesh (Phase 2), plus a status/aggregate/bouncer-compatible + API and dashboard (Phase 3). All IOCs flow into the shared threat_intel table + that secubox-blacklist-sync drains into the nft blacklist. No central account, + no enrollment, no paywall, no IP-blocklisting by a third party. diff --git a/packages/secubox-threatmesh/debian/postinst b/packages/secubox-threatmesh/debian/postinst new file mode 100644 index 00000000..9f9e8a83 --- /dev/null +++ b/packages/secubox-threatmesh/debian/postinst @@ -0,0 +1,23 @@ +#!/bin/bash +set -e +case "$1" in + configure) + install -d -m 0755 /var/lib/secubox/threatmesh + systemctl daemon-reload 2>/dev/null || true + # mesh-port firewall (lock :8780 to wg*/lo under DEFAULT DROP) + if [ -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft ]; then + nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft 2>/dev/null || true + fi + if [ "$(systemctl is-enabled secubox-threatmesh.service 2>/dev/null)" != "masked" ]; then + systemctl enable --now secubox-threatmesh.service 2>/dev/null || true + fi + systemctl enable --now secubox-threatfeed.timer 2>/dev/null || true + # first feed population (background; logs to /tmp to avoid /var/log perms) + /usr/sbin/secubox-threatfeed >/tmp/secubox-threatfeed-firstrun.log 2>&1 & + if command -v nginx >/dev/null 2>&1 && nginx -t 2>/dev/null; then + systemctl reload nginx 2>/dev/null || true + fi + ;; +esac +#DEBHELPER# +exit 0 diff --git a/packages/secubox-threatmesh/debian/prerm b/packages/secubox-threatmesh/debian/prerm new file mode 100644 index 00000000..48cfa455 --- /dev/null +++ b/packages/secubox-threatmesh/debian/prerm @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +case "$1" in + remove|deconfigure) + systemctl stop secubox-threatmesh.service 2>/dev/null || true + systemctl disable secubox-threatmesh.service secubox-threatfeed.timer 2>/dev/null || true + nft delete table inet secubox_threatmesh 2>/dev/null || true + ;; +esac +#DEBHELPER# +exit 0 diff --git a/packages/secubox-threatmesh/debian/rules b/packages/secubox-threatmesh/debian/rules new file mode 100755 index 00000000..5cec86a1 --- /dev/null +++ b/packages/secubox-threatmesh/debian/rules @@ -0,0 +1,23 @@ +#!/usr/bin/make -f +%: + dh $@ + +override_dh_auto_install: + install -d debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api + cp -r api/. debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api/ + install -d debian/secubox-threatmesh/usr/sbin + install -m 755 sbin/secubox-threatfeed debian/secubox-threatmesh/usr/sbin/ + install -d debian/secubox-threatmesh/usr/lib/systemd/system + cp systemd/*.service systemd/*.timer debian/secubox-threatmesh/usr/lib/systemd/system/ + install -d debian/secubox-threatmesh/usr/share/secubox/www + cp -r www/. debian/secubox-threatmesh/usr/share/secubox/www/ + install -d debian/secubox-threatmesh/usr/share/secubox/menu.d + cp menu.d/. -r debian/secubox-threatmesh/usr/share/secubox/menu.d/ + install -d debian/secubox-threatmesh/etc/nginx/secubox-routes.d debian/secubox-threatmesh/etc/nginx/secubox.d + cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox-routes.d/ + cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox.d/ + install -d debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d + cp nftables.d/secubox-threatmesh.nft debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d/ + +override_dh_installsystemd: + true diff --git a/packages/secubox-threatmesh/debian/source/format b/packages/secubox-threatmesh/debian/source/format new file mode 100644 index 00000000..89ae9db8 --- /dev/null +++ b/packages/secubox-threatmesh/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/packages/secubox-threatmesh/menu.d/37-threatmesh.json b/packages/secubox-threatmesh/menu.d/37-threatmesh.json new file mode 100644 index 00000000..81c2e91d --- /dev/null +++ b/packages/secubox-threatmesh/menu.d/37-threatmesh.json @@ -0,0 +1,9 @@ +{ + "id": "threatmesh", + "name": "ThreatMesh", + "icon": "🛰️", + "path": "/threatmesh/", + "category": "security", + "order": 37, + "description": "Sovereign threat-intel (feeds + mesh, no CAPI)" +} diff --git a/packages/secubox-threatmesh/nftables.d/secubox-threatmesh.nft b/packages/secubox-threatmesh/nftables.d/secubox-threatmesh.nft new file mode 100644 index 00000000..6c39debd --- /dev/null +++ b/packages/secubox-threatmesh/nftables.d/secubox-threatmesh.nft @@ -0,0 +1,10 @@ +# ThreatMesh peer port (#728): only reachable over the WireGuard mesh + loopback. +# The base inet filter INPUT chain is DEFAULT DROP, so this only OPENS 8780 to wg. +table inet secubox_threatmesh { + chain input { + type filter hook input priority 0; policy accept; + iifname "lo" tcp dport 8780 accept + iifname "wg*" tcp dport 8780 accept + tcp dport 8780 drop + } +} diff --git a/packages/secubox-threatmesh/nginx/threatmesh.conf b/packages/secubox-threatmesh/nginx/threatmesh.conf new file mode 100644 index 00000000..3f2221dc --- /dev/null +++ b/packages/secubox-threatmesh/nginx/threatmesh.conf @@ -0,0 +1,10 @@ +# /etc/nginx/secubox-routes.d/threatmesh.conf — secubox-threatmesh (#728) +location /threatmesh/ { + alias /usr/share/secubox/www/threatmesh/; + index index.html; + try_files $uri $uri/ /threatmesh/index.html; +} +location /api/v1/threatmesh/ { + proxy_pass http://127.0.0.1:8780/api/v1/threatmesh/; + include /etc/nginx/snippets/secubox-proxy.conf; +} diff --git a/packages/secubox-threatmesh/sbin/secubox-threatfeed b/packages/secubox-threatmesh/sbin/secubox-threatfeed new file mode 100644 index 00000000..b67ee389 --- /dev/null +++ b/packages/secubox-threatmesh/sbin/secubox-threatfeed @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: LicenseRef-CMSD-1.0 +# Copyright (c) 2026 CyberMind — Gérald Kerma +# Source-Disclosed License — All rights reserved except as expressly granted. +# See LICENCE-CMSD-1.0.md for terms. +""" +SecuBox-Deb :: secubox-threatfeed (Phase 1 — sovereign threat-intel #728) +CyberMind — https://cybermind.fr + +Pulls FREE, no-enrollment public IP/CIDR/domain blocklists and writes them into +the shared `threat_intel` table (toolbox.db) that secubox-blacklist-sync already +drains into the nft blacklist. This REPLACES the CrowdSec CAPI community feed +with self-sourced lists — no central account, no paywall, no IP blocklisting. + +Pure stdlib (urllib) — no extra Debian deps. Idempotent, safe on a timer. +""" +import os +import re +import sqlite3 +import sys +import time +import urllib.request +from pathlib import Path + +DB = Path(os.environ.get("SECUBOX_TI_DB", "/var/lib/secubox/toolbox/toolbox.db")) +TIMEOUT = int(os.environ.get("SECUBOX_TF_TIMEOUT", "30")) +UA = "SecuBox-ThreatFeed/1.0 (+https://secubox.in)" +TTL_DAYS = int(os.environ.get("SECUBOX_TF_TTL_DAYS", "10")) + +# (name, url, ioc_type, weight). All free + no enrollment. CIDRs allowed for ip. +FEEDS = [ + ("feodo", "https://feodotracker.abuse.ch/downloads/ipblocklist.txt", "ip", 90), + ("sslbl", "https://sslbl.abuse.ch/blacklist/sslipblacklist.txt", "ip", 85), + ("firehol1", "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset", "ip", 80), + ("spamhaus-drop", "https://www.spamhaus.org/drop/drop.txt", "ip", 85), + ("blocklist-de", "https://lists.blocklist.de/lists/all.txt", "ip", 60), + ("cins-army", "https://cinsscore.com/list/ci-badguys.txt", "ip", 70), + ("et-compromised", "https://rules.emergingthreats.net/blockrules/compromised-ips.txt", "ip", 75), + ("dshield", "https://feeds.dshield.org/block.txt", "ip", 65), +] +# Optional (set SECUBOX_TF_TOR=1 to also pull Tor exit nodes) +if os.environ.get("SECUBOX_TF_TOR") == "1": + FEEDS.append(("tor-exit", "https://check.torproject.org/torbulkexitlist", "ip", 40)) + +_IPV4 = re.compile(r"^\s*(\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?)") +_IPV6 = re.compile(r"^\s*([0-9A-Fa-f:]{2,}(?:/\d{1,3})?)\s*$") + + +def log(msg): + print(f"[threatfeed] {msg}", file=sys.stderr, flush=True) + + +def fetch(url): + req = urllib.request.Request(url, headers={"User-Agent": UA}) + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + return r.read().decode("utf-8", "replace") + + +def parse_ips(text): + out = [] + for line in text.splitlines(): + s = line.strip() + if not s or s.startswith("#") or s.startswith(";"): + continue + # spamhaus DROP: "1.2.3.0/24 ; SBL..." -> take the first field + token = s.split(";")[0].split()[0].strip() if (";" in s or " " in s) else s + m = _IPV4.match(token) + if m: + out.append((m.group(1), "ip")) + continue + m = _IPV6.match(token) + if m and ":" in m.group(1): + out.append((m.group(1), "ip")) + return out + + +def ensure_schema(c): + c.execute("""CREATE TABLE IF NOT EXISTS threat_intel ( + ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL, + weight INTEGER NOT NULL DEFAULT 50, label TEXT, + first_seen INTEGER, last_seen INTEGER, + PRIMARY KEY (ioc, ioc_type, source))""") + + +def main(): + if not DB.exists(): + log(f"db missing: {DB}") + return 1 + now = int(time.time()) + total = 0 + with sqlite3.connect(DB, timeout=20) as c: + ensure_schema(c) + for name, url, ioc_type, weight in FEEDS: + src = f"feed:{name}" + try: + iocs = parse_ips(fetch(url)) + except Exception as e: + log(f"{name}: fetch failed: {e}") + continue + n = 0 + for ioc, t in iocs: + c.execute( + "INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) " + "VALUES(?,?,?,?,?,?,?) " + "ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen, weight=excluded.weight", + (ioc, t, src, weight, name, now, now)) + n += 1 + log(f"{name}: {n} iocs") + total += n + # prune stale feed entries (older than TTL) — keeps the set fresh. + cutoff = now - TTL_DAYS * 86400 + pruned = c.execute( + "DELETE FROM threat_intel WHERE source LIKE 'feed:%' AND last_seen < ?", + (cutoff,)).rowcount + c.commit() + log(f"done: {total} iocs across {len(FEEDS)} feeds, pruned {pruned} stale") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/secubox-threatmesh/systemd/secubox-threatfeed.service b/packages/secubox-threatmesh/systemd/secubox-threatfeed.service new file mode 100644 index 00000000..6db29b3f --- /dev/null +++ b/packages/secubox-threatmesh/systemd/secubox-threatfeed.service @@ -0,0 +1,12 @@ +[Unit] +Description=SecuBox ThreatFeed — pull free public blocklists into threat_intel (#728) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +ExecStart=/usr/sbin/secubox-threatfeed +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=600 diff --git a/packages/secubox-threatmesh/systemd/secubox-threatfeed.timer b/packages/secubox-threatmesh/systemd/secubox-threatfeed.timer new file mode 100644 index 00000000..02029ee7 --- /dev/null +++ b/packages/secubox-threatmesh/systemd/secubox-threatfeed.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Refresh sovereign threat-intel feeds (#728) + +[Timer] +OnBootSec=5min +OnUnitActiveSec=6h +RandomizedDelaySec=15min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/packages/secubox-threatmesh/systemd/secubox-threatmesh.service b/packages/secubox-threatmesh/systemd/secubox-threatmesh.service new file mode 100644 index 00000000..a28134b9 --- /dev/null +++ b/packages/secubox-threatmesh/systemd/secubox-threatmesh.service @@ -0,0 +1,22 @@ +[Unit] +Description=SecuBox ThreatMesh — sovereign threat-intel mesh API (#728) +After=network.target secubox-runtime.service secubox-p2p.service +Wants=secubox-p2p.service + +[Service] +Type=simple +User=root +WorkingDirectory=/usr/lib/secubox/threatmesh +# Listens on :8780 — nginx proxies the dashboard; peers POST mesh decisions +# over the WireGuard mesh. nft drop-in restricts :8780 to wg* + loopback. +ExecStartPre=-/usr/sbin/nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft +ExecStart=/usr/bin/python3 -m uvicorn api.main:app --host 0.0.0.0 --port 8780 --workers 1 +Restart=on-failure +RestartSec=5 +NoNewPrivileges=no +ProtectSystem=full +PrivateTmp=true +ReadWritePaths=/run/secubox /var/lib/secubox /etc/secubox + +[Install] +WantedBy=multi-user.target diff --git a/packages/secubox-threatmesh/www/threatmesh/index.html b/packages/secubox-threatmesh/www/threatmesh/index.html new file mode 100644 index 00000000..6e666b7b --- /dev/null +++ b/packages/secubox-threatmesh/www/threatmesh/index.html @@ -0,0 +1,87 @@ + + + + + + + +SecuBox · ThreatMesh + + + + + + +
+
🛰️

THREATMESH

+ sovereign · CAPI off
+
+ + + +
+
+

BLOCKED IPs (aggregate)

…
feeds + mesh + local → nft
+

FREE FEEDS

…
+

MESH

…
+

SOURCES

+

MESH PEERS

…
+

TOP CONSENSUS IPs

IPconsensussources
+
+
+
+ + + +