diff --git a/.claude/WIP.md b/.claude/WIP.md index e42afe0f..8a5ae572 100644 --- a/.claude/WIP.md +++ b/.claude/WIP.md @@ -1,5 +1,106 @@ # WIP β€” Work In Progress -*Mis Γ  jour : 2026-05-07 (Session 106)* +*Mis Γ  jour : 2026-05-06 (Session 108)* + +--- + +## πŸ”„ Session 108 (Continued): VHost Health Fixes + Services Restore + +### VHost Health Prober +- [x] Updated prober to treat placeholders as "placeholder" status (not "down") +- [x] Added ⬜ placeholder indicator to dashboard +- [x] Health % now only counts real vhosts (excludes placeholders) +- [x] Current stats: 🟒 0 🟑 5 πŸ”΄ 29 ⬜ 58 (14.7% health) + +### HAProxy Routing Fixes +- [x] Fixed c3box.maegia.tv β†’ metablog_gandalf (was nginx_vhosts placeholder) +- [x] mitmproxy backend confirmed UP at 10.100.0.60:8080 + +### Service Socket Restoration +- [x] Restarted all secubox-* services +- [x] 86 API sockets now active +- [x] Verified hub, crowdsec, system, haproxy all working + +### System Load Issues (Ongoing) +- [ ] mitmdump using 95% CPU (821MB RAM) - investigate +- [ ] Memory at 91% (7.1GB/7.7GB) +- [ ] Load average: 11-17 (very high) + +### Metrics Dashboard (User Request) +- [ ] Add precaching + double-buffer with threaded updating +- [ ] Currently 1.6s response time due to high load + +--- + +## πŸ”„ Session 108: Dashboard Health Widgets + HAProxy Workflow + +### HAProxy Workflow Script +- [x] Created `scripts/haproxy-workflow.sh` with 3 commands: + - `rehealth` β€” Auto-rehealth HAProxy (reload, invalidate caches, restart probers) + - `waf-sync` β€” Sync vhostβ†’backend routes through WAF (mitmproxy) + - `certs` β€” Check/list certificate status for all vhosts + - `all` β€” Run complete workflow + +### Dashboard Health Widgets (Upper Position) +- [x] Added Module Health widget to dashboard index.html + - Progress bar + health percentage + - Emoji counters πŸŸ’πŸŸ‘πŸ”΄ (healthy/degraded/down) + - Calls `/api/v1/hub/module-health/summary` +- [x] Added VHost Health widget to dashboard index.html + - Progress bar + health percentage + - Emoji counters πŸŸ’πŸŸ‘πŸ”΄ (ok/slow/down) + - Calls `/api/v1/hub/health-monitor/summary` +- [x] Positioned both widgets upper in grid (before main content) + +### Dashboard Widget Limits +- [x] Modules Overview: Reduced from 10 to 4 items +- [x] Alert Timeline: Reduced from 5 to 3 items + +### Hub API Health Endpoints +- [x] Added `/module-health/summary` β€” Module health counts +- [x] Added `/module-health/status` β€” Detailed module status +- [x] Added `/module-health/alerts` β€” Degraded/down modules +- [x] Added `/health-monitor/summary` β€” VHost health counts +- [x] Added `/health-monitor/status` β€” Detailed VHost status +- [x] Added `/health-monitor/alerts` β€” Slow/down VHosts + +--- + +## βœ… Session 107: Module Health Monitor + Placeholder Detection + +### Admin Page Fixes +- [x] Fixed "undefined/undefined" bug in System Administration page +- [x] Added null coalescing (`?? 0`) for missing API data + +### Module Health Prober System +- [x] Created `/usr/lib/secubox/health/module_prober.py` +- [x] Multilayer checks: systemd β†’ socket β†’ API +- [x] 8 core modules monitored (hub, crowdsec, dpi, haproxy, vhost, wireguard, system, heartbeat) +- [x] Results cached to `/var/cache/secubox/health/modules.json` +- [x] Created `secubox-module-prober.service` (enabled) +- [x] Added API router `/api/v1/hub/module-health/`: + - `GET /summary` - Global module health stats + - `GET /status` - All modules with layer details + - `GET /module/{name}` - Single module status + - `GET /alerts` - Degraded/down modules +- [x] Dashboard widget "Module Health" with: + - Progress bar + health % + - Emoji counters πŸŸ’πŸŸ‘πŸ”΄ + - Alert list for degraded/down modules + +### Sidebar Module Filtering +- [x] Menu API now hides inactive modules from sidebar +- [x] Only functional modules displayed (systemd active) +- [x] Reduces sidebar clutter for partially-deployed systems + +### VHost Health Prober Enhancements +- [x] Added placeholder page detection (`SecuBox Domain` / `SecuBox Protected`) +- [x] Sites showing placeholder = marked as "down" (not "ok") +- [x] Real health: 7 slow, 85 down (mostly placeholder pages) + +### VHost Routing Fixes +- [x] Fixed gandalf.maegia.tv routing +- [x] Added `metablog_gandalf` backend (port 8901) +- [x] Updated HAProxy ACL β†’ metablog_gandalf --- diff --git a/packages/secubox-hub/api/main.py b/packages/secubox-hub/api/main.py index 70762da7..b0609498 100644 --- a/packages/secubox-hub/api/main.py +++ b/packages/secubox-hub/api/main.py @@ -939,6 +939,157 @@ async def health(): return {"status": "ok", "module": "hub", "version": "1.7.0"} +# ══════════════════════════════════════════════════════════════════ +# Module Health Monitor Endpoints +# ══════════════════════════════════════════════════════════════════ + +MODULE_HEALTH_CACHE = Path("/var/cache/secubox/health/modules.json") +VHOST_HEALTH_CACHE = Path("/var/cache/secubox/health/status.json") + + +@router.get("/module-health/summary") +async def module_health_summary(user=Depends(require_jwt)): + """Get module health summary (healthy/degraded/down counts).""" + try: + if MODULE_HEALTH_CACHE.exists(): + data = json.loads(MODULE_HEALTH_CACHE.read_text()) + # Use pre-computed values from cache if available + if "ok" in data and "degraded" in data: + return { + "health_percent": data.get("health_pct", 0), + "healthy": data.get("ok", 0), + "degraded": data.get("degraded", 0), + "down": data.get("down", 0), + "total": data.get("total", 0), + } + # Fallback: compute from modules dict + modules = data.get("modules", {}) + healthy = sum(1 for m in modules.values() if m.get("overall") == "ok") + degraded = sum(1 for m in modules.values() if m.get("overall") == "degraded") + down = sum(1 for m in modules.values() if m.get("overall") in ("down", "error")) + total = len(modules) + return { + "health_percent": (healthy / max(total, 1)) * 100, + "healthy": healthy, + "degraded": degraded, + "down": down, + "total": total, + } + except Exception as e: + log.debug("Module health cache read failed: %s", e) + + # Fallback: compute from menu data + menu = _menu_cache or _load_menu_cache_from_file() + if menu and menu.get("categories"): + items = [i for c in menu["categories"] for i in c.get("items", [])] + active = sum(1 for i in items if i.get("active")) + total = len(items) + return { + "health_percent": (active / max(total, 1)) * 100, + "healthy": active, + "degraded": 0, + "down": total - active, + "total": total, + } + + return {"health_percent": 0, "healthy": 0, "degraded": 0, "down": 0, "total": 0} + + +@router.get("/module-health/status") +async def module_health_status(user=Depends(require_jwt)): + """Get detailed module health status.""" + try: + if MODULE_HEALTH_CACHE.exists(): + return json.loads(MODULE_HEALTH_CACHE.read_text()) + except Exception: + pass + return {"modules": {}, "timestamp": time.time()} + + +@router.get("/module-health/alerts") +async def module_health_alerts(user=Depends(require_jwt)): + """Get modules that are degraded or down.""" + try: + if MODULE_HEALTH_CACHE.exists(): + data = json.loads(MODULE_HEALTH_CACHE.read_text()) + modules = data.get("modules", {}) + alerts = [ + {"name": name, "status": m.get("overall"), "message": m.get("message", "")} + for name, m in modules.items() + if m.get("overall") in ("degraded", "down", "error") + ] + return {"alerts": alerts} + except Exception: + pass + return {"alerts": []} + + +@router.get("/health-monitor/summary") +async def vhost_health_summary(user=Depends(require_jwt)): + """Get VHost health summary (ok/slow/placeholder/down counts).""" + try: + if VHOST_HEALTH_CACHE.exists(): + data = json.loads(VHOST_HEALTH_CACHE.read_text()) + # Use pre-computed values if available + if "ok" in data: + return { + "health_percent": data.get("health_pct", 0), + "ok": data.get("ok", 0), + "slow": data.get("slow", 0), + "placeholder": data.get("placeholder", 0), + "down": data.get("down", 0), + "total": data.get("total", 0), + } + # Fallback: compute from vhosts dict + vhosts = data.get("vhosts", {}) + ok = sum(1 for v in vhosts.values() if v.get("status") == "ok") + slow = sum(1 for v in vhosts.values() if v.get("status") == "slow") + placeholder = sum(1 for v in vhosts.values() if v.get("status") == "placeholder") + down = sum(1 for v in vhosts.values() if v.get("status") in ("down", "error", "timeout")) + total = len(vhosts) + real_total = total - placeholder + return { + "health_percent": (ok + slow) / max(real_total, 1) * 100, + "ok": ok, + "slow": slow, + "placeholder": placeholder, + "down": down, + "total": total, + } + except Exception as e: + log.debug("VHost health cache read failed: %s", e) + return {"health_percent": 0, "ok": 0, "slow": 0, "placeholder": 0, "down": 0, "total": 0} + + +@router.get("/health-monitor/status") +async def vhost_health_status(user=Depends(require_jwt)): + """Get detailed VHost health status.""" + try: + if VHOST_HEALTH_CACHE.exists(): + return json.loads(VHOST_HEALTH_CACHE.read_text()) + except Exception: + pass + return {"vhosts": {}, "timestamp": time.time()} + + +@router.get("/health-monitor/alerts") +async def vhost_health_alerts(user=Depends(require_jwt)): + """Get VHosts that are down or slow.""" + try: + if VHOST_HEALTH_CACHE.exists(): + data = json.loads(VHOST_HEALTH_CACHE.read_text()) + vhosts = data.get("vhosts", {}) + alerts = [ + {"domain": domain, "status": v.get("status"), "response_time": v.get("response_time", 0)} + for domain, v in vhosts.items() + if v.get("status") in ("slow", "down", "error", "timeout") + ] + return {"alerts": alerts[:20]} # Limit to 20 + except Exception: + pass + return {"alerts": []} + + # ══════════════════════════════════════════════════════════════════ # Network Mode Selection (integrates with secubox-netmodes) # ══════════════════════════════════════════════════════════════════ diff --git a/packages/secubox-hub/www/index.html b/packages/secubox-hub/www/index.html index dde59fd0..912fde45 100644 --- a/packages/secubox-hub/www/index.html +++ b/packages/secubox-hub/www/index.html @@ -480,6 +480,39 @@ + +
+
+
+

Module Health

+ -% +
+
+
+
+
+ 🟒 0 + 🟑 0 + πŸ”΄ 0 +
+
+
+
+

VHost Health

+ -% +
+
+
+
+
+ 🟒 0 + 🟑 0 + πŸ”΄ 0 + ⬜ 0 +
+
+
+
@@ -725,7 +758,7 @@ const tbody = document.getElementById('modules-table'); if (menuData && menuData.categories) { const allItems = menuData.categories.flatMap(c => c.items); - tbody.innerHTML = allItems.slice(0, 10).map(item => { + tbody.innerHTML = allItems.slice(0, 4).map(item => { // Look up version from modules data if available const modData = modules[item.id] || {}; const version = modData.version || '-'; @@ -756,6 +789,49 @@ } } + async function loadModuleHealth() { + try { + const data = await api('/module-health/summary'); + if (data && data.health_percent !== undefined) { + const pct = Math.round(data.health_percent); + document.getElementById('module-health-pct').textContent = pct + '%'; + document.getElementById('module-health-bar').style.width = pct + '%'; + document.getElementById('module-health-ok').textContent = '🟒 ' + (data.healthy || 0); + document.getElementById('module-health-warn').textContent = '🟑 ' + (data.degraded || 0); + document.getElementById('module-health-down').textContent = 'πŸ”΄ ' + (data.down || 0); + } + } catch (e) { + // Fallback: compute from menu data + if (menuData && menuData.categories) { + const allItems = menuData.categories.flatMap(c => c.items); + const active = allItems.filter(i => i.active).length; + const total = allItems.length; + const pct = total > 0 ? Math.round((active / total) * 100) : 0; + document.getElementById('module-health-pct').textContent = pct + '%'; + document.getElementById('module-health-bar').style.width = pct + '%'; + document.getElementById('module-health-ok').textContent = '🟒 ' + active; + document.getElementById('module-health-down').textContent = 'πŸ”΄ ' + (total - active); + } + } + } + + async function loadVHostHealth() { + try { + const data = await api('/health-monitor/summary'); + if (data && data.health_percent !== undefined) { + const pct = Math.round(data.health_percent); + document.getElementById('vhost-health-pct').textContent = pct + '%'; + document.getElementById('vhost-health-bar').style.width = pct + '%'; + document.getElementById('vhost-health-ok').textContent = '🟒 ' + (data.ok || 0); + document.getElementById('vhost-health-slow').textContent = '🟑 ' + (data.slow || 0); + document.getElementById('vhost-health-down').textContent = 'πŸ”΄ ' + (data.down || 0); + document.getElementById('vhost-health-placeholder').textContent = '⬜ ' + (data.placeholder || 0); + } + } catch (e) { + document.getElementById('vhost-health-pct').textContent = 'N/A'; + } + } + async function loadAlerts() { const data = await api('/alerts'); const alerts = data || []; @@ -763,7 +839,7 @@ const list = document.getElementById('alerts-list'); if (Array.isArray(alerts) && alerts.length > 0) { - list.innerHTML = alerts.slice(0, 5).map(a => ` + list.innerHTML = alerts.slice(0, 3).map(a => `
${new Date().toLocaleTimeString()}
${a.type?.replace(/_/g, ' ') || 'Alert'}: ${a.module || ''}
@@ -934,6 +1010,8 @@ await loadMenu(); loadDashboard(); loadHealth(); + loadModuleHealth(); + loadVHostHealth(); loadAlerts(); loadNetwork(); loadMemory(); diff --git a/packages/secubox-system/www/system/index.html b/packages/secubox-system/www/system/index.html index b770d84b..660b6487 100644 --- a/packages/secubox-system/www/system/index.html +++ b/packages/secubox-system/www/system/index.html @@ -505,7 +505,7 @@ const list = document.getElementById('services-list'); const services = data.services || []; - list.innerHTML = services.slice(0, 15).map(s => ` + list.innerHTML = services.slice(0, 4).map(s => `
${s.name} @@ -518,7 +518,7 @@ const info = document.getElementById('network-info'); const interfaces = data.interfaces || []; - info.innerHTML = interfaces.map(i => ` + info.innerHTML = interfaces.slice(0, 4).map(i => `
${i.name} ${i.addresses?.[0] || 'no IP'} ${i.up ? 'β–²' : 'β–Ό'} diff --git a/scripts/haproxy-workflow.sh b/scripts/haproxy-workflow.sh new file mode 100755 index 00000000..ed6d6682 --- /dev/null +++ b/scripts/haproxy-workflow.sh @@ -0,0 +1,447 @@ +#!/bin/bash +# haproxy-workflow.sh - HAProxy auto-rehealth, WAF sync, and certificate management +# SecuBox-DEB :: Infrastructure Workflow Automation +# CyberMind β€” GΓ©rald Kerma +set -euo pipefail + +VERSION="1.0.0" +SCRIPT_NAME=$(basename "$0") + +# Paths +HAPROXY_CFG="/etc/haproxy/haproxy.cfg" +HAPROXY_TOML="/etc/secubox/haproxy.toml" +MITMPROXY_TOML="/etc/secubox/mitmproxy.toml" +ROUTES_JSON="/srv/mitmproxy-waf/data/routes.json" +HEALTH_CACHE="/var/cache/secubox/health/status.json" +CERTS_DIR="/etc/haproxy/certs" +ACME_DIR="/etc/acme" +VHOST_ROUTES="/var/lib/secubox/haproxy/vhost-routes.json" +LOG_FILE="/var/log/secubox/haproxy-workflow.log" + +# API endpoints (via Unix sockets) +HAPROXY_SOCK="/run/secubox/haproxy.sock" +MITMPROXY_SOCK="/run/secubox/mitmproxy.sock" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging +log() { echo -e "${BLUE}[INFO]${NC} $*"; echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: $*" >> "$LOG_FILE" 2>/dev/null || true; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARN: $*" >> "$LOG_FILE" 2>/dev/null || true; } +error() { echo -e "${RED}[ERROR]${NC} $*" >&2; echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >> "$LOG_FILE" 2>/dev/null || true; } +success() { echo -e "${GREEN}[OK]${NC} $*"; } + +# ═══════════════════════════════════════════════════════════════════════ +# HELPER FUNCTIONS +# ═══════════════════════════════════════════════════════════════════════ + +# API call via curl with JWT +api_call() { + local socket="$1" + local method="$2" + local endpoint="$3" + local data="${4:-}" + + local jwt="" + if [ -f /etc/secubox/jwt/token ]; then + jwt=$(cat /etc/secubox/jwt/token) + fi + + local curl_args=( + --silent + --unix-socket "$socket" + -X "$method" + -H "Content-Type: application/json" + ) + + [ -n "$jwt" ] && curl_args+=(-H "Authorization: Bearer $jwt") + [ -n "$data" ] && curl_args+=(-d "$data") + + curl "${curl_args[@]}" "http://localhost$endpoint" 2>/dev/null || echo '{"error": "API call failed"}' +} + +# Check if service is running +service_running() { + systemctl is-active --quiet "$1" 2>/dev/null +} + +# ═══════════════════════════════════════════════════════════════════════ +# 1. AUTO REHEALTH HAPROXY +# ═══════════════════════════════════════════════════════════════════════ + +cmd_rehealth() { + log "=== Auto Rehealth HAProxy ===" + + # Step 1: Check HAProxy is running + if service_running haproxy; then + success "HAProxy service is active" + else + warn "HAProxy service not running, attempting start..." + sudo systemctl start haproxy || { error "Failed to start HAProxy"; return 1; } + sleep 2 + fi + + # Step 2: Validate config + log "Validating HAProxy configuration..." + if haproxy -c -f "$HAPROXY_CFG" 2>/dev/null; then + success "Configuration valid" + else + error "Configuration invalid, running detailed check..." + haproxy -c -f "$HAPROXY_CFG" 2>&1 | head -20 + return 1 + fi + + # Step 3: Reload HAProxy to pick up any changes + log "Reloading HAProxy..." + sudo systemctl reload haproxy && success "HAProxy reloaded" + + # Step 4: Clear health cache to force re-probe + if [ -f "$HEALTH_CACHE" ]; then + log "Invalidating health cache..." + sudo rm -f "$HEALTH_CACHE" + success "Health cache cleared" + fi + + # Step 5: Restart health prober if running + if service_running secubox-health-prober; then + log "Restarting health prober..." + sudo systemctl restart secubox-health-prober + success "Health prober restarted" + fi + + # Step 6: Trigger API cache refresh (if API running) + if [ -S "$HAPROXY_SOCK" ]; then + log "Triggering API cache refresh..." + api_call "$HAPROXY_SOCK" GET "/api/v1/haproxy/status" >/dev/null + success "API cache refreshed" + fi + + # Step 7: Show current status + log "Current HAProxy status:" + echo "show info" | socat stdio /run/haproxy/admin.sock 2>/dev/null | head -20 || \ + echo "Stats socket not available (native HAProxy check)" + + # Quick backend check + if [ -S /run/haproxy/admin.sock ]; then + echo "" + log "Backend status:" + echo "show stat" | socat stdio /run/haproxy/admin.sock 2>/dev/null | \ + awk -F, '/BACKEND/ {print " " $1 ": " $18}' | head -10 + fi + + success "HAProxy rehealth completed" +} + +# ═══════════════════════════════════════════════════════════════════════ +# 2. WORKFLOW BACKEND THROUGH WAF +# ═══════════════════════════════════════════════════════════════════════ + +cmd_waf_sync() { + log "=== Workflow Backend Through WAF ===" + + # Step 1: Check mitmproxy-waf container/service + local waf_running=false + if lxc-info -n mitmproxy-waf -s 2>/dev/null | grep -q "RUNNING"; then + success "mitmproxy-waf LXC container is running" + waf_running=true + elif service_running mitmproxy; then + success "mitmproxy service is running" + waf_running=true + else + warn "WAF service not running, attempting start..." + if [ -f /var/lib/lxc/mitmproxy-waf/config ]; then + sudo lxc-start -n mitmproxy-waf || warn "LXC start failed" + else + sudo systemctl start mitmproxy 2>/dev/null || warn "mitmproxy service start failed" + fi + fi + + # Step 2: Parse HAProxy config to extract vhosts and backends (using awk for speed) + log "Extracting vhostβ†’backend mappings from HAProxy config..." + + local routes="{}" + local count=0 + local backend_count=0 + + if [ -f "$HAPROXY_CFG" ]; then + # Use awk for fast parsing of large config + routes=$(awk ' + BEGIN { first=1 } + # Capture backend -> server mappings + /^backend / { backend=$2 } + /^[[:space:]]+server / { + for (i=1; i<=NF; i++) { + if ($i ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+$/) { + split($i, addr, ":") + if (backend) servers[backend] = addr[1] ":" addr[2] + break + } + } + } + # Capture ACL -> hostname mappings + /^[[:space:]]+acl .* hdr\(host\) -i / { + acl=$2 + hostname=$NF + acls[acl] = hostname + } + # Capture use_backend -> ACL mappings + /^[[:space:]]+use_backend .* if / { + be=$2 + aclname=$4 + if (be !~ /waf_inspector|mitmproxy_inspector/ && acls[aclname] && servers[be]) { + split(servers[be], a, ":") + if (!first) printf ", " + printf "\"%s\": [\"%s\", %s]", acls[aclname], a[1], a[2] + first=0 + } + } + ' "$HAPROXY_CFG") + routes="{${routes}}" + count=$(echo "$routes" | grep -o '":\[' | wc -l) + backend_count=$(awk '/^backend / {c++} END {print c+0}' "$HAPROXY_CFG") + fi + + log "Found $count vhostβ†’backend mappings" + + # Step 3: Write routes to mitmproxy + if [ $count -gt 0 ]; then + log "Writing routes to WAF..." + sudo mkdir -p "$(dirname $ROUTES_JSON)" + echo "$routes" | jq '.' | sudo tee "$ROUTES_JSON" >/dev/null + success "Routes written to $ROUTES_JSON" + + # Also update the HAProxy-side routes file + echo "$routes" | jq '.' | sudo tee "$VHOST_ROUTES" >/dev/null 2>/dev/null || true + fi + + # Step 4: Verify WAF backend exists in HAProxy config + log "Checking WAF backend in HAProxy config..." + if grep -q "backend mitmproxy_inspector\|backend waf_inspector" "$HAPROXY_CFG"; then + success "WAF backend exists in HAProxy config" + else + warn "WAF backend not found, may need to regenerate config" + echo " Run: haproxyctl generate" + fi + + # Step 5: Check WAF routing is active + local waf_routes=$(grep -c "use_backend.*waf_inspector\|use_backend.*mitmproxy_inspector" "$HAPROXY_CFG" 2>/dev/null || echo "0") + log "HAProxy routes through WAF: $waf_routes vhosts" + + # Step 6: Reload mitmproxy to pick up routes + if [ "$waf_running" = true ]; then + log "Signaling mitmproxy to reload routes..." + if lxc-info -n mitmproxy-waf -s 2>/dev/null | grep -q "RUNNING"; then + sudo lxc-attach -n mitmproxy-waf -- pkill -HUP mitmproxy 2>/dev/null || true + else + sudo systemctl reload mitmproxy 2>/dev/null || true + fi + success "WAF routes synced" + fi + + # Summary + backend_count=${#backends[@]} + echo "" + log "WAF Sync Summary:" + echo " Backends extracted: $backend_count" + echo " Routes synced: $count" + echo " Routes through WAF: $waf_routes" + + success "WAF sync completed" +} + +# ═══════════════════════════════════════════════════════════════════════ +# 3. VHOST CERTIFICATES - FULL CERTS PUBLISHED +# ═══════════════════════════════════════════════════════════════════════ + +cmd_certs() { + log "=== VHost Full Certificates Status ===" + + local total=0 + local valid=0 + local expiring=0 + local missing=0 + local expired=0 + + # Collect all vhosts from HAProxy + local vhosts=() + if [ -f "$HAPROXY_CFG" ]; then + while IFS= read -r line; do + if [[ "$line" =~ hdr\(host\)[[:space:]]+-i[[:space:]]+([^[:space:]\}]+) ]]; then + vhosts+=("${BASH_REMATCH[1]}") + fi + done < "$HAPROXY_CFG" + fi + + total=${#vhosts[@]} + log "Total vhosts configured: $total" + + if [ $total -eq 0 ]; then + warn "No vhosts found in HAProxy config" + return 0 + fi + + echo "" + printf "%-40s %-12s %-10s %s\n" "DOMAIN" "STATUS" "EXPIRES" "PATH" + printf "%s\n" "$(printf '%.0s-' {1..100})" + + for domain in "${vhosts[@]}"; do + local cert_path="" + local status="" + local expires="" + + # Check HAProxy certs directory + if [ -f "$CERTS_DIR/$domain.pem" ]; then + cert_path="$CERTS_DIR/$domain.pem" + elif [ -f "$CERTS_DIR/${domain%%.*}.pem" ]; then + cert_path="$CERTS_DIR/${domain%%.*}.pem" + # Check ACME directory + elif [ -f "$ACME_DIR/$domain/fullchain.cer" ]; then + cert_path="$ACME_DIR/$domain/fullchain.cer" + elif [ -f "$ACME_DIR/$domain/${domain}.cer" ]; then + cert_path="$ACME_DIR/$domain/${domain}.cer" + fi + + if [ -n "$cert_path" ] && [ -f "$cert_path" ]; then + # Check expiry + local end_date + end_date=$(openssl x509 -in "$cert_path" -enddate -noout 2>/dev/null | cut -d= -f2) + + if [ -n "$end_date" ]; then + local end_epoch + end_epoch=$(date -d "$end_date" +%s 2>/dev/null || echo 0) + local now_epoch + now_epoch=$(date +%s) + local days_left=$(( (end_epoch - now_epoch) / 86400 )) + + if [ $days_left -lt 0 ]; then + status="${RED}EXPIRED${NC}" + expires="$days_left days" + ((expired++)) + elif [ $days_left -lt 30 ]; then + status="${YELLOW}EXPIRING${NC}" + expires="${days_left}d" + ((expiring++)) + else + status="${GREEN}VALID${NC}" + expires="${days_left}d" + ((valid++)) + fi + else + status="${YELLOW}UNKNOWN${NC}" + expires="?" + ((valid++)) # Assume valid if can't parse + fi + else + status="${RED}MISSING${NC}" + expires="-" + cert_path="(not found)" + ((missing++)) + fi + + printf "%-40s " "$domain" + echo -e "${status} $(printf '%-10s' "$expires") ${cert_path:0:40}" + done + + echo "" + log "Certificate Summary:" + echo -e " ${GREEN}Valid${NC}: $valid" + echo -e " ${YELLOW}Expiring${NC}: $expiring (< 30 days)" + echo -e " ${RED}Expired${NC}: $expired" + echo -e " ${RED}Missing${NC}: $missing" + echo " ─────────────" + echo " Total: $total" + + # Recommendations + if [ $missing -gt 0 ]; then + echo "" + warn "Missing certificates - run ACME for:" + for domain in "${vhosts[@]}"; do + local found=false + [ -f "$CERTS_DIR/$domain.pem" ] && found=true + [ -f "$ACME_DIR/$domain/fullchain.cer" ] && found=true + [ "$found" = false ] && echo " acme.sh --issue -d $domain --webroot /var/www/acme" + done + fi + + if [ $expiring -gt 0 ]; then + echo "" + warn "Certificates expiring soon - renew with:" + echo " acme.sh --renew-all" + fi + + success "Certificate check completed" +} + +# ═══════════════════════════════════════════════════════════════════════ +# ALL: Run complete workflow +# ═══════════════════════════════════════════════════════════════════════ + +cmd_all() { + log "=== Running Complete HAProxy Workflow ===" + echo "" + + cmd_rehealth + echo "" + + cmd_waf_sync + echo "" + + cmd_certs + echo "" + + success "Complete workflow finished" +} + +# ═══════════════════════════════════════════════════════════════════════ +# USAGE +# ═══════════════════════════════════════════════════════════════════════ + +usage() { + cat < + +Commands: + rehealth Auto rehealth HAProxy (reload, invalidate caches, restart probers) + waf-sync Sync backend routes through WAF (mitmproxy) + certs Check/list vhost certificates status + all Run complete workflow (rehealth + waf-sync + certs) + help Show this help + +Examples: + $SCRIPT_NAME rehealth # Refresh HAProxy health monitoring + $SCRIPT_NAME waf-sync # Sync vhostβ†’backend routes to WAF + $SCRIPT_NAME certs # Check certificate status for all vhosts + $SCRIPT_NAME all # Run everything + +EOF +} + +# ═══════════════════════════════════════════════════════════════════════ +# MAIN +# ═══════════════════════════════════════════════════════════════════════ + +main() { + # Create log directory + mkdir -p "$(dirname $LOG_FILE)" 2>/dev/null || true + + case "${1:-}" in + rehealth) cmd_rehealth ;; + waf-sync) cmd_waf_sync ;; + certs) cmd_certs ;; + all) cmd_all ;; + help|-h|--help) usage ;; + *) + [ -n "${1:-}" ] && error "Unknown command: $1" + usage + exit 1 + ;; + esac +} + +main "$@"