From c0c65fcb513428b9546960129a34aec826f3aea7 Mon Sep 17 00:00:00 2001 From: CyberMind-FR Date: Sat, 25 Jul 2026 16:20:22 +0200 Subject: [PATCH] fix: aggregator-watchdog startup grace + wpad-vhost listen :9080 (backport live gk2 fixes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two board-stability fixes verified live on gk2 during the assist deploy: - secubox-aggregator-watchdog.sh: 480s startup grace so the auto-heal watchdog stops restart-looping the aggregator during its slow (~5min) in-process cold mount (RuntimeDirectoryPreserve stale socket → probe fails → kill mid-start). - secubox-proxypac wpad-vhost: listen :9080 not :80 (HAProxy owns :80; the bind conflict aborted every nginx reload, silently freezing config board-wide). Co-Authored-By: Gerald KERMA --- packages/secubox-aggregator/debian/changelog | 10 ++++++++++ .../sbin/secubox-aggregator-watchdog.sh | 15 +++++++++++++++ packages/secubox-proxypac/debian/changelog | 8 ++++++++ packages/secubox-proxypac/nginx/wpad-vhost.conf | 7 ++++++- 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/packages/secubox-aggregator/debian/changelog b/packages/secubox-aggregator/debian/changelog index 0ba9cf43..58e3e4c9 100644 --- a/packages/secubox-aggregator/debian/changelog +++ b/packages/secubox-aggregator/debian/changelog @@ -1,3 +1,13 @@ +secubox-aggregator (0.2.4-1~bookworm1) bookworm; urgency=medium + + * Watchdog: 480s startup grace before auto-heal restart. The aggregator's + in-process cold mount of ~110 modules takes minutes and RuntimeDirectoryPreserve + keeps the socket file, so the probe saw "present but unresponsive" and the + 2-strike auto-heal restarted it before it could bind (self-inflicted loop → + board-wide 502). Skip the restart while MainPID uptime < SECUBOX_AGG_WD_GRACE. + + -- Gerald KERMA Fri, 25 Jul 2026 12:00:00 +0200 + secubox-aggregator (0.2.3-1~bookworm1) bookworm; urgency=medium * #727 auto-heal watchdog: ship secubox-aggregator-watchdog.{sh,service,timer}. diff --git a/packages/secubox-aggregator/sbin/secubox-aggregator-watchdog.sh b/packages/secubox-aggregator/sbin/secubox-aggregator-watchdog.sh index 87890a76..ff8158de 100755 --- a/packages/secubox-aggregator/sbin/secubox-aggregator-watchdog.sh +++ b/packages/secubox-aggregator/sbin/secubox-aggregator-watchdog.sh @@ -40,6 +40,21 @@ n=$(( $(cat "$STATE" 2>/dev/null || echo 0) + 1 )) echo "$n" > "$STATE" 2>/dev/null || true logger -t "$MODULE" "aggregator probe failed (code=$code, streak=$n/$FAIL_THRESHOLD)" +# Startup grace: the aggregator mounts ~110 modules in-process on cold start +# (several minutes). RuntimeDirectoryPreserve keeps the socket FILE across +# restarts, so a still-starting aggregator looks "present but unresponsive" to +# the probe. Do NOT auto-heal while it is within the grace window, or we kill it +# before it can finish binding (self-inflicted restart loop). +GRACE="${SECUBOX_AGG_WD_GRACE:-480}" +_mainpid=$(systemctl show secubox-aggregator -p MainPID --value 2>/dev/null || echo 0) +if [ -n "$_mainpid" ] && [ "$_mainpid" != 0 ]; then + _up=$(ps -o etimes= -p "$_mainpid" 2>/dev/null | tr -d " ") + if [ -n "$_up" ] && [ "$_up" -lt "$GRACE" ]; then + logger -t "$MODULE" "aggregator up ${_up}s (<${GRACE}s grace) — still starting, not restarting" + exit 0 + fi +fi + if [ "$n" -ge "$FAIL_THRESHOLD" ]; then logger -t "$MODULE" "restarting secubox-aggregator (auto-heal)" systemctl restart secubox-aggregator.service diff --git a/packages/secubox-proxypac/debian/changelog b/packages/secubox-proxypac/debian/changelog index fdcce915..a6aca902 100644 --- a/packages/secubox-proxypac/debian/changelog +++ b/packages/secubox-proxypac/debian/changelog @@ -1,3 +1,11 @@ +secubox-proxypac (1.2.1-1~bookworm1) bookworm; urgency=medium + + * wpad-vhost: listen on :9080 (internal) instead of :80. HAProxy fronts :80, + so `listen 80;` never bound and made every `nginx -s reload` abort on the + bind conflict — config changes silently stopped applying board-wide. + + -- Gerald KERMA Fri, 25 Jul 2026 12:00:00 +0200 + secubox-proxypac (1.2.0-1~bookworm1) bookworm; urgency=medium * Package proxypac.toml (conffile) + proxypac-wpad ctl + scoped sudoers diff --git a/packages/secubox-proxypac/nginx/wpad-vhost.conf b/packages/secubox-proxypac/nginx/wpad-vhost.conf index f5b9d65f..a39310d6 100644 --- a/packages/secubox-proxypac/nginx/wpad-vhost.conf +++ b/packages/secubox-proxypac/nginx/wpad-vhost.conf @@ -1,6 +1,11 @@ # secubox-proxypac — WPAD auto-discovery vhost (LAN + mesh only, never public) server { - listen 80; + # Listen on the internal SecuBox nginx port (HAProxy fronts :80/:443 and + # routes wpad. here by Host). A bare `listen 80;` collides with + # HAProxy's :80 and makes EVERY `nginx -s reload` abort on the bind, so + # config changes silently stop applying board-wide. See + # project_gk2_nginx_multimaster_blocker. + listen 9080; server_name wpad.gk2.secubox.in; allow 192.168.0.0/16; allow 10.10.0.0/24;