diff --git a/api/agent_health.py b/api/agent_health.py index ea3bc572..2ee564e5 100644 --- a/api/agent_health.py +++ b/api/agent_health.py @@ -14,8 +14,10 @@ volume is shared, those checks always return ``None`` and the dashboard incorrectly shows "Gateway not running". To stay accurate without forcing a ``pid: "service:hermes-agent"`` compose workaround, we accept a recent ``updated_at`` timestamp on ``gateway_state.json`` (combined with -``gateway_state == "running"``) as an equivalent live-process signal — the -gateway already writes that file on every tick. +``gateway_state == "running"``) as an equivalent live-process signal. Older +gateway builds do not refresh that file periodically, so a stale +``gateway_state == "running"`` record is treated as inconclusive rather than a +confirmed outage. """ from __future__ import annotations @@ -126,6 +128,41 @@ def _runtime_status_is_stale_stopped( return age_s > threshold_s +def _runtime_status_is_stale_running( + runtime_status: dict[str, Any] | None, + *, + now: datetime | None = None, + threshold_s: float = GATEWAY_FRESHNESS_THRESHOLD_S, +) -> bool: + """Return ``True`` when the gateway last self-reported running, but stale. + + WebUI often runs in a separate container from the gateway. In that shape PID + checks can be impossible, and older gateway versions only update + ``gateway_state.json`` on lifecycle/platform changes. A stale ``running`` + file therefore means "not enough information from WebUI" rather than + "gateway is down". + """ + if not isinstance(runtime_status, dict): + return False + if runtime_status.get("gateway_state") != "running": + return False + + raw_updated_at = runtime_status.get("updated_at") + if not isinstance(raw_updated_at, str) or not raw_updated_at: + return False + + try: + updated_at = datetime.fromisoformat(raw_updated_at) + except (TypeError, ValueError): + return False + if updated_at.tzinfo is None: + return False + + reference = now if now is not None else datetime.now(timezone.utc) + age_s = (reference - updated_at).total_seconds() + return age_s > threshold_s + + def _gateway_status_module(): """Load gateway.status lazily so tests and WebUI-only installs stay isolated.""" return importlib.import_module("gateway.status") @@ -309,6 +346,17 @@ def build_agent_health_payload() -> dict[str, Any]: }, } + if _runtime_status_is_stale_running(runtime_status): + return { + "alive": None, + "checked_at": checked_at, + "details": { + "state": "unknown", + "reason": "gateway_stale_running_state", + **safe_details, + }, + } + if isinstance(runtime_status, dict): return { "alive": False, diff --git a/tests/test_issue1879_cross_container_gateway_liveness.py b/tests/test_issue1879_cross_container_gateway_liveness.py index 2eeaf38e..73a10b75 100644 --- a/tests/test_issue1879_cross_container_gateway_liveness.py +++ b/tests/test_issue1879_cross_container_gateway_liveness.py @@ -15,7 +15,7 @@ cross-container liveness signal. These tests pin every behavior the fix promises: * fresh + running gateway_state, no PID → alive (cross-container path) - * stale updated_at + running → down (no false positives) + * stale updated_at + running → unknown (old gateways may not tick) * fresh updated_at + non-running state → down (crash-without-cleanup case) * stale updated_at + stopped state → unknown (old root gateway was intentionally stopped; do not nag profile-gateway users) @@ -116,8 +116,8 @@ def test_cross_container_alive_path_does_not_leak_raw_process_fields(monkeypatch # -- Stale / missing / malformed timestamps ----------------------------------- -def test_stale_updated_at_reports_down_even_when_gateway_state_running(monkeypatch): - """A long-dead gateway with a fossilised state file must surface as down.""" +def test_stale_updated_at_with_running_state_reports_unknown(monkeypatch): + """Older gateways may not refresh the file while still processing messages.""" from api import agent_health stale_ts = _iso(datetime.now(timezone.utc) - timedelta(seconds=300)) @@ -130,9 +130,10 @@ def test_stale_updated_at_reports_down_even_when_gateway_state_running(monkeypat payload = agent_health.build_agent_health_payload() - assert payload["alive"] is False - assert payload["details"]["state"] == "down" - assert payload["details"]["reason"] == "gateway_not_running" + assert payload["alive"] is None + assert payload["details"]["state"] == "unknown" + assert payload["details"]["reason"] == "gateway_stale_running_state" + assert payload["details"]["gateway_state"] == "running" def test_fresh_updated_at_with_non_running_state_reports_down(monkeypatch):