release: #7160 stale-runtime report + hardened Agent-update marker read (#7502)

* rebase #7160 onto master (report unverified Agent updates, no auto-restart)

Co-authored-by: hrdwdmrbl <hrdwdmrbl@users.noreply.github.com>

* harden agent-update marker read: no-follow, fstat regular-file, bounded (fixes gate CORE)

The Agent update marker is attacker-adjacent shared state. _read_live_agent_update()
used Path.read_text(), which follows symlinks, blocks forever on a FIFO, and reads an
unbounded regular file — a rejected/stale-runtime request could hang or OOM. Now opens
O_RDONLY|O_NONBLOCK|O_NOFOLLOW, fstat-verifies a small regular file, and reads a bounded
max, classifying anything else as 'unknown' (fail-closed). Adds FIFO/oversized/symlink/
happy-path regression tests.

Co-authored-by: hrdwdmrbl <hrdwdmrbl@users.noreply.github.com>

* fix marker-read portability: gate os.open fast-path on O_NONBLOCK+O_NOFOLLOW

os.O_NONBLOCK is Unix-only; accessing it unconditionally raised AttributeError
on native Windows, breaking every stale-revision barrier (chat/compression/etc).
Now both flags resolve via getattr and the os.open() fast path is taken only
when BOTH are available (_MARKER_SAFE_OPEN_AVAILABLE); otherwise fall back to an
lstat-only classification (absent vs unknown) so a fallback platform can never
symlink-traverse or crash. Adds a Windows-fallback regression test.

Co-authored-by: hrdwdmrbl <hrdwdmrbl@users.noreply.github.com>

* docs(changelog): stamp #7160 stale-runtime report + hardened marker read

---------

Co-authored-by: n <a@n>
Co-authored-by: hrdwdmrbl <hrdwdmrbl@users.noreply.github.com>

CHANGELOG.md

@@ -5,6 +5,8 @@
 
 ### Fixed
 
+- **A stale WebUI runtime after an Agent update is now reported and rejected without an unsafe automatic restart, and its update-marker read can no longer hang or exhaust memory.** When the running Agent revision no longer matches the on-disk checkout, WebUI keeps returning the manual-restart `409` and now surfaces richer update diagnostics while leaving the actual restart to the operator (it never restarts itself into a possibly half-applied checkout). The shared update marker — attacker-adjacent state any process with write access to the Agent home can create — is read defensively: the read never follows a symlink, never blocks on a FIFO/device, and never reads an unbounded file, classifying anything that is not a small regular file as unknown (fail-closed), with a portable fallback on platforms lacking the atomic open flags. Thanks @hrdwdmrbl. (#7160)
+
 - **Sidebar session polling no longer retains full transcripts and run-journal detail in the response cache.** Cached rows are now projected to the canonical sidebar field allowlist before storage, heavy metadata is stripped across index and full-scan paths, nested request values are isolated, and invalidated rebuilds are rejected atomically instead of being reinserted after a concurrent clear. Thanks @martindell. (#7489)
 
 - **The workspace switcher now lists workspaces in your configured order instead of re-sorting them alphabetically.** The chat-header workspace dropdown alphabetized its entries client-side, so it disagreed with the drag-and-drop order shown in the Workspaces settings panel (and never kept the default "Home" workspace first). The dropdown now renders the server's stored order verbatim — the same order as the settings view — with no client-side re-sort. Thanks @CharlesMcquade. (#7317)

api/agent_runtime.py

@@ -8,20 +8,54 @@ mixed runtime and require a clean WebUI restart instead.
 
 from __future__ import annotations
 
+import errno
+import math
+import os
 from pathlib import Path
+import stat
 import sys
 import subprocess
 import threading
+import time
 
 # Retain the discovered path as a diagnostic/test-visible compatibility value;
 # runtime identity is deliberately captured from the loaded module below.
-from api.config import _AGENT_DIR  # noqa: F401
+from api.config import (
+    PYTHON_EXE,
+    _AGENT_DIR,  # noqa: F401
+    _DEFAULT_STATE_HOME,
+)
 from api.subprocess_utils import windows_hide_flags
 
-_RESTART_MESSAGE = (
+_RESTART_REQUIRED_MESSAGE = (
     "Hermes Agent was updated while Hermes WebUI was running. "
-    "Restart Hermes WebUI before retrying this action."
+    "WebUI cannot verify that the Agent update completed safely. "
+    "Check the Agent update outcome and environment first. "
+    "Restart Hermes WebUI manually before retrying this action."
 )
+_AGENT_UPDATE_MARKER = ".hermes-update-in-progress"
+_AGENT_RECOVERY_MARKERS = (".update-incomplete", ".lazy-refresh-incomplete")
+_AGENT_UPDATE_MAX_AGE_SECONDS = 20 * 60
+# The update marker holds a PID and a start timestamp (two short numeric lines).
+# Anything larger is not a legitimate marker; cap the read so a huge or growing
+# regular file can never exhaust memory on the stale-runtime request path.
+_AGENT_UPDATE_MARKER_MAX_BYTES = 64 * 1024
+# O_NOFOLLOW is POSIX; on platforms that lack it the fast os.open() path is not
+# taken at all (see _MARKER_SAFE_OPEN_AVAILABLE below).
+# The marker read hardening relies on two POSIX-only open flags to stay both
+# non-blocking (never hang on a FIFO/device) and symlink-safe. O_NONBLOCK is
+# Unix-only and O_NOFOLLOW is absent on some platforms; accessing them
+# unconditionally raises AttributeError on native Windows. Resolve them safely
+# and only take the os.open() fast path when BOTH are genuinely available —
+# otherwise the read cannot prove non-blocking + no-follow and must fall back to
+# an lstat-only classification (see _read_live_agent_update).
+_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
+_O_NONBLOCK = getattr(os, "O_NONBLOCK", 0)
+_MARKER_SAFE_OPEN_AVAILABLE = bool(getattr(os, "O_NOFOLLOW", 0)) and bool(
+    getattr(os, "O_NONBLOCK", 0)
+)
+_HERMES_HOME = Path(_DEFAULT_STATE_HOME)
+_AGENT_PYTHON = Path(PYTHON_EXE).expanduser() if PYTHON_EXE else None
 
 
 def _read_agent_revision(
@@ -100,6 +134,233 @@ _RUNTIME_LOCK = threading.Lock()
 class AgentRuntimeChangedError(RuntimeError):
     """Raised when the loaded Agent runtime no longer matches its source tree."""
 
+    def __init__(
+        self,
+        message: str,
+        *,
+        agent_update_state: str | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.agent_update_state = agent_update_state
+
+
+def agent_runtime_stale_payload(exc: AgentRuntimeChangedError) -> dict:
+    """Return the shared retry response for every stale-runtime entry point."""
+    payload = {
+        "error": str(exc),
+        "type": "agent_runtime_stale",
+        "retryable": True,
+        "restart_scheduled": False,
+    }
+    if exc.agent_update_state is not None:
+        payload["agent_update_state"] = exc.agent_update_state
+    return payload
+
+
+def _pid_is_alive(pid: int) -> bool | None:
+    """Return PID liveness, or ``None`` when the platform cannot confirm it."""
+    if pid <= 0:
+        return False
+    if pid.bit_length() > 32:
+        return None
+    if sys.platform == "win32":
+        try:
+            import ctypes
+            from ctypes import wintypes
+
+            kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+            kernel32.OpenProcess.argtypes = (
+                wintypes.DWORD,
+                wintypes.BOOL,
+                wintypes.DWORD,
+            )
+            kernel32.OpenProcess.restype = wintypes.HANDLE
+            kernel32.GetExitCodeProcess.argtypes = (
+                wintypes.HANDLE,
+                ctypes.POINTER(wintypes.DWORD),
+            )
+            kernel32.GetExitCodeProcess.restype = wintypes.BOOL
+            kernel32.CloseHandle.argtypes = (wintypes.HANDLE,)
+            kernel32.CloseHandle.restype = wintypes.BOOL
+            handle = kernel32.OpenProcess(0x1000, False, pid)
+            if not handle:
+                error = ctypes.get_last_error()
+                if error == 5:  # ERROR_ACCESS_DENIED still proves the PID exists.
+                    return True
+                if error == 87:  # ERROR_INVALID_PARAMETER for a missing PID.
+                    return False
+                return None
+            try:
+                exit_code = wintypes.DWORD()
+                if not kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)):
+                    return None
+                return exit_code.value == 259  # STILL_ACTIVE
+            finally:
+                kernel32.CloseHandle(handle)
+        except (AttributeError, OSError, TypeError, ValueError):
+            return None
+
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    except (OverflowError, ValueError):
+        return None
+    except OSError as exc:
+        if exc.errno == errno.ESRCH:
+            return False
+        if exc.errno == errno.EPERM:
+            return True
+        return None
+    return True
+
+
+def _read_live_agent_update(marker: Path) -> str:
+    """Classify the shared Agent update marker without changing Agent state.
+
+    The marker is attacker-adjacent shared state (any process that can write the
+    Agent home can create it), so the read is hardened: never follow a symlink,
+    never block on a FIFO/device, and never read an unbounded regular file.
+    Anything that is not a small regular file is classified ``unknown`` rather
+    than allowed to hang or exhaust memory on a stale-runtime request path.
+    """
+    if not _MARKER_SAFE_OPEN_AVAILABLE:
+        # Without both O_NONBLOCK and O_NOFOLLOW we cannot prove the read is
+        # non-blocking and symlink-safe (e.g. native Windows), so never open the
+        # marker: an unverifiable marker fails closed to ``unknown``, and only a
+        # genuinely missing path is ``absent``.
+        try:
+            marker.lstat()
+        except FileNotFoundError:
+            return "absent"
+        except (OSError, ValueError, TypeError):
+            return "unknown"
+        return "unknown"
+    try:
+        fd = os.open(marker, os.O_RDONLY | _O_NONBLOCK | _O_NOFOLLOW)
+    except FileNotFoundError:
+        try:
+            marker.lstat()
+        except FileNotFoundError:
+            return "absent"
+        except OSError:
+            return "unknown"
+        # Path exists to lstat (e.g. a dangling/looping symlink) but O_NOFOLLOW
+        # refused to open it — treat as an unverifiable marker.
+        return "unknown"
+    except (OSError, ValueError, TypeError):
+        # ELOOP (symlink under O_NOFOLLOW), ENXIO/EWOULDBLOCK (FIFO with no
+        # writer under O_NONBLOCK), a non-path marker object, or any other open
+        # failure — fail closed: an unreadable marker is never proof of safety.
+        return "unknown"
+
+    try:
+        try:
+            st = os.fstat(fd)
+        except OSError:
+            return "unknown"
+        if not stat.S_ISREG(st.st_mode):
+            # FIFO, device, directory, socket — never a legitimate marker.
+            return "unknown"
+        if st.st_size > _AGENT_UPDATE_MARKER_MAX_BYTES:
+            return "unknown"
+        try:
+            # Read one byte past the cap so an oversized file that lied about
+            # st_size (or grew mid-read) is still rejected rather than truncated.
+            data = os.read(fd, _AGENT_UPDATE_MARKER_MAX_BYTES + 1)
+        except (OSError, BlockingIOError):
+            return "unknown"
+    finally:
+        try:
+            os.close(fd)
+        except OSError:
+            pass
+
+    if len(data) > _AGENT_UPDATE_MARKER_MAX_BYTES:
+        return "unknown"
+    try:
+        raw = data.decode("utf-8")
+    except UnicodeError:
+        return "unknown"
+
+    lines = raw.splitlines()
+    try:
+        pid = int(lines[0].strip())
+        started_at = float(lines[1].strip())
+    except (IndexError, TypeError, ValueError):
+        return "unknown"
+    if pid <= 0 or not math.isfinite(started_at):
+        return "unknown"
+
+    age_seconds = time.time() - started_at
+    if age_seconds < 0:
+        return "unknown"
+    if age_seconds > _AGENT_UPDATE_MAX_AGE_SECONDS:
+        return "stale"
+    alive = _pid_is_alive(pid)
+    if alive is None:
+        return "unknown"
+    return "active" if alive else "stale"
+
+
+def _marker_presence(marker: Path) -> str:
+    """Return ``present``, ``absent``, or ``unknown`` for a recovery marker."""
+    try:
+        marker.lstat()
+    except FileNotFoundError:
+        return "absent"
+    except OSError:
+        return "unknown"
+    return "present"
+
+
+def _agent_install_roots() -> tuple[Path, ...]:
+    """Return portable roots that can own the Agent's venv recovery markers."""
+    candidates: list[Path] = []
+    if _AGENT_SOURCE_DIR is not None:
+        candidates.append(_AGENT_SOURCE_DIR)
+    if _AGENT_PYTHON is not None:
+        # A venv Python is commonly a symlink to a shared interpreter. Keep the
+        # configured venv path so its installation's recovery markers are read.
+        python_path = _AGENT_PYTHON
+        if python_path.parent.name.lower() in {"bin", "scripts"}:
+            venv_dir = python_path.parent.parent
+            if venv_dir.name.lower() in {"venv", ".venv"}:
+                candidates.append(venv_dir.parent)
+
+    roots: list[Path] = []
+    seen: set[str] = set()
+    for candidate in candidates:
+        key = os.path.normcase(os.path.abspath(str(candidate)))
+        if key not in seen:
+            seen.add(key)
+            roots.append(candidate)
+    return tuple(roots)
+
+
+def _agent_update_transaction_state() -> str:
+    """Report marker diagnostics, never proof of successful completion.
+
+    The Agent removes its active marker on failed/interrupted exits too. Neither
+    its absence nor a stale PID proves the checkout or environment is healthy.
+    """
+    live_state = _read_live_agent_update(_HERMES_HOME / _AGENT_UPDATE_MARKER)
+    if live_state == "unknown":
+        return "unknown"
+
+    recovery_present = False
+    for root in _agent_install_roots():
+        for marker_name in _AGENT_RECOVERY_MARKERS:
+            presence = _marker_presence(root / marker_name)
+            if presence == "unknown":
+                return "unknown"
+            recovery_present = recovery_present or presence == "present"
+    if recovery_present:
+        return "incomplete"
+    return "unverified" if live_state == "absent" else live_state
+
 
 def _loaded_agent_source_identity() -> tuple[Path, Path] | None:
     """Return the source directory and file that supplied ``run_agent``."""
@@ -140,7 +401,14 @@ def ensure_agent_runtime_current() -> None:
         _read_agent_revision(_AGENT_SOURCE_DIR, module_path=_AGENT_MODULE_PATH)
         != _AGENT_REVISION
     ):
-        raise AgentRuntimeChangedError(_RESTART_MESSAGE)
+        # Automatic restart needs an Agent-owned success receipt bound to this
+        # transaction, final revision and healthy environment, plus an atomic
+        # handoff excluding mutations across replacement. Marker polling and a
+        # final revision read supply neither contract. Keep this path manual.
+        raise AgentRuntimeChangedError(
+            _RESTART_REQUIRED_MESSAGE,
+            agent_update_state=_agent_update_transaction_state(),
+        )
 
 
 def require_ai_agent_class():

api/routes.py

@@ -36,6 +36,7 @@ from urllib.error import HTTPError, URLError
 from urllib.request import HTTPRedirectHandler, HTTPSHandler, ProxyHandler, Request, build_opener
 from api.agent_runtime import (
     AgentRuntimeChangedError,
+    agent_runtime_stale_payload,
     ensure_agent_runtime_current,
     require_ai_agent_class,
 )
@@ -23005,11 +23006,7 @@ def _agent_runtime_barrier_response(
     try:
         ensure_agent_runtime_current()
     except AgentRuntimeChangedError as exc:
-        return {
-            "error": str(exc),
-            "type": "agent_runtime_stale",
-            "retryable": True,
-        }
+        return agent_runtime_stale_payload(exc)
     return None
 
 
@@ -25120,11 +25117,7 @@ def _handle_git_commit_message(handler, body):
     except GitWorkspaceError as e:
         return _git_bad(handler, e)
     except AgentRuntimeChangedError as e:
-        return j(handler, {
-            "error": str(e),
-            "type": "agent_runtime_stale",
-            "retryable": True,
-        }, status=409)
+        return j(handler, agent_runtime_stale_payload(e), status=409)
     except Exception as e:
         logger.exception("git commit message generation failed")
         return bad(handler, _sanitize_error(e), 500)
@@ -25157,11 +25150,7 @@ def _handle_git_commit_message_selected(handler, body):
     except GitWorkspaceError as e:
         return _git_bad(handler, e)
     except AgentRuntimeChangedError as e:
-        return j(handler, {
-            "error": str(e),
-            "type": "agent_runtime_stale",
-            "retryable": True,
-        }, status=409)
+        return j(handler, agent_runtime_stale_payload(e), status=409)
     except Exception as e:
         logger.exception("selected git commit message generation failed")
         return bad(handler, _sanitize_error(e), 500)
@@ -26718,6 +26707,10 @@ def _manual_compression_status_payload(job):
             payload["type"] = job["error_type"]
         if job.get("retryable") is not None:
             payload["retryable"] = bool(job["retryable"])
+        if job.get("restart_scheduled") is not None:
+            payload["restart_scheduled"] = bool(job["restart_scheduled"])
+        if job.get("agent_update_state") is not None:
+            payload["agent_update_state"] = job["agent_update_state"]
     elif status == "cancelled":
         payload["ok"] = False
         payload["error"] = job.get("error") or "Compression cancelled"
@@ -26754,6 +26747,8 @@ def _run_manual_compression_job(sid, body):
                         "error_status": status,
                         "error_type": (payload or {}).get("type"),
                         "retryable": (payload or {}).get("retryable"),
+                        "restart_scheduled": (payload or {}).get("restart_scheduled"),
+                        "agent_update_state": (payload or {}).get("agent_update_state"),
                         "updated_at": now,
                     }
                 )
@@ -26767,16 +26762,19 @@ def _run_manual_compression_job(sid, body):
                 )
     except AgentRuntimeChangedError as exc:
         logger.warning("Manual compression worker found stale Agent runtime for session %s", sid)
+        stale_payload = agent_runtime_stale_payload(exc)
         with _MANUAL_COMPRESSION_JOBS_LOCK:
             job = _MANUAL_COMPRESSION_JOBS.get(sid)
             if job:
                 job.update(
                     {
                         "status": "error",
-                        "error": str(exc),
+                        "error": stale_payload["error"],
                         "error_status": 409,
-                        "error_type": "agent_runtime_stale",
-                        "retryable": True,
+                        "error_type": stale_payload["type"],
+                        "retryable": stale_payload["retryable"],
+                        "restart_scheduled": stale_payload.get("restart_scheduled"),
+                        "agent_update_state": stale_payload.get("agent_update_state"),
                         "updated_at": time.time(),
                     }
                 )
@@ -26834,11 +26832,7 @@ def _handle_session_compress_start(handler, body):
     except AgentRuntimeChangedError as exc:
         return j(
             handler,
-            {
-                "error": str(exc),
-                "type": "agent_runtime_stale",
-                "retryable": True,
-            },
+            agent_runtime_stale_payload(exc),
             status=409,
         )
 
@@ -27202,11 +27196,7 @@ def _handle_session_compress(handler, body):
             },
         )
     except AgentRuntimeChangedError as e:
-        return j(handler, {
-            "error": str(e),
-            "type": "agent_runtime_stale",
-            "retryable": True,
-        }, status=409)
+        return j(handler, agent_runtime_stale_payload(e), status=409)
     except Exception as e:
         logger.warning("Manual session compression failed: %s", e)
         return bad(handler, f"Compression failed: {_sanitize_error(e)}")
@@ -27881,11 +27871,7 @@ def _handle_handoff_summary(handler, body):
             "fallback": fallback,
         })
     except AgentRuntimeChangedError as e:
-        return j(handler, {
-            "error": str(e),
-            "type": "agent_runtime_stale",
-            "retryable": True,
-        }, status=409)
+        return j(handler, agent_runtime_stale_payload(e), status=409)
     except api_config.AmbiguousCustomProviderError as e:
         # A custom-provider slug collision is a user-fixable misconfiguration,
         # not a transient summary failure. Return 400 with the actionable rename

docs/troubleshooting.md

@@ -179,13 +179,23 @@ turn in the exhausted session instead of being blocked with recovery guidance.
 
 ## "Hermes Agent was updated while Hermes WebUI was running"
 
-**Symptom.** An action that uses the in-process Agent runtime stops with a message telling you to restart Hermes WebUI. This can happen after `hermes update`, a Git checkout/pull in the Agent source tree, or another tool updates Hermes Agent without restarting the already-running WebUI backend.
+**Symptom.** An action that uses the in-process Agent runtime stops with a message telling you to restart Hermes WebUI manually. This can happen after `hermes update`, a Git checkout/pull in the Agent source tree, or another tool updates Hermes Agent without restarting the already-running WebUI backend.
 
-**Why.** WebUI currently imports `run_agent.AIAgent` into its long-lived Python process. Python keeps imported modules in memory. Continuing after a known Agent Git revision changes could combine cached modules from the old revision with source read from the new revision, producing misleading `ImportError`s or inconsistent runtime state. For local Agent-backed chat, WebUI therefore returns a retryable `409 agent_runtime_stale` before claiming or mutating session state instead of attempting a partial in-process reload. Gateway-backed chat runs in the gateway process and is not blocked by this WebUI-local check. Non-Git Agent installs preserve their existing behavior because there is no revision identity to compare.
+**Why.** WebUI imports `run_agent.AIAgent` into its long-lived Python process. Continuing after a known Agent Git revision changes could combine cached modules from the old revision with source read from the new revision. Local Agent-backed actions return a retryable `409 agent_runtime_stale` with `restart_scheduled: false` before accepting a new turn. Gateway- and runner-owned chat keep their existing runtime ownership. Non-Git Agent installs preserve their existing behavior because there is no revision identity to compare; losing a previously known revision remains fail-closed.
 
-**Diagnostic.** Compare the running WebUI process start time with the Agent checkout revision and recent update history. If the Agent was updated after WebUI started, restart WebUI before investigating individual missing-symbol errors.
+**Diagnostic.** The stale-runtime response includes `agent_update_state`, also preserved in asynchronous compression error status:
 
-**Fix.** Restart using the same launch method that started WebUI:
+| Value | Observation |
+| --- | --- |
+| `active` | A recent Agent update marker names a live PID. |
+| `incomplete` | An Agent recovery marker exists in the loaded checkout or configured venv installation. |
+| `stale` | The update marker names a dead PID or is older than the diagnostic age limit. |
+| `unknown` | Marker contents, PID liveness, or recovery-marker presence cannot be read or classified. |
+| `unverified` | No active or recovery marker was found. Update completion and environment health remain unverified. |
+
+These are observations, not success receipts. Hermes Agent removes `.hermes-update-in-progress` on failed and interrupted exits too. A missing or stale marker, or a readable Git revision, does not prove a completed update or a healthy environment. WebUI only reads these markers; it does not remove or repair them.
+
+**Fix.** Check the Agent updater's outcome and resolve any failed or incomplete Agent update first. Once the Agent checkout and environment are healthy and no updater is running, restart WebUI using the same launch method that started it:
 
 ```bash
 ./ctl.sh restart
@@ -193,9 +203,11 @@ turn in the exhausted session instead of being blocked with recovery guidance.
 systemctl --user restart hermes-webui.service
 ```
 
-If you launched `python3 bootstrap.py` in the foreground, stop it with Ctrl-C and start it again. Restarting the whole computer or WSL is not required when restarting the WebUI backend succeeds.
+For a foreground `python3 bootstrap.py`, stop it with Ctrl-C and start it again. Restarting the whole computer or WSL is not required when restarting the WebUI backend succeeds. Retry the action after restarting the backend; refreshing the browser alone does not replace its imported Agent modules.
 
-**When to file a bug.** File a WebUI bug if the restart-required message appears even though the Agent revision did not change, or if a clean WebUI restart still produces the same import error. Include the WebUI launch method, WebUI revision, Agent revision, and the sanitized error text.
+**Automatic restart prerequisite.** Revision mismatch does not schedule a WebUI restart. Safe automation requires an Agent-owned terminal success receipt bound to the exact update transaction, final revision, and healthy environment, plus an Agent-owned atomic handoff or lease that excludes new mutations across process replacement (or an Agent updater that performs the restart itself). No such public contract is verified for this integration. Repeated readiness checks followed by `os.execv()` leave a race; WebUI's own update lock does not exclude an external Agent updater. Explicit updates initiated through WebUI retain their existing behavior and are outside this revision-mismatch guard.
+
+**When to file a bug.** File a WebUI bug if the restart-required message appears even though the Agent revision did not change or become unreadable, or if a clean WebUI restart still produces the same import error. Include the launch method, WebUI and Agent revisions, the marker diagnostic, and sanitized error text.
 
 ---
 

tests/test_agent_runtime_revision_guard.py

@@ -6,6 +6,7 @@ import os
 from pathlib import Path
 import subprocess
 import sys
+import time
 import types
 
 import pytest
@@ -25,6 +26,195 @@ def _git(repo: Path, *args: str) -> str:
     return result.stdout.strip()
 
 
+@pytest.fixture
+def changed_agent_checkout(monkeypatch, tmp_path):
+    """Exercise the real revision guard and scheduler without replacing a process."""
+    from api import agent_runtime, routes, updates
+
+    agent_root = tmp_path / "agent"
+    agent_root.mkdir()
+    module_path = agent_root / "run_agent.py"
+    module_path.write_text("class AIAgent: pass\n", encoding="utf-8")
+    _git(agent_root, "init", "-q")
+    _git(agent_root, "add", "run_agent.py")
+    _git(agent_root, "commit", "-qm", "loaded")
+    monkeypatch.setattr(agent_runtime, "_AGENT_SOURCE_DIR", agent_root)
+    monkeypatch.setattr(agent_runtime, "_AGENT_MODULE_PATH", module_path)
+    monkeypatch.setattr(agent_runtime, "_AGENT_REVISION", _git(agent_root, "rev-parse", "HEAD"))
+    module_path.write_text("class AIAgent: revision = 'changed'\n", encoding="utf-8")
+    _git(agent_root, "commit", "-qam", "changed")
+
+    hermes_home = tmp_path / "home"
+    hermes_home.mkdir()
+    venv_root = tmp_path / "separate-install"
+    (venv_root / "venv" / "bin").mkdir(parents=True)
+    monkeypatch.setattr(agent_runtime, "_HERMES_HOME", hermes_home)
+    monkeypatch.setattr(agent_runtime, "_AGENT_PYTHON", venv_root / "venv" / "bin" / "python")
+    workers = []
+
+    class CapturedThread:
+        def __init__(self, *, target, daemon):
+            self.target = target
+
+        def start(self):
+            workers.append(self.target)
+
+    monkeypatch.setattr(updates, "threading", types.SimpleNamespace(Thread=CapturedThread))
+    monkeypatch.setattr(routes, "get_config", lambda: {})
+    monkeypatch.setattr(routes, "webui_gateway_chat_enabled", lambda _cfg: False)
+    monkeypatch.setattr(
+        routes, "j",
+        lambda _handler, payload, status=200: {"status": status, "payload": payload},
+    )
+
+    def no_session_mutation(*_args, **_kwargs):
+        pytest.fail("stale runtime reached session materialization")
+
+    monkeypatch.setattr(routes, "_get_or_materialize_session", no_session_mutation)
+    return types.SimpleNamespace(
+        root=agent_root, home=hermes_home, venv_root=venv_root, workers=workers,
+        request=lambda: routes._handle_chat_start(object(), {"session_id": "stale-session"}),
+    )
+
+
+@pytest.mark.parametrize(
+    ("marker_state", "diagnostic"),
+    [
+        ("absent", "unverified"),
+        ("failed-exit", "unverified"),
+        ("interrupted-exit", "unverified"),
+        ("active", "active"),
+        ("dead", "stale"),
+        ("over-age", "stale"),
+        ("malformed", "unknown"),
+        ("invalid-encoding", "unknown"),
+        ("future", "unknown"),
+        ("nonfinite", "unknown"),
+        ("oversized-pid", "unknown"),
+        ("dangling", "unknown"),
+        ("unreadable", "unknown"),
+        ("update-incomplete", "incomplete"),
+        ("lazy-refresh-incomplete", "incomplete"),
+        ("separate-venv-incomplete", "incomplete"),
+        ("symlink-venv-incomplete", "incomplete"),
+    ],
+)
+def test_unverified_update_keeps_manual_409_without_restart(
+    changed_agent_checkout, marker_state, diagnostic,
+):
+    """Readable HEAD and lifecycle markers never authorize process replacement."""
+    checkout = changed_agent_checkout
+    marker = checkout.home / ".hermes-update-in-progress"
+    if marker_state in {"active", "failed-exit", "interrupted-exit"}:
+        marker.write_text(f"{os.getpid()}\n{time.time()}\n", encoding="utf-8")
+        if marker_state != "active":
+            marker.unlink()  # Agent removes its marker on failed/interrupted exits.
+    elif marker_state == "dead":
+        process = subprocess.Popen([sys.executable, "-c", "pass"])
+        process.wait(timeout=10)
+        marker.write_text(f"{process.pid}\n{time.time()}\n", encoding="utf-8")
+    elif marker_state == "over-age":
+        marker.write_text(f"{os.getpid()}\n{time.time() - 86400}\n", encoding="utf-8")
+    elif marker_state == "malformed":
+        marker.write_text("not-a-pid\nnot-a-time\n", encoding="utf-8")
+    elif marker_state == "invalid-encoding":
+        marker.write_bytes(b"\xff\xfe")
+    elif marker_state == "oversized-pid":
+        marker.write_text(f"{2 ** 100}\n{time.time()}\n", encoding="utf-8")
+    elif marker_state in {"future", "nonfinite"}:
+        timestamp = time.time() + 86400 if marker_state == "future" else "nan"
+        marker.write_text(f"{os.getpid()}\n{timestamp}\n", encoding="utf-8")
+    elif marker_state == "dangling":
+        marker.symlink_to(checkout.home / "missing")
+    elif marker_state == "unreadable":
+        marker.mkdir()
+    elif marker_state in {"update-incomplete", "lazy-refresh-incomplete"}:
+        (checkout.root / f".{marker_state}").touch()
+    elif marker_state == "separate-venv-incomplete":
+        (checkout.venv_root / ".update-incomplete").touch()
+    elif marker_state == "symlink-venv-incomplete":
+        (checkout.venv_root / "venv/bin/python").symlink_to(sys.executable)
+        (checkout.venv_root / ".update-incomplete").touch()
+
+    response = checkout.request()
+
+    assert checkout.workers == [], "revision mismatch scheduled an automatic restart"
+    assert response["status"] == 409
+    payload = response["payload"]
+    assert payload["type"] == "agent_runtime_stale"
+    assert payload["retryable"] is True
+    assert payload["restart_scheduled"] is False
+    assert payload["agent_update_state"] == diagnostic
+    assert "Restart Hermes WebUI manually" in payload["error"]
+    assert "success" not in payload["error"].lower()
+
+
+def test_final_read_cannot_authorize_restart_without_atomic_handoff(
+    monkeypatch, changed_agent_checkout,
+):
+    """An updater can acquire its marker after a read; no worker may be queued."""
+    from api import agent_runtime
+
+    checkout = changed_agent_checkout
+    read_revision = agent_runtime._read_agent_revision
+    marker = checkout.home / ".hermes-update-in-progress"
+    revision_reads = []
+
+    def read_then_start_update(*args, **kwargs):
+        revision = read_revision(*args, **kwargs)
+        revision_reads.append(revision)
+        marker.write_text(f"{os.getpid()}\n{time.time()}\n", encoding="utf-8")
+        return revision
+
+    monkeypatch.setattr(agent_runtime, "_read_agent_revision", read_then_start_update)
+    response = checkout.request()
+
+    assert checkout.workers == [], "an unprotected read authorized automatic restart"
+    assert revision_reads == [_git(checkout.root, "rev-parse", "HEAD")]
+    assert marker.is_file()
+    assert response["status"] == 409
+    assert response["payload"]["restart_scheduled"] is False
+
+
+@pytest.mark.parametrize("response_kind", ["http-error", "raised-error"])
+def test_async_compression_preserves_manual_restart_diagnostics(
+    monkeypatch, changed_agent_checkout, response_kind,
+):
+    """The real guard reaches the job's terminal error and repeated status reads."""
+    from api import agent_runtime, helpers, routes
+
+    checkout = changed_agent_checkout
+    sid = "stale-compression"
+    (checkout.root / ".update-incomplete").touch()
+    monkeypatch.setattr(routes, "get_session", lambda _sid: None)
+    monkeypatch.setattr(routes, "_MANUAL_COMPRESSION_JOBS", {
+        sid: {"session_id": sid, "status": "running", "updated_at": time.time()},
+    })
+
+    def compress(handler, _body):
+        try:
+            agent_runtime.ensure_agent_runtime_current()
+        except agent_runtime.AgentRuntimeChangedError as exc:
+            if response_kind == "raised-error":
+                raise
+            helpers.j(handler, agent_runtime.agent_runtime_stale_payload(exc), status=409)
+
+    monkeypatch.setattr(routes, "_handle_session_compress", compress)
+    routes._run_manual_compression_job(sid, {"session_id": sid})
+
+    for _ in range(2):
+        response = routes._handle_session_compress_status(object(), sid)
+        payload = response["payload"]
+        assert payload["status"] == "error"
+        assert payload["error_status"] == 409
+        assert payload["type"] == "agent_runtime_stale"
+        assert payload["retryable"] is True
+        assert payload["restart_scheduled"] is False
+        assert payload["agent_update_state"] == "incomplete"
+        assert "Restart Hermes WebUI manually" in payload["error"]
+    assert checkout.workers == []
+
+
 def test_loaded_agent_runtime_fails_closed_after_source_revision_changes(tmp_path: Path):
     agent_dir = tmp_path / "hermes-agent"
     agent_dir.mkdir()
@@ -85,14 +275,14 @@ try:
 except RuntimeError as exc:
     message = str(exc)
     assert "Hermes Agent was updated" in message
-    assert "Restart Hermes WebUI" in message
+    assert "Restart Hermes WebUI manually" in message
 else:
     raise AssertionError("stale in-process AIAgent was reused after its source revision changed")
 
 try:
     agent_runtime.require_ai_agent_class()
 except agent_runtime.AgentRuntimeChangedError as exc:
-    assert "Restart Hermes WebUI" in str(exc)
+    assert "Restart Hermes WebUI manually" in str(exc)
 else:
     raise AssertionError("unguarded AIAgent import was allowed after its source revision changed")
 """.strip()
@@ -121,6 +311,105 @@ else:
     assert result.returncode == 0, result.stdout + result.stderr
 
 
+def test_read_live_agent_update_rejects_fifo_marker_without_hanging(tmp_path: Path):
+    """A FIFO in the marker path must classify ``unknown`` fast, not block.
+
+    Regression guard: the marker read once used ``Path.read_text()``, which
+    blocks forever on a FIFO (no writer) and would wedge the stale-runtime
+    request path. The hardened read opens O_NONBLOCK|O_NOFOLLOW and fstat-checks
+    for a small regular file, so a FIFO is rejected immediately.
+    """
+    from api import agent_runtime
+
+    fifo = tmp_path / "fifo-marker"
+    os.mkfifo(fifo)
+    start = time.time()
+    result = agent_runtime._read_live_agent_update(fifo)
+    elapsed = time.time() - start
+    assert result == "unknown"
+    assert elapsed < 1.0, f"marker read blocked on FIFO for {elapsed:.2f}s"
+
+
+def test_read_live_agent_update_rejects_oversized_marker(tmp_path: Path):
+    """An oversized regular marker must classify ``unknown``, never OOM-read.
+
+    Regression guard against unbounded ``read_text()``: even though the file
+    starts with a valid PID/timestamp, its size exceeds the cap so it is
+    rejected rather than read whole.
+    """
+    from api import agent_runtime
+
+    big = tmp_path / "big-marker"
+    big.write_bytes(
+        f"{os.getpid()}\n{time.time()}\n".encode("utf-8")
+        + b"x" * (agent_runtime._AGENT_UPDATE_MARKER_MAX_BYTES + 1024)
+    )
+    assert agent_runtime._read_live_agent_update(big) == "unknown"
+
+
+def test_read_live_agent_update_does_not_follow_symlink_marker(tmp_path: Path):
+    """A symlinked marker must classify ``unknown`` (O_NOFOLLOW), not be read
+    through to its target."""
+    from api import agent_runtime
+
+    if not getattr(os, "O_NOFOLLOW", 0):
+        pytest.skip("O_NOFOLLOW unavailable on this platform")
+
+    target = tmp_path / "real-marker"
+    target.write_text(f"{os.getpid()}\n{time.time()}\n", encoding="utf-8")
+    link = tmp_path / "link-marker"
+    link.symlink_to(target)
+    assert agent_runtime._read_live_agent_update(link) == "unknown"
+
+
+def test_read_live_agent_update_still_classifies_valid_markers(tmp_path: Path):
+    """The hardening must not regress the happy path: a small regular marker
+    with a live PID still reads as ``active`` and a stale one as ``stale``."""
+    from api import agent_runtime
+
+    active = tmp_path / "active-marker"
+    active.write_text(f"{os.getpid()}\n{time.time()}\n", encoding="utf-8")
+    assert agent_runtime._read_live_agent_update(active) == "active"
+
+    stale = tmp_path / "stale-marker"
+    stale.write_text(
+        f"{os.getpid()}\n{time.time() - agent_runtime._AGENT_UPDATE_MAX_AGE_SECONDS - 60}\n",
+        encoding="utf-8",
+    )
+    assert agent_runtime._read_live_agent_update(stale) == "stale"
+
+    assert agent_runtime._read_live_agent_update(tmp_path / "absent") == "absent"
+
+
+def test_read_live_agent_update_windows_fallback_never_opens_marker(
+    tmp_path: Path, monkeypatch
+):
+    """When the atomic open flags are unavailable (e.g. native Windows), the
+    read must not call os.open (which would raise on a missing O_NONBLOCK) and
+    must fail closed: missing -> absent, anything present -> unknown.
+
+    Regression guard for the portability CORE: os.O_NONBLOCK is Unix-only, so
+    the fast path must be gated on both flags being present.
+    """
+    from api import agent_runtime
+
+    monkeypatch.setattr(agent_runtime, "_MARKER_SAFE_OPEN_AVAILABLE", False)
+
+    def _boom(*_a, **_k):  # os.open must never be reached on the fallback path
+        raise AssertionError("os.open called despite unavailable atomic flags")
+
+    monkeypatch.setattr(agent_runtime.os, "open", _boom)
+
+    # A present, otherwise-valid marker is unverifiable without the safe open →
+    # classified unknown (never active/absent), and does not raise.
+    present = tmp_path / "present-marker"
+    present.write_text(f"{os.getpid()}\n{time.time()}\n", encoding="utf-8")
+    assert agent_runtime._read_live_agent_update(present) == "unknown"
+
+    # A genuinely missing marker is still absent.
+    assert agent_runtime._read_live_agent_update(tmp_path / "missing") == "absent"
+
+
 def test_initial_non_git_source_preserves_supported_runtime(monkeypatch):
     """Non-Git installs cannot be compared, so they preserve existing behavior."""
     from api import agent_runtime
@@ -247,6 +536,83 @@ def test_known_revision_becoming_unreadable_fails_closed(monkeypatch):
         agent_runtime.ensure_agent_runtime_current()
 
 
+def test_live_agent_update_marker_reports_active(monkeypatch, tmp_path):
+    """A fresh marker owned by a live PID means the Agent update is active."""
+    from api import agent_runtime
+
+    hermes_home = tmp_path / "hermes-home"
+    agent_root = tmp_path / "hermes-agent"
+    hermes_home.mkdir()
+    agent_root.mkdir()
+    (hermes_home / ".hermes-update-in-progress").write_text(
+        f"{os.getpid()}\n{time.time()}\n",
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(agent_runtime, "_HERMES_HOME", hermes_home)
+    monkeypatch.setattr(agent_runtime, "_AGENT_SOURCE_DIR", agent_root)
+    monkeypatch.setattr(agent_runtime, "_AGENT_PYTHON", None)
+
+    assert agent_runtime._agent_update_transaction_state() == "active"
+
+
+def test_dead_or_over_age_agent_update_marker_is_stale(monkeypatch, tmp_path):
+    """A dead or over-age owner does not prove update completion."""
+    from api import agent_runtime
+
+    hermes_home = tmp_path / "hermes-home"
+    agent_root = tmp_path / "hermes-agent"
+    hermes_home.mkdir()
+    agent_root.mkdir()
+    marker = hermes_home / ".hermes-update-in-progress"
+    monkeypatch.setattr(agent_runtime, "_HERMES_HOME", hermes_home)
+    monkeypatch.setattr(agent_runtime, "_AGENT_SOURCE_DIR", agent_root)
+    monkeypatch.setattr(agent_runtime, "_AGENT_PYTHON", None)
+
+    marker.write_text(f"99999999\n{time.time()}\n", encoding="utf-8")
+    monkeypatch.setattr(agent_runtime, "_pid_is_alive", lambda _pid: False)
+    assert agent_runtime._agent_update_transaction_state() == "stale"
+
+    marker.write_text(
+        f"{os.getpid()}\n{time.time() - agent_runtime._AGENT_UPDATE_MAX_AGE_SECONDS - 1}\n",
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(agent_runtime, "_pid_is_alive", lambda _pid: True)
+    assert agent_runtime._agent_update_transaction_state() == "stale"
+
+
+@pytest.mark.parametrize(
+    "marker_name",
+    [".update-incomplete", ".lazy-refresh-incomplete"],
+)
+def test_agent_recovery_markers_block_automatic_restart(
+    monkeypatch, tmp_path, marker_name
+):
+    """Agent recovery markers report an incomplete environment."""
+    from api import agent_runtime
+
+    hermes_home = tmp_path / "hermes-home"
+    agent_root = tmp_path / "hermes-agent"
+    hermes_home.mkdir()
+    agent_root.mkdir()
+    (agent_root / marker_name).write_text("incomplete\n", encoding="utf-8")
+    monkeypatch.setattr(agent_runtime, "_HERMES_HOME", hermes_home)
+    monkeypatch.setattr(agent_runtime, "_AGENT_SOURCE_DIR", agent_root)
+    monkeypatch.setattr(agent_runtime, "_AGENT_PYTHON", None)
+
+    assert agent_runtime._agent_update_transaction_state() == "incomplete"
+
+
+def test_unreadable_agent_update_state_fails_closed(monkeypatch):
+    """An unreadable marker is unknown, never proof that restart is safe."""
+    from api import agent_runtime
+
+    class UnreadableMarker:
+        def read_text(self, **_kwargs):
+            raise PermissionError("denied")
+
+    assert agent_runtime._read_live_agent_update(UnreadableMarker()) == "unknown"
+
+
 def test_import_recapture_cannot_downgrade_known_revision(monkeypatch, tmp_path: Path):
     """A second unreadable revision read must not erase a known identity."""
     from api import agent_runtime
@@ -352,6 +718,7 @@ def test_runner_flag_does_not_bypass_webui_owned_hidden_turns(monkeypatch):
         "error": "restart required",
         "type": "agent_runtime_stale",
         "retryable": True,
+        "restart_scheduled": False,
     }
 
 
@@ -385,6 +752,7 @@ def test_chat_start_rejects_stale_runtime_before_session_materialization(monkeyp
             "error": "restart required",
             "type": "agent_runtime_stale",
             "retryable": True,
+            "restart_scheduled": False,
         },
     }
 
@@ -583,6 +951,7 @@ def test_git_commit_message_stale_runtime_returns_typed_409(
             "error": "restart required",
             "type": "agent_runtime_stale",
             "retryable": True,
+            "restart_scheduled": False,
         },
     }
 
@@ -688,6 +1057,7 @@ def test_server_side_turn_rejects_stale_runtime_before_session_acceptance(monkey
             "error": "restart required",
             "type": "agent_runtime_stale",
             "retryable": True,
+            "restart_scheduled": False,
         },
     )
     monkeypatch.setattr(
@@ -704,5 +1074,6 @@ def test_server_side_turn_rejects_stale_runtime_before_session_acceptance(monkey
         "error": "restart required",
         "type": "agent_runtime_stale",
         "retryable": True,
+        "restart_scheduled": False,
         "_status": 409,
     }

tests/test_issue1013_handoff_dock.py

@@ -322,6 +322,7 @@ def test_stale_runtime_handoff_summary_returns_typed_409_without_persisting(monk
             "error": "restart required",
             "type": "agent_runtime_stale",
             "retryable": True,
+            "restart_scheduled": False,
         },
     }
 

tests/test_sprint46.py

@@ -155,6 +155,7 @@ def test_session_compress_stale_runtime_returns_typed_409_before_mutation(
         "error": "restart required",
         "type": "agent_runtime_stale",
         "retryable": True,
+        "restart_scheduled": False,
     }
     loaded_after = Session.load(sid)
     assert loaded_after is not None
@@ -187,6 +188,7 @@ def test_session_compress_start_stale_runtime_returns_409_before_job_creation(
         "error": "restart required",
         "type": "agent_runtime_stale",
         "retryable": True,
+        "restart_scheduled": False,
     }
     with routes._MANUAL_COMPRESSION_JOBS_LOCK:
         assert sid not in routes._MANUAL_COMPRESSION_JOBS