jinghui/hermes-webui
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(wakeup): route async-delegation completions by origin + durable-claim delivery (#6283)

Browse Source 
Route async-delegation completions by the immutable origin_ui_session_id (exact origin tab) and deliver them through a durable claim/complete/release lifecycle so they arrive exactly once, restart-safe, on both the background wakeup and next-turn drain paths. Combines #6185 (@carlotestor) + #6159 (@sysophelper-droid); supersedes #6002/#6225.

Co-authored-by: carlotestor <carlotestor@users.noreply.github.com>
Co-authored-by: sysophelper-droid <sysophelper-droid@users.noreply.github.com>
This commit is contained in:
nesquena-hermescarlotestorsysophelper-droid
 authored and GitHub committed 2026-07-19 02:26:55 +00:00
1 parent c67fd2dd27
commit 6ad84d7dd3
7 files changed
+2081 -71
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

api/background_process.py
+295 -39
View File
@@ -48,6 +48,16 @@ import time
import uuid
from typing import Any, Optional
from api.process_event_utils import (
ASYNC_DELIVERY_ROUTING_RETRY_SECONDS,
claim_async_delegation_delivery,
complete_async_delegation_delivery,
completion_delivery_id,
release_async_delegation_delivery,
requeue_async_delegation_event,
schedule_async_delegation_claim_retry,
)
logger = logging.getLogger(__name__)
_DRAIN_THREAD: Optional[threading.Thread] = None
@@ -541,9 +551,9 @@ def _build_payload(evt: dict, session_id: str) -> dict:
``(session_id, event_id)`` to dedupe across reconnects.
"""
# ProcessRegistry completion events use the field name ``session_id`` for
# the process id. Alias it locally before exposing it as payload ``task_id``
# to avoid confusing that wire-format name with the WebUI session id.
process_id = str(evt.get("session_id") or "")
# the process id. Async delegation completions carry ``delegation_id``
# instead; expose either stable delivery id as payload ``task_id``.
process_id = completion_delivery_id(evt)
payload: dict[str, Any] = {
"session_id": str(session_id),
"task_id": process_id,
@@ -822,7 +832,12 @@ def _mark_registry_completion_consumed(process_id: str) -> None:
# registry, pre-Option-1 spawns) it is a PURE PASS-THROUGH — it never
# suppresses a legitimate Option Z wakeup on uncertainty (Option Z must keep
# working).
_ENV_IMMUNE_OWNER_ATTRS = ("spawn_session_id", "owner_session_id", "turn_session_id")
_ENV_IMMUNE_OWNER_ATTRS = (
"origin_ui_session_id", # modern hermes-agent exact browser-tab return address
"spawn_session_id",
"owner_session_id",
"turn_session_id",
)
def _env_immune_spawn_owner(proc_session) -> str:
@@ -872,6 +887,211 @@ def _resolve_wakeup_target(
return owner
def _requeue_async_delegation_event(
process_registry,
evt: dict,
*,
claim=None,
delay: float = 0.5,
) -> bool:
"""Retry without enqueueing new work once drain shutdown has started."""
completion_queue = getattr(process_registry, "completion_queue", None)
return requeue_async_delegation_event(
evt,
completion_queue,
delay=delay,
stop_event=_DRAIN_STOP,
durable=(bool(getattr(claim, "durable", False)) if claim is not None else None),
)
def _retry_unmapped_async_delegation_event(process_registry, evt: dict) -> None:
"""Retry durable routing, or one bounded best-effort legacy routing pass."""
completion_queue = getattr(process_registry, "completion_queue", None)
if schedule_async_delegation_claim_retry(
evt,
completion_queue,
delay=ASYNC_DELIVERY_ROUTING_RETRY_SECONDS,
):
return
if evt.get("_webui_routing_retry_attempted"):
return
retry_evt = dict(evt)
retry_evt["_webui_routing_retry_attempted"] = True
_requeue_async_delegation_event(
process_registry,
retry_evt,
delay=ASYNC_DELIVERY_ROUTING_RETRY_SECONDS,
)
def _record_async_delegation_accepted(
evt: dict,
*,
session_id: str,
claim,
) -> None:
"""ACK durable delivery and publish live-view state after turn acceptance."""
complete_async_delegation_delivery(evt, claim)
payload = _build_payload(evt, session_id)
try:
_emit_bg_task_complete_events_coalesced(session_id, payload)
except Exception:
logger.debug(
"async delegation live-view emit failed for session %s",
session_id,
exc_info=True,
)
def _start_async_delegation_wakeup_turn(
session_id: str,
wakeup_prompt: str,
*,
delegation_id: str,
evt: dict,
claim,
process_registry,
) -> None:
"""Start one autonomous delegation turn and ACK only after acceptance."""
def _runner() -> None:
try:
from api.routes import start_session_turn
resp = start_session_turn(
session_id,
wakeup_prompt,
source="process_wakeup",
)
raw_status = (resp or {}).get("_status")
if raw_status is None:
status = 200 if (resp or {}).get("stream_id") else 500
else:
status = int(raw_status)
if 200 <= status < 300:
_record_async_delegation_accepted(
evt,
session_id=session_id,
claim=claim,
)
logger.info(
"async delegation wakeup turn accepted for session %s "
"(stream_id=%s)",
session_id,
(resp or {}).get("stream_id"),
)
return
release_async_delegation_delivery(evt, claim)
_requeue_async_delegation_event(process_registry, evt, claim=claim)
if status == 409 and (resp or {}).get("error") == "process_wakeup_paused":
logger.info(
"async delegation wakeup paused for session %s; delivery remains retryable",
session_id,
)
else:
logger.debug(
"async delegation wakeup not accepted for session %s: "
"status=%s err=%r; requeued",
session_id,
status,
(resp or {}).get("error"),
)
except Exception:
release_async_delegation_delivery(evt, claim)
_requeue_async_delegation_event(process_registry, evt, claim=claim)
logger.warning(
"async delegation wakeup turn failed for session %s; requeued",
session_id,
exc_info=True,
)
threading.Thread(
target=_runner,
name=f"hermes-webui-delegation-wakeup-{str(session_id)[:8]}",
daemon=True,
).start()
def _process_async_delegation_event(
evt: dict,
*,
session_id: str,
delegation_id: str,
process_registry,
) -> None:
"""Claim and route one async completion without private registry markers."""
try:
claim = claim_async_delegation_delivery(evt, "webui-background")
except Exception:
_requeue_async_delegation_event(process_registry, evt)
return
if claim is None:
completion_queue = getattr(process_registry, "completion_queue", None)
schedule_async_delegation_claim_retry(evt, completion_queue)
return
try:
wakeup_prompt_raw = format_wakeup_prompt(evt)
wakeup_prompt = wakeup_prompt_raw.strip() if wakeup_prompt_raw else ""
if not wakeup_prompt:
raise RuntimeError("async delegation completion could not be formatted")
# Do not persist async results in the process-local deferred list. If a
# foreground turn owns the session, release the durable claim and retry
# from the shared queue; the core record therefore remains restart-safe.
if _session_has_active_turn(session_id):
release_async_delegation_delivery(evt, claim)
_requeue_async_delegation_event(process_registry, evt, claim=claim)
return
_start_async_delegation_wakeup_turn(
session_id,
wakeup_prompt,
delegation_id=delegation_id,
evt=evt,
claim=claim,
process_registry=process_registry,
)
except Exception:
release_async_delegation_delivery(evt, claim)
_requeue_async_delegation_event(process_registry, evt, claim=claim)
logger.warning(
"server-side async delegation dispatch failed for session %s",
session_id,
exc_info=True,
)
def _resolve_completion_target(
*,
session_key_resolved_sid: str,
origin_ui_session_id: str,
) -> str:
"""Return the WebUI session that owns a detached completion event.
Modern Hermes Agent events carry ``origin_ui_session_id`` as an exact,
immutable return address captured from the commissioning browser turn.
It is authoritative over the mutable/legacy session-key index. Older
Agent events omit it and retain the existing session-key fallback.
"""
resolved = str(session_key_resolved_sid or "")
owner = str(origin_ui_session_id or "")
if not owner:
return resolved
if resolved and resolved != owner:
logger.error(
"cross-session completion route BLOCKED: session_key resolved to %r "
"but exact origin_ui_session_id is %r; routing to the exact owner",
resolved,
owner,
)
return owner
def _process_one(evt: dict) -> None:
"""Route a single completion_queue event to the matching WebUI session."""
from api import config as _cfg
@@ -887,55 +1107,59 @@ def _process_one(evt: dict) -> None:
except Exception:
_process_registry = None
process_id = str(evt.get("session_id") or "")
process_id = completion_delivery_id(evt)
session_key = str(evt.get("session_key") or "")
origin_ui_session_id = str(evt.get("origin_ui_session_id") or "")
# Root-cause fix (t_0f447014): the notify_on_complete completion event
# enqueued by ProcessRegistry._move_to_finished() carries NO "session_key"
# field — only the watch_match enqueue includes one. Without it the old
# `evt.get("session_key") or process_id` fell back to the process id
# ("proc_xxxx"), which is never a PROCESS_SESSION_INDEX key (only
# webui_session_id -> webui_session_id is registered at chat-start), so
# every wakeup was silently dropped here and the frontend never POSTed an
# ack. Recover the spawn-time session_key from the process registry's
# ProcessSession: the terminal tool captured it synchronously at spawn
# (while the turn's env was active), so it survives the turn-end env
# restore and is the WebUI session_id for WebUI-spawned processes.
if not session_key and process_id:
# enqueued by ProcessRegistry._move_to_finished() historically carried NO
# "session_key" field — only the watch_match enqueue included one. Without
# it the old `evt.get("session_key") or process_id` fell back to the
# process id ("proc_xxxx"), which is never a PROCESS_SESSION_INDEX key.
# Recover spawn-time routing metadata from the process registry.
if process_id and (not session_key or not origin_ui_session_id):
try:
if _process_registry is not None:
_ps = _process_registry.get(process_id)
if _ps is not None and getattr(_ps, "session_key", ""):
session_key = str(_ps.session_key)
if _ps is not None:
if not session_key and getattr(_ps, "session_key", ""):
session_key = str(_ps.session_key)
if not origin_ui_session_id:
origin_ui_session_id = (
str(getattr(_ps, "origin_ui_session_id", "") or "")
or str(getattr(_ps, "spawn_session_id", "") or "")
)
except Exception:
logger.debug(
"session_key recovery from process registry failed for %r",
"session ownership recovery from process registry failed for %r",
process_id,
exc_info=True,
)
if not session_key:
if not session_key and not origin_ui_session_id:
logger.debug(
"process_complete drop: no recoverable session_key for process_id=%r",
"process_complete drop: no recoverable session_key or exact UI owner "
"for process_id=%r",
process_id,
)
if evt.get("type") == "async_delegation":
_retry_unmapped_async_delegation_event(_process_registry, evt)
return
with _cfg.PROCESS_SESSION_INDEX_LOCK:
session_id = _cfg.PROCESS_SESSION_INDEX.get(session_key)
if not session_id:
session_id = ""
if session_key:
with _cfg.PROCESS_SESSION_INDEX_LOCK:
session_id = _cfg.PROCESS_SESSION_INDEX.get(session_key) or ""
if not session_id and not origin_ui_session_id:
# No mapping — could be a cron/gateway process that uses the same
# registry but a non-WebUI session_key. Ignore.
# registry but a non-WebUI session_key. Durable delegation events stay
# pending and are retried because their WebUI ownership mapping can be
# registered shortly after process restore.
logger.debug("process_complete drop: no session mapping for key=%r", session_key)
if evt.get("type") == "async_delegation":
_retry_unmapped_async_delegation_event(_process_registry, evt)
return
# ── xsession wakeup misroute defense-in-depth (Option 3) ──────────────
# session_id above came from PROCESS_SESSION_INDEX.get(session_key), and
# session_key was captured by the terminal tool from the (historically
# racy) process-global env at spawn. Option 1 binds the per-turn identity
# to a contextvar so that capture is no longer racy — but as an INDEPENDENT
# safety net, cross-check the resolved target against the env-immune
# spawn owner (when the core ProcessSession exposes one). On a positive
# mismatch this re-routes the wakeup (and the live-view emit + dedupe
# markers below) to the TRUE owner instead of waking the wrong session.
# Pure pass-through when no env-immune owner is available (today's core,
# cron/CLI procs, pre-Option-1 spawns) — never suppresses a valid wakeup.
# First retain the process-registry spawn-owner cross-check for legacy
# terminal events. Then apply origin_ui_session_id as the final authority;
# that exact owner is shared by process and async-delegation completions.
try:
_ps_xs = _process_registry.get(process_id) if (_process_registry is not None and process_id) else None
except Exception:
@@ -945,6 +1169,35 @@ def _process_one(evt: dict) -> None:
session_key_resolved_sid=session_id,
proc_session=_ps_xs,
)
# origin_ui_session_id is the exact, immutable return address; it is the
# FINAL routing authority over the (mutable) session-key/Option-3 result.
session_id = _resolve_completion_target(
session_key_resolved_sid=session_id,
origin_ui_session_id=origin_ui_session_id,
)
if not session_id:
logger.debug("process_complete drop: completion target resolved empty")
# An async delegation event that resolves empty here must NOT silently
# return: no durable claim was taken, so the core row stays pending and
# the restart-restore sweep would re-deliver it forever. Route it into
# the bounded retry instead (arms a durable retry / one best-effort
# legacy requeue), so it stays retryable without a spurious ACK.
if evt.get("type") == "async_delegation":
_retry_unmapped_async_delegation_event(_process_registry, evt)
return
# ── THE SEAM: async delegations take the durable-claim delivery path,
# routed to the origin-resolved session. The claim/complete/release
# lifecycle (keyed on the immutable delegation_id) is the sole dedupe +
# restart-safety authority for async events, so they early-return before
# the terminal-process idempotency/emit/Option-Z machinery below. ──
if evt.get("type") == "async_delegation":
_process_async_delegation_event(
evt,
session_id=session_id,
delegation_id=process_id,
process_registry=_process_registry,
)
return
# ── Idempotency vs the REAL merged upstream #2279 (shared dedupe key) ──
# The real merged #2279 next-turn drain
# (api/streaming._drain_webui_process_notifications) dedupes ONLY via
@@ -1056,7 +1309,7 @@ def _process_one(evt: dict) -> None:
)
def record_deferred_wakeup(session_id: str, process_id: str, wakeup_prompt: str) -> None:
def record_deferred_wakeup(session_id: str, process_id: str, wakeup_prompt: str) -> bool:
"""Persist a deferred process-completion wakeup for later redelivery.
Called from ``_process_one`` when a completion arrives while a turn is
@@ -1068,10 +1321,11 @@ def record_deferred_wakeup(session_id: str, process_id: str, wakeup_prompt: str)
Idempotent per process_id: if the same process_id is already queued for
this session (kill_process racing the reader thread), it is not appended
twice. Best-effort — never raises into the drain loop.
twice. Returns whether the prompt is safely queued; never raises into the
drain loop.
"""
if not session_id or not wakeup_prompt:
return
return False
from api import config as _cfg
try:
@@ -1080,14 +1334,16 @@ def record_deferred_wakeup(session_id: str, process_id: str, wakeup_prompt: str)
if process_id and any(
e.get("process_id") == process_id for e in entries
):
return
return True
entries.append(
{"process_id": process_id, "wakeup_prompt": wakeup_prompt}
)
return True
except Exception:
logger.debug(
"record_deferred_wakeup failed for session %s", session_id, exc_info=True
)
return False
def claim_deferred_wakeups(session_id: str) -> list[dict]: