From 39fc16983434e1b8e1b3f82bc8bc0dc85fec18d0 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 13 Jul 2026 17:56:08 -0400 Subject: [PATCH 01/20] fix(#6008): route gateway approvals by run ID --- api/gateway_chat.py | 16 +- api/route_approvals.py | 102 ++++++++-- api/routes.py | 37 +++- tests/test_gateway_approval_runs_api.py | 251 ++++++++++++++++++++++++ 4 files changed, 375 insertions(+), 31 deletions(-) diff --git a/api/gateway_chat.py b/api/gateway_chat.py index 4f1fbdee..b19eb2e1 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -338,8 +338,8 @@ def _gateway_runs_approval_event(payload: dict) -> dict | None: args = payload.get("args") if isinstance(payload.get("args"), (list, dict)) else [] run_id = str(payload.get("run_id") or "").strip() approval_id = str(payload.get("approval_id") or payload.get("id") or "").strip() - if not approval_id: - approval_id = uuid.uuid4().hex + if not approval_id and run_id: + approval_id = f"gwrun:{run_id}" risk = str(payload.get("risk_level") or "high").strip() choices = payload.get("choices") if isinstance(payload.get("choices"), list) else [] allow_permanent = payload.get("allow_permanent") @@ -478,6 +478,10 @@ def _run_gateway_runs_api_streaming( approval_data = _gateway_runs_approval_event(payload) if approval_data: approval_data["run_id"] = run_id + if not approval_data.get("approval_id"): + approval_data["approval_id"] = f"gwrun:{run_id}" + from api.route_approvals import submit_gateway_pending_mirror + submit_gateway_pending_mirror(session_id, approval_data) put_gateway_event("approval", approval_data) sse_event = "message" continue @@ -522,6 +526,8 @@ def _run_gateway_runs_api_streaming( sse_event = "message" continue if payload_event == "run.completed": + from api.route_approvals import retire_gateway_pending_mirror + retire_gateway_pending_mirror(session_id, run_id=run_id) if payload.get("error"): raise RuntimeError(str(payload["error"])) output = str(payload.get("output") or "") @@ -533,8 +539,12 @@ def _run_gateway_runs_api_streaming( sse_event = "message" continue if payload_event == "run.failed": + from api.route_approvals import retire_gateway_pending_mirror + retire_gateway_pending_mirror(session_id, run_id=run_id) raise RuntimeError(str(payload.get("error") or "Gateway run failed")) if payload_event == "run.cancelled": + from api.route_approvals import retire_gateway_pending_mirror + retire_gateway_pending_mirror(session_id, run_id=run_id) put_gateway_event("cancel", {"message": "Cancelled by gateway"}) return None, usage reasoning_delta = _gateway_sse_reasoning_delta(payload) @@ -908,12 +918,12 @@ def _run_gateway_chat_streaming( _approval_run_id = str(approval_data.get("run_id") or "").strip() if _approval_run_id: _STREAM_RUN_IDS[stream_id] = _approval_run_id - put_gateway_event("approval", approval_data) try: from api.route_approvals import submit_gateway_pending_mirror submit_gateway_pending_mirror(session_id, approval_data) except Exception: logger.debug("submit_gateway_pending_mirror failed", exc_info=True) + put_gateway_event("approval", approval_data) else: logger.debug("Ignoring malformed gateway approval payload") sse_event = "message" diff --git a/api/route_approvals.py b/api/route_approvals.py index 33e9dca7..fda32fc3 100644 --- a/api/route_approvals.py +++ b/api/route_approvals.py @@ -155,6 +155,9 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No if not _is_gateway_mirror_entry(entry): rebuilt.append(entry) continue + if str(entry.get("run_id") or "").strip(): + rebuilt.append(entry) + continue if live_token and entry.get(_GATEWAY_MIRROR_TOKEN) == live_token and not live_mirror_present: rebuilt.append(entry) live_mirror_present = True @@ -163,7 +166,8 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No if live_token and not live_mirror_present: mirror_entry = dict(live_head_data) - mirror_entry.setdefault("approval_id", uuid.uuid4().hex) + mirror_run_id = str(mirror_entry.get("run_id") or "").strip() + mirror_entry.setdefault("approval_id", f"gwrun:{mirror_run_id}" if mirror_run_id else uuid.uuid4().hex) mirror_entry[_GATEWAY_MIRROR_FLAG] = True mirror_entry[_GATEWAY_MIRROR_TOKEN] = live_token rebuilt.append(mirror_entry) @@ -184,35 +188,95 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No return head, total, changed -def _gateway_mirrored_pending_run_id(session_key: str, approval_id: str) -> str | None: - """Return the mirrored gateway approval run_id for a matching pending card. +def _gateway_pending_mirror_locked(session_key: str, approval_id: str = "", run_id: str = "") -> dict | None: + """Return the exact live run-backed mirror under `_lock`.""" + approval_id = str(approval_id or "").strip() + run_id = str(run_id or "").strip() + queue = _pending.get(session_key) + entries = queue if isinstance(queue, list) else [queue] if queue else [] + if approval_id: + for entry in entries: + if not _is_gateway_mirror_entry(entry) or not str(entry.get("run_id") or "").strip(): + continue + if entry.get("approval_id") == approval_id: + if run_id and entry.get("run_id") != run_id: + return None + return entry + return None + for entry in entries: + if not _is_gateway_mirror_entry(entry) or not str(entry.get("run_id") or "").strip(): + continue + if run_id and entry.get("run_id") == run_id: + return entry + return None - Reconciles the mirror first so a live gateway head still survives a lost - `active_stream_id` pointer. - """ + +def gateway_pending_mirror(session_key: str, approval_id: str = "", run_id: str = "") -> dict | None: + """Return an exact live run-backed mirror for this session.""" + with _lock: + reconcile_gateway_pending_mirror_locked(session_key) + entry = _gateway_pending_mirror_locked(session_key, approval_id, run_id) + return dict(entry) if entry else None + + +def retire_gateway_pending_mirror(session_key: str, approval_id: str = "", run_id: str = "") -> bool: + """Retire only the exact run-backed mirror and notify its new queue head.""" + with _lock: + reconcile_gateway_pending_mirror_locked(session_key) + queue = _pending.get(session_key) + entries = queue if isinstance(queue, list) else [queue] if queue else [] + match = _gateway_pending_mirror_locked(session_key, approval_id, run_id) + if not match: + return False + entries.remove(match) + if entries: + _pending[session_key] = entries + else: + _pending.pop(session_key, None) + head = entries[0] if entries else None + _approval_sse_notify_locked(session_key, head, len(entries)) + publish_session_list_changed("attention_resolved") + return True + + +def _gateway_mirrored_pending_run_id(session_key: str, approval_id: str) -> str | None: + """Compatibility wrapper for exact run-backed lookup.""" approval_id = str(approval_id or "").strip() if not approval_id: return None with _lock: - reconcile_gateway_pending_mirror_locked(session_key) - queue = _pending.get(session_key) - if isinstance(queue, list): - entries = queue - elif queue: - entries = [queue] - else: - return None - for entry in entries: - if isinstance(entry, dict) and entry.get("approval_id") == approval_id and entry.get(_GATEWAY_MIRROR_FLAG): - run_id = str(entry.get("run_id") or "").strip() - return run_id or None + entry = _gateway_pending_mirror_locked(session_key, approval_id=approval_id) + if entry: + return str(entry.get("run_id") or "").strip() or None + return None + + +def _gateway_mirrored_pending_approval_id_by_run_id(session_key: str, run_id: str) -> str | None: + """Return the mirrored approval_id for a matching gateway run.""" + run_id = str(run_id or "").strip() + if not run_id: + return None + with _lock: + entry = _gateway_pending_mirror_locked(session_key, run_id=run_id) + if entry: + approval_id = str(entry.get("approval_id") or "").strip() + return approval_id or None return None def submit_gateway_pending_mirror(session_key: str, approval: dict) -> None: """Mirror the live gateway head into WebUI polling state under a typed tag.""" - del approval # mirror from the live gateway head under `_lock`, not from callback input with _lock: + run_id = str(approval.get("run_id") or "").strip() + if run_id: + live_gateway_queue = _gateway_queues.get(session_key) or [] + if not live_gateway_queue: + mirror_entry = dict(approval) + mirror_entry["run_id"] = run_id + mirror_entry["approval_id"] = str(mirror_entry.get("approval_id") or f"gwrun:{run_id}").strip() + mirror_entry[_GATEWAY_MIRROR_FLAG] = True + if not _gateway_pending_mirror_locked(session_key, approval_id=mirror_entry["approval_id"], run_id=run_id): + _normalize_pending_queue_locked(session_key).append(mirror_entry) head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) _approval_sse_notify_locked(session_key, head, total) publish_session_list_changed("attention_pending") diff --git a/api/routes.py b/api/routes.py index 8a96dd4d..5d72e0f8 100644 --- a/api/routes.py +++ b/api/routes.py @@ -9715,7 +9715,8 @@ from api.route_approvals import ( # noqa: F401 — re-exports for backward comp _approval_sse_notify_locked, _approval_sse_notify, _GATEWAY_MIRROR_FLAG, - _gateway_mirrored_pending_run_id, + gateway_pending_mirror, + retire_gateway_pending_mirror, reconcile_gateway_pending_mirror_locked, submit_gateway_pending_mirror, submit_pending, @@ -23914,29 +23915,47 @@ def _handle_approval_respond(handler, body): ) from api.config import get_config as _get_config s = get_session(sid) - _run_id = None + _candidate_run_id = None if s is not None: active_sid = getattr(s, "active_stream_id", None) if active_sid: - _run_id = _STREAM_RUN_IDS.get(active_sid) - if not _run_id and approval_id: - _run_id = _gateway_mirrored_pending_run_id(sid, approval_id) + _candidate_run_id = _STREAM_RUN_IDS.get(active_sid) + matched_mirror = gateway_pending_mirror(sid, approval_id=approval_id, run_id=_candidate_run_id) + _run_id = matched_mirror["run_id"] if matched_mirror else None + if not matched_mirror and approval_id: + with _lock: + queue = _pending.get(sid) + entries = queue if isinstance(queue, list) else [queue] if queue else [] + local_match = any( + isinstance(entry, dict) + and entry.get("approval_id") == approval_id + and not entry.get(_GATEWAY_MIRROR_FLAG) + for entry in entries + ) + if local_match: + _candidate_run_id = None if _run_id: - if not approval_id: - return bad(handler, "approval_id is required for gateway approvals") from api.runner_client import HttpRunnerClient, RunnerClientError _cfg = _get_config() _base = _gateway_base_url(_cfg) _key = _gateway_api_key() try: - HttpRunnerClient(base_url=_base, api_key=_key).respond_approval(_run_id, approval_id, choice) + HttpRunnerClient(base_url=_base, api_key=_key).respond_approval( + _run_id, matched_mirror["approval_id"], choice + ) except (RunnerClientError, ValueError) as exc: return j(handler, {"ok": False, "choice": choice, "relayed": True, "error": str(exc)}, status=502) # The outbound relay only resumes the remote run; the local mirror # still needs the same cleanup path so the parked entry, mirrored # card, and agent signal all settle here too. - _resolve_approval_legacy(sid, approval_id, choice) + cleanup_approval_id = matched_mirror["approval_id"] + _resolve_approval_legacy(sid, cleanup_approval_id, choice) + retire_gateway_pending_mirror(sid, approval_id=cleanup_approval_id, run_id=_run_id) return j(handler, {"ok": True, "choice": choice, "relayed": True}) + if _candidate_run_id or approval_id.startswith("gwrun:"): + return j(handler, {"ok": False, "choice": choice, "relayed": False, + "code": "gateway_run_unavailable", + "error": _GATEWAY_APPROVAL_RELAY_UNAVAILABLE}, status=409) # Only a still-mirrored gateway approval with a missing run should 409; # stale or empty gateway clicks fall through to local resolution. if webui_gateway_chat_enabled(_get_config()) and _gateway_pending_approval_without_run_id( diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index af10c445..a1c7a2a0 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -6,6 +6,7 @@ import json import socket import threading import urllib.error +from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -301,6 +302,8 @@ def test_gateway_approval_event_translation(): assert result["risk_level"] == "high" assert result["run_id"] == "run-999" assert result["approval_id"] == "appr-1" + empty_id = _gateway_runs_approval_event({**payload, "approval_id": "", "id": ""}) + assert empty_id["approval_id"] == "gwrun:run-999" downgraded = _gateway_runs_approval_event({ "command": "rm -rf /tmp/x", @@ -412,6 +415,169 @@ def test_gateway_runs_api_streaming_parses_real_run_events(): assert events[0][1]["approval_id"] == "appr-1" assert events[1] == ("reasoning", {"text": "thinking..."}) assert events[2] == ("token", {"text": "Hello"}) + import api.route_approvals as approvals + assert approvals.gateway_pending_mirror("sess1", run_id="run-abc") is None + + +def test_empty_id_runs_approval_reaches_real_response_lifecycle(): + """Runs API empty IDs are carried and resolved through the live mirror.""" + from api.gateway_chat import _run_gateway_runs_api_streaming + import api.route_approvals as approvals + import api.routes as routes + + for choice in ("once", "deny"): + sid = f"sess-real-{choice}" + stream_id = f"stream-real-{choice}" + events = [] + + class _JsonResponse: + def read(self, _limit=None): + return b'{"run_id":"run-real-' + choice.encode() + b'"}' + def __enter__(self): + return self + def __exit__(self, *args): + return None + + class _SseResponse: + def __iter__(self): + return iter([ + b'data: {"event":"approval.request","command":"echo x",' + b'"description":"approval","approval_id":""}\n', + b'\n', b'data: [DONE]\n', b'\n' + ]) + def __enter__(self): + return self + def __exit__(self, *args): + return None + + def fake_urlopen(req, *, timeout=None): + return _JsonResponse() if req.full_url.endswith("/v1/runs") else _SseResponse() + + handler = MagicMock() + handler.wfile = io.BytesIO() + with patch("urllib.request.urlopen", side_effect=fake_urlopen), \ + patch("api.routes.get_session", return_value=SimpleNamespace(active_stream_id=None)), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.runner_client.HttpRunnerClient._request_json", return_value={"ok": True}): + _run_gateway_runs_api_streaming( + sid, "hi", "test", "/tmp", stream_id, "http://gw:8642", "", [], {}, + put_gateway_event=lambda event, data: events.append((event, data)), + cancel_event=threading.Event(), + ) + approval_id = events[0][1]["approval_id"] + assert approval_id == f"gwrun:run-real-{choice}" + assert approvals.gateway_pending_mirror(sid, approval_id=approval_id)["run_id"] == f"run-real-{choice}" + routes._handle_approval_respond(handler, { + "session_id": sid, "choice": choice, "approval_id": approval_id, + }) + assert handler.send_response.call_args.args[0] == 200 + assert json.loads(handler.wfile.getvalue().decode())["ok"] is True + assert approvals.gateway_pending_mirror(sid, approval_id=approval_id) is None + approvals._pending.pop(sid, None) + + +def test_gateway_mirror_match_is_session_scoped_and_retires_one_run(): + import api.route_approvals as approvals + + sid = "sess-multi-run" + other_sid = "sess-other-run" + approvals._pending.pop(sid, None) + approvals._pending.pop(other_sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-a", "command": "a"}) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-b", "command": "b"}) + first = approvals.gateway_pending_mirror(sid, approval_id="gwrun:run-a") + second = approvals.gateway_pending_mirror(sid, run_id="run-b") + assert first["approval_id"] == "gwrun:run-a" + assert second["approval_id"] == "gwrun:run-b" + assert approvals.gateway_pending_mirror(other_sid, run_id="run-a") is None + assert approvals.retire_gateway_pending_mirror(sid, run_id="run-a") is True + assert approvals.gateway_pending_mirror(sid, run_id="run-a") is None + assert approvals.gateway_pending_mirror(sid, run_id="run-b") is not None + finally: + approvals._pending.pop(sid, None) + approvals._pending.pop(other_sid, None) + + +def test_stale_gateway_card_does_not_relay_live_run(): + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + sid = "sess-stale-gateway" + stream_id = "sid-stale-live-run" + _STREAM_RUN_IDS[stream_id] = "run-live" + approvals._pending.pop(sid, None) + approvals.submit_gateway_pending_mirror(sid, { + "run_id": "run-live", "approval_id": "gwrun:run-live", "command": "echo live" + }) + + mock_session = MagicMock() + mock_session.active_stream_id = stream_id + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.runner_client.HttpRunnerClient._request_json") as request_json: + from api.routes import _handle_approval_respond + _handle_approval_respond(handler, { + "session_id": sid, + "choice": "once", + "approval_id": "gwrun:run-stale", + }) + + payload = json.loads(handler.wfile.getvalue().decode("utf-8")) + assert handler.send_response.call_args.args[0] == 409 + assert payload["code"] == "gateway_run_unavailable" + request_json.assert_not_called() + assert approvals.gateway_pending_mirror(sid, run_id="run-live") is not None + + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + + +def test_retained_gateway_card_does_not_relay_different_active_run(): + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + sid = "sess-retained-gateway" + stream_id = "sid-retained-live-run" + _STREAM_RUN_IDS[stream_id] = "run-live" + approvals._pending.pop(sid, None) + approvals.submit_gateway_pending_mirror(sid, { + "run_id": "run-retained", "approval_id": "gwrun:run-retained", "command": "echo retained" + }) + approvals.submit_gateway_pending_mirror(sid, { + "run_id": "run-live", "approval_id": "gwrun:run-live", "command": "echo live" + }) + + mock_session = MagicMock() + mock_session.active_stream_id = stream_id + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.runner_client.HttpRunnerClient._request_json") as request_json: + from api.routes import _handle_approval_respond + _handle_approval_respond(handler, { + "session_id": sid, + "choice": "once", + "approval_id": "gwrun:run-retained", + }) + + payload = json.loads(handler.wfile.getvalue().decode("utf-8")) + assert handler.send_response.call_args.args[0] == 409 + assert payload["code"] == "gateway_run_unavailable" + request_json.assert_not_called() + assert approvals.gateway_pending_mirror(sid, approval_id="gwrun:run-retained") is not None + assert approvals.gateway_pending_mirror(sid, approval_id="gwrun:run-live") is not None + + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) def test_gateway_runs_api_streaming_preserves_multimodal_input(): @@ -546,9 +712,13 @@ def test_gateway_runs_api_cancel_does_not_emit_empty_response(): def test_gateway_approval_response_relay(): """_handle_approval_respond relays the real gateway approval body.""" from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals # Seed the mapping. _STREAM_RUN_IDS["sid-relay"] = "run abc/1" + approvals.submit_gateway_pending_mirror("sess-relay", { + "run_id": "run abc/1", "approval_id": "appr x/y", "command": "echo x" + }) mock_session = MagicMock() mock_session.active_stream_id = "sid-relay" @@ -578,14 +748,89 @@ def test_gateway_approval_response_relay(): # Cleanup. _STREAM_RUN_IDS.pop("sid-relay", None) + approvals._pending.pop("sess-relay", None) + + +def test_gateway_approval_response_without_approval_id_routes_by_run_id(): + """Runs API approvals resolve by the server-owned run ID.""" + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + _STREAM_RUN_IDS["sid-relay-empty-id"] = "run abc/1" + approvals.submit_gateway_pending_mirror("sess-relay", { + "run_id": "run abc/1", "approval_id": "", "command": "echo x" + }) + + mock_session = MagicMock() + mock_session.active_stream_id = "sid-relay-empty-id" + captured = {} + + def fake_request_json(self, req): + captured["url"] = req.full_url + captured["body"] = json.loads(req.data) + return {"ok": True} + + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.runner_client.HttpRunnerClient._request_json", new=fake_request_json), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.routes._resolve_approval_legacy", return_value=True) as cleanup: + from api.routes import _handle_approval_respond + _handle_approval_respond( + handler, + {"session_id": "sess-relay", "choice": "once", "approval_id": ""}, + ) + + payload = json.loads(handler.wfile.getvalue().decode("utf-8")) + status = handler.send_response.call_args.args[0] + assert status == 200, f"status={status} payload={payload} outbound={captured}" + assert captured["url"] == "http://gw:8642/v1/runs/run%20abc%2F1/approval" + assert captured["body"] == {"choice": "once", "approval_id": "gwrun:run abc/1"} + assert payload["ok"] is True + assert payload["relayed"] is True + cleanup.assert_called_once_with("sess-relay", "gwrun:run abc/1", "once") + + _STREAM_RUN_IDS.pop("sid-relay-empty-id", None) + approvals._pending.pop("sess-relay", None) + + +def test_gateway_mirrored_pending_approval_id_by_run_id_skips_older_local_entries(): + """Run-scoped cleanup must target the mirrored gateway card, not queue head.""" + import api.route_approvals as approvals + + sid = "sess-relay-mirror" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._pending[sid] = [{"approval_id": "local-1", "command": "echo local"}] + approvals._gateway_queues[sid] = [SimpleNamespace(data={ + "approval_id": "gateway-real", + "command": "echo gateway", + "run_id": "run-xyz", + })] + approvals.submit_gateway_pending_mirror(sid, {}) + queue = approvals._pending[sid] + assert queue[0]["approval_id"] == "local-1" + assert queue[1]["approval_id"] == "gateway-real" + assert approvals._gateway_mirrored_pending_approval_id_by_run_id(sid, "run-xyz") == "gateway-real" + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) def test_gateway_approval_response_relay_failure_returns_502(): """Gateway relay failures must surface as HTTP errors to the frontend.""" from api.gateway_chat import _STREAM_RUN_IDS from api.runner_client import RunnerClientError + import api.route_approvals as approvals _STREAM_RUN_IDS["sid-relay-fail"] = "run-abc" + approvals.submit_gateway_pending_mirror("sess-relay", { + "run_id": "run-abc", "approval_id": "appr-x", "command": "echo x" + }) mock_session = MagicMock() mock_session.active_stream_id = "sid-relay-fail" @@ -609,13 +854,18 @@ def test_gateway_approval_response_relay_failure_returns_502(): assert "relay failed" in payload["error"] _STREAM_RUN_IDS.pop("sid-relay-fail", None) + approvals._pending.pop("sess-relay", None) def test_gateway_approval_response_invalid_gateway_base_returns_502(): """Misconfigured gateway bases must not fall through to the local approval path.""" from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals _STREAM_RUN_IDS["sid-relay-invalid-base"] = "run-abc" + approvals.submit_gateway_pending_mirror("sess-relay", { + "run_id": "run-abc", "approval_id": "appr-x", "command": "echo x" + }) mock_session = MagicMock() mock_session.active_stream_id = "sid-relay-invalid-base" @@ -638,6 +888,7 @@ def test_gateway_approval_response_invalid_gateway_base_returns_502(): assert "runner base_url must be http(s)" in payload["error"] _STREAM_RUN_IDS.pop("sid-relay-invalid-base", None) + approvals._pending.pop("sess-relay", None) # --------------------------------------------------------------------------- From 489728d71fef4680969b77a54a7cc24da68f997e Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 13 Jul 2026 19:36:37 -0400 Subject: [PATCH 02/20] fix(gateway): scope approval mirror cleanup to live heads (#6008) --- api/route_approvals.py | 35 ++++++++++++++++++++++--- tests/test_gateway_approval_runs_api.py | 7 ++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/api/route_approvals.py b/api/route_approvals.py index fda32fc3..97bdba37 100644 --- a/api/route_approvals.py +++ b/api/route_approvals.py @@ -148,6 +148,8 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No live_head_entry = live_gateway_queue[0] if live_gateway_queue else None live_head_data = getattr(live_head_entry, "data", None) or {} live_token = _gateway_mirror_entry_token(live_head_entry) if live_head_entry and live_head_data else None + live_run_id = str(live_head_data.get("run_id") or "").strip() + live_approval_id = str(live_head_data.get("approval_id") or "").strip() rebuilt: list[dict] = [] live_mirror_present = False @@ -155,13 +157,40 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No if not _is_gateway_mirror_entry(entry): rebuilt.append(entry) continue - if str(entry.get("run_id") or "").strip(): - rebuilt.append(entry) + entry_run_id = str(entry.get("run_id") or "").strip() + entry_approval_id = str(entry.get("approval_id") or "").strip() + entry_token = str(entry.get(_GATEWAY_MIRROR_TOKEN) or "").strip() + matches_live_head = False + if live_token: + if entry_token and entry_token == live_token: + matches_live_head = True + elif live_run_id and entry_run_id == live_run_id: + matches_live_head = True + elif live_approval_id and entry_approval_id == live_approval_id: + matches_live_head = True + + if entry_run_id: + if matches_live_head and not live_mirror_present: + if entry_token != live_token: + entry[_GATEWAY_MIRROR_TOKEN] = live_token + changed = True + rebuilt.append(entry) + live_mirror_present = True + continue + if not entry_token: + rebuilt.append(entry) + continue + changed = True continue - if live_token and entry.get(_GATEWAY_MIRROR_TOKEN) == live_token and not live_mirror_present: + + if matches_live_head and not live_mirror_present: + if entry_token != live_token: + entry[_GATEWAY_MIRROR_TOKEN] = live_token + changed = True rebuilt.append(entry) live_mirror_present = True continue + changed = True if live_token and not live_mirror_present: diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index a1c7a2a0..3b6cb1eb 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -426,13 +426,14 @@ def test_empty_id_runs_approval_reaches_real_response_lifecycle(): import api.routes as routes for choice in ("once", "deny"): + choice_bytes = choice.encode() sid = f"sess-real-{choice}" stream_id = f"stream-real-{choice}" events = [] class _JsonResponse: - def read(self, _limit=None): - return b'{"run_id":"run-real-' + choice.encode() + b'"}' + def read(self, _limit=None, _choice_bytes=choice_bytes): + return b'{"run_id":"run-real-' + _choice_bytes + b'"}' def __enter__(self): return self def __exit__(self, *args): @@ -462,7 +463,7 @@ def test_empty_id_runs_approval_reaches_real_response_lifecycle(): patch("api.runner_client.HttpRunnerClient._request_json", return_value={"ok": True}): _run_gateway_runs_api_streaming( sid, "hi", "test", "/tmp", stream_id, "http://gw:8642", "", [], {}, - put_gateway_event=lambda event, data: events.append((event, data)), + put_gateway_event=lambda event, data, _events=events: _events.append((event, data)), cancel_event=threading.Event(), ) approval_id = events[0][1]["approval_id"] From b907630f8f8cf672b49cff032591f0a4f188a238 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Wed, 15 Jul 2026 18:48:40 -0400 Subject: [PATCH 03/20] fix(#6008): keep gateway approvals attached to the right run --- api/gateway_chat.py | 246 ++-- api/route_approvals.py | 139 ++- api/routes.py | 127 +- static/boot.js | 15 +- static/commands.js | 11 +- static/messages.js | 4 +- static/sessions.js | 4 +- tests/test_cancel_stream_owner_guard.py | 47 +- tests/test_gateway_approval_runs_api.py | 1052 ++++++++++++++++- ...est_issue4771_local_approval_regression.py | 96 ++ .../test_issue6008_sidebar_stop_failure_ui.py | 144 +++ 11 files changed, 1698 insertions(+), 187 deletions(-) create mode 100644 tests/test_issue6008_sidebar_stop_failure_ui.py diff --git a/api/gateway_chat.py b/api/gateway_chat.py index b19eb2e1..db157b68 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -8,7 +8,9 @@ import threading import time import uuid import urllib.error +import urllib.parse import urllib.request +import uuid from typing import Any from api.config import ( @@ -39,6 +41,13 @@ logger = logging.getLogger(__name__) # Maps stream_id -> gateway run_id for approval response relay. _STREAM_RUN_IDS: dict[str, str] = {} +_STREAM_RUN_STARTING: set[str] = set() +_STREAM_RUN_STARTING_LOCK = threading.Lock() + + +def gateway_run_id_pending(stream_id: str) -> bool: + with _STREAM_RUN_STARTING_LOCK: + return stream_id in _STREAM_RUN_STARTING _WEBUI_CHAT_BACKEND_ENV = "HERMES_WEBUI_CHAT_BACKEND" _WEBUI_GATEWAY_BASE_URL_ENV = "HERMES_WEBUI_GATEWAY_BASE_URL" @@ -338,8 +347,6 @@ def _gateway_runs_approval_event(payload: dict) -> dict | None: args = payload.get("args") if isinstance(payload.get("args"), (list, dict)) else [] run_id = str(payload.get("run_id") or "").strip() approval_id = str(payload.get("approval_id") or payload.get("id") or "").strip() - if not approval_id and run_id: - approval_id = f"gwrun:{run_id}" risk = str(payload.get("risk_level") or "high").strip() choices = payload.get("choices") if isinstance(payload.get("choices"), list) else [] allow_permanent = payload.get("allow_permanent") @@ -369,88 +376,107 @@ def _run_gateway_runs_api_streaming( attachments=None, cfg=None, session=None, ): """Submit via POST /v1/runs and relay SSE events including approval.""" - url_runs = f"{base_url.rstrip('/')}/v1/runs" - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - "X-Hermes-Session-Id": session_id, - } - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - headers["X-Hermes-Session-Key"] = f"webui:{session_id}" - message_content: Any = str(msg_text or "") - if attachments: - try: - from api.streaming import _build_native_multimodal_message + with _STREAM_RUN_STARTING_LOCK: + _STREAM_RUN_STARTING.add(stream_id) + try: + url_runs = f"{base_url.rstrip('/')}/v1/runs" + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "X-Hermes-Session-Id": session_id, + } + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + headers["X-Hermes-Session-Key"] = f"webui:{session_id}" + message_content: Any = str(msg_text or "") + if attachments: + try: + from api.streaming import _build_native_multimodal_message - message_content = _build_native_multimodal_message("", str(msg_text or ""), attachments, str(workspace), cfg=cfg) - except Exception: - logger.debug("Failed to build runs-API multimodal attachment payload", exc_info=True) - message_content = str(msg_text or "") - from api.streaming import _strip_oob_blocks + message_content = _build_native_multimodal_message("", str(msg_text or ""), attachments, str(workspace), cfg=cfg) + except Exception: + logger.debug("Failed to build runs-API multimodal attachment payload", exc_info=True) + message_content = str(msg_text or "") + from api.streaming import _strip_oob_blocks - instructions_parts = [] - conversation_history = [] - for entry in getattr(session, "context_messages", None) or []: - if not isinstance(entry, dict): - continue - role = str(entry.get("role") or "").strip().lower() - if role not in {"user", "assistant"}: - continue - content = entry.get("content") - if content is not None: - content = _strip_oob_blocks(content) + instructions_parts = [] + conversation_history = [] + for entry in getattr(session, "context_messages", None) or []: + if not isinstance(entry, dict): + continue + role = str(entry.get("role") or "").strip().lower() + if role not in {"user", "assistant"}: + continue + content = entry.get("content") + if content is not None: + content = _strip_oob_blocks(content) + conversation_history.append({"role": role, "content": content}) + for entry in prefill_messages or []: + if not isinstance(entry, dict): + continue + role = str(entry.get("role") or "").strip().lower() + content = entry.get("content") + if role == "system": + if isinstance(content, str) and content.strip(): + instructions_parts.append(content) + elif content is not None: + instructions_parts.append(str(content)) + continue + if role not in {"user", "assistant"}: + continue + if content is not None: + content = _strip_oob_blocks(content) conversation_history.append({"role": role, "content": content}) - for entry in prefill_messages or []: - if not isinstance(entry, dict): - continue - role = str(entry.get("role") or "").strip().lower() - content = entry.get("content") - if role == "system": - if isinstance(content, str) and content.strip(): - instructions_parts.append(content) - elif content is not None: - instructions_parts.append(str(content)) - continue - if role not in {"user", "assistant"}: - continue - if content is not None: - content = _strip_oob_blocks(content) - conversation_history.append({"role": role, "content": content}) - run_input = message_content - if isinstance(run_input, list): - run_input = [{"role": "user", "content": run_input}] - run_body = { - "model": model or "default", - "input": run_input, - **body_extras, - "session_id": session_id, - } - if instructions_parts: - run_body["instructions"] = "\n\n".join(part for part in instructions_parts if part) - if conversation_history: - run_body["conversation_history"] = conversation_history - req = urllib.request.Request( - url_runs, - data=json.dumps(run_body).encode("utf-8"), - headers=headers, - method="POST", - ) - update_active_run(stream_id, phase="gateway-request") - with urllib.request.urlopen(req, timeout=30) as resp: - run_data = json.loads(resp.read(65536)) - run_id = str(run_data.get("run_id") or run_data.get("id") or "").strip() - if not run_id: - raise ValueError(f"Gateway runs API returned no run_id: {run_data!r}") + run_input = message_content + if isinstance(run_input, list): + run_input = [{"role": "user", "content": run_input}] + run_body = { + "model": model or "default", + "input": run_input, + **body_extras, + "session_id": session_id, + } + if instructions_parts: + run_body["instructions"] = "\n\n".join(part for part in instructions_parts if part) + if conversation_history: + run_body["conversation_history"] = conversation_history + req = urllib.request.Request( + url_runs, + data=json.dumps(run_body).encode("utf-8"), + headers=headers, + method="POST", + ) + update_active_run(stream_id, phase="gateway-request") + with urllib.request.urlopen(req, timeout=30) as resp: + run_data = json.loads(resp.read(65536)) + run_id = str(run_data.get("run_id") or run_data.get("id") or "").strip() + if not run_id: + raise ValueError(f"Gateway runs API returned no run_id: {run_data!r}") + except Exception: + with _STREAM_RUN_STARTING_LOCK: + _STREAM_RUN_STARTING.discard(stream_id) + raise + usage: dict = {} _STREAM_RUN_IDS[stream_id] = run_id + with _STREAM_RUN_STARTING_LOCK: + _STREAM_RUN_STARTING.discard(stream_id) + if cancel_event.is_set(): + stop_confirmed = False + try: + stop_confirmed = stop_gateway_run(stream_id) + except Exception: + logger.debug("Failed to stop gateway run after pre-stream cancellation", exc_info=True) + if stop_confirmed: + put_gateway_event("cancel", {"message": "Cancelled by user"}) + return None, usage + cancel_event.clear() url_events = f"{base_url.rstrip('/')}/v1/runs/{run_id}/events" headers_sse = dict(headers) headers_sse["Accept"] = "text/event-stream" req_events = urllib.request.Request(url_events, headers=headers_sse, method="GET") final_text = "" - usage: dict = {} sse_event = "message" with urllib.request.urlopen(req_events, timeout=_gateway_read_timeout_secs()) as resp: for raw_line in _iter_sse_lines_cancellable(resp, cancel_event): @@ -479,7 +505,7 @@ def _run_gateway_runs_api_streaming( if approval_data: approval_data["run_id"] = run_id if not approval_data.get("approval_id"): - approval_data["approval_id"] = f"gwrun:{run_id}" + approval_data["approval_id"] = f"gwrun:{run_id}:{uuid.uuid4().hex}" from api.route_approvals import submit_gateway_pending_mirror submit_gateway_pending_mirror(session_id, approval_data) put_gateway_event("approval", approval_data) @@ -562,6 +588,40 @@ def _run_gateway_runs_api_streaming( return final_text, usage +def stop_gateway_run(stream_id: str) -> bool: + """Request gateway interruption and report whether it was acknowledged.""" + run_id = str(_STREAM_RUN_IDS.get(stream_id) or "").strip() + if not run_id: + return False + from api.config import get_config + + cfg = get_config() + base_url = _gateway_base_url(cfg) + api_key = _gateway_api_key() + headers = {"Accept": "application/json", "Content-Type": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + req = urllib.request.Request( + f"{base_url.rstrip('/')}/v1/runs/{urllib.parse.quote(run_id, safe='')}/stop", + data=b"{}", + headers=headers, + method="POST", + ) + class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + try: + opener = urllib.request.build_opener(_NoRedirect) + with opener.open(req, timeout=10) as response: + final_url = str(getattr(response, "geturl", lambda: req.full_url)() or "") + status = int(getattr(response, "status", getattr(response, "code", 0)) or 0) + return 200 <= status < 300 and final_url == req.full_url + except (urllib.error.HTTPError, urllib.error.URLError, OSError, ValueError): + logger.debug("Gateway stop failed for run %s", run_id, exc_info=True) + return False + + def _settle_gateway_terminal_error(session_id, stream_id, workspace, model, model_provider, terminal_error): from api.streaming import ( _classify_provider_error, @@ -701,6 +761,7 @@ def _run_gateway_chat_streaming( STREAM_LIVE_TOOL_CALLS[stream_id] = [] success_writeback_committed = False + runs_api_pending_marked = False def put_gateway_event(event, data): if cancel_event.is_set() and not success_writeback_committed and event not in ("cancel", "error", "apperror"): @@ -742,6 +803,27 @@ def _run_gateway_chat_streaming( model=model, model_provider=model_provider, ) + base_url = _gateway_base_url(cfg) + api_key = _gateway_api_key() + try: + from api.config import _main_model_request_overrides + _gw_overrides = _main_model_request_overrides( + cfg, + effective_model=model, + effective_provider=model_provider, + ) + except Exception: + _gw_overrides = {} + _runs_api_enabled = _gateway_use_runs_api_enabled(cfg) + if _runs_api_enabled: + with _STREAM_RUN_STARTING_LOCK: + _STREAM_RUN_STARTING.add(stream_id) + runs_api_pending_marked = True + _use_runs_api = _runs_api_enabled and gateway_supports_approval(base_url, api_key) + if not _use_runs_api and runs_api_pending_marked: + with _STREAM_RUN_STARTING_LOCK: + _STREAM_RUN_STARTING.discard(stream_id) + runs_api_pending_marked = False try: from api.streaming import ( _load_webui_prefill_context, @@ -780,19 +862,6 @@ def _run_gateway_chat_streaming( except Exception: logger.debug("Failed to load WebUI gateway prefill context", exc_info=True) prefill_messages = [] - base_url = _gateway_base_url(cfg) - api_key = _gateway_api_key() - try: - from api.config import _main_model_request_overrides - _gw_overrides = _main_model_request_overrides( - cfg, - effective_model=model, - effective_provider=model_provider, - ) - except Exception: - _gw_overrides = {} - # Capability gate: use runs API when gateway advertises approval support. - _use_runs_api = _gateway_use_runs_api_enabled(cfg) and gateway_supports_approval(base_url, api_key) if _use_runs_api: body_extras = {} if model_provider: @@ -1207,5 +1276,8 @@ def _run_gateway_chat_streaming( STREAM_LAST_EVENT_ID.pop(stream_id, None) STREAMS.pop(stream_id, None) _STREAM_RUN_IDS.pop(stream_id, None) + if runs_api_pending_marked and gateway_run_id_pending(stream_id): + _finish_gateway_run_starting(stream_id) + _clear_gateway_run_starting(stream_id) unregister_stream_owner(stream_id) unregister_active_run(stream_id) diff --git a/api/route_approvals.py b/api/route_approvals.py index 97bdba37..6ab739d8 100644 --- a/api/route_approvals.py +++ b/api/route_approvals.py @@ -149,9 +149,12 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No live_head_data = getattr(live_head_entry, "data", None) or {} live_token = _gateway_mirror_entry_token(live_head_entry) if live_head_entry and live_head_data else None live_run_id = str(live_head_data.get("run_id") or "").strip() + if live_token and live_run_id and not str(live_head_data.get("approval_id") or "").strip(): + live_head_data["approval_id"] = f"gwrun:{live_run_id}:{live_token}" live_approval_id = str(live_head_data.get("approval_id") or "").strip() rebuilt: list[dict] = [] + deferred_run_entries: list[dict] = [] live_mirror_present = False for entry in queue_list: if not _is_gateway_mirror_entry(entry): @@ -164,9 +167,12 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No if live_token: if entry_token and entry_token == live_token: matches_live_head = True - elif live_run_id and entry_run_id == live_run_id: - matches_live_head = True - elif live_approval_id and entry_approval_id == live_approval_id: + elif ( + live_approval_id + and live_run_id + and entry_approval_id == live_approval_id + and entry_run_id == live_run_id + ): matches_live_head = True if entry_run_id: @@ -177,6 +183,12 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No rebuilt.append(entry) live_mirror_present = True continue + if live_token: + if entry_token: + changed = True + continue + deferred_run_entries.append(entry) + continue if not entry_token: rebuilt.append(entry) continue @@ -196,13 +208,19 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No if live_token and not live_mirror_present: mirror_entry = dict(live_head_data) mirror_run_id = str(mirror_entry.get("run_id") or "").strip() - mirror_entry.setdefault("approval_id", f"gwrun:{mirror_run_id}" if mirror_run_id else uuid.uuid4().hex) + mirror_entry.setdefault( + "approval_id", + f"gwrun:{mirror_run_id}:{live_token}" if mirror_run_id else uuid.uuid4().hex, + ) mirror_entry[_GATEWAY_MIRROR_FLAG] = True mirror_entry[_GATEWAY_MIRROR_TOKEN] = live_token rebuilt.append(mirror_entry) live_mirror_present = True changed = True + if deferred_run_entries: + rebuilt.extend(deferred_run_entries) + if rebuilt: if rebuilt != queue_list: _pending[session_key] = rebuilt @@ -224,14 +242,25 @@ def _gateway_pending_mirror_locked(session_key: str, approval_id: str = "", run_ queue = _pending.get(session_key) entries = queue if isinstance(queue, list) else [queue] if queue else [] if approval_id: + matched_entry: dict | None = None for entry in entries: - if not _is_gateway_mirror_entry(entry) or not str(entry.get("run_id") or "").strip(): + if not _is_gateway_mirror_entry(entry): continue - if entry.get("approval_id") == approval_id: - if run_id and entry.get("run_id") != run_id: + if entry.get("approval_id") != approval_id: + continue + entry_run_id = str(entry.get("run_id") or "").strip() + if not entry_run_id: + if not run_id: return None + continue + if run_id and entry_run_id != run_id: + continue + if run_id: return entry - return None + if matched_entry is not None: + return None + matched_entry = entry + return matched_entry for entry in entries: if not _is_gateway_mirror_entry(entry) or not str(entry.get("run_id") or "").strip(): continue @@ -249,21 +278,47 @@ def gateway_pending_mirror(session_key: str, approval_id: str = "", run_id: str def retire_gateway_pending_mirror(session_key: str, approval_id: str = "", run_id: str = "") -> bool: - """Retire only the exact run-backed mirror and notify its new queue head.""" + """Retire one approval, or every mirror for a terminal run.""" with _lock: reconcile_gateway_pending_mirror_locked(session_key) queue = _pending.get(session_key) entries = queue if isinstance(queue, list) else [queue] if queue else [] - match = _gateway_pending_mirror_locked(session_key, approval_id, run_id) - if not match: + normalized_run_id = str(run_id or "").strip() + gateway_queue = _gateway_queues.get(session_key) or [] + retained_gateway_queue = gateway_queue + gateway_queue_changed = False + if approval_id: + match = _gateway_pending_mirror_locked(session_key, approval_id, run_id) + retired = [match] if match else [] + else: + retired = [ + entry for entry in entries + if _is_gateway_mirror_entry(entry) + and str(entry.get("run_id") or "").strip() == normalized_run_id + ] if normalized_run_id else [] + if normalized_run_id: + retained_gateway_queue = [] + for entry in gateway_queue: + data = getattr(entry, "data", None) or {} + if str(data.get("run_id") or "").strip() == normalized_run_id: + gateway_queue_changed = True + continue + retained_gateway_queue.append(entry) + if not retired and not gateway_queue_changed: return False - entries.remove(match) + for match in retired: + entries.remove(match) + if normalized_run_id and not approval_id: + if retained_gateway_queue: + _gateway_queues[session_key] = retained_gateway_queue + else: + _gateway_queues.pop(session_key, None) if entries: _pending[session_key] = entries else: _pending.pop(session_key, None) - head = entries[0] if entries else None - _approval_sse_notify_locked(session_key, head, len(entries)) + head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) + _approval_sse_notify_locked(session_key, head, total) publish_session_list_changed("attention_resolved") return True @@ -280,33 +335,57 @@ def _gateway_mirrored_pending_run_id(session_key: str, approval_id: str) -> str return None -def _gateway_mirrored_pending_approval_id_by_run_id(session_key: str, run_id: str) -> str | None: - """Return the mirrored approval_id for a matching gateway run.""" - run_id = str(run_id or "").strip() - if not run_id: - return None - with _lock: - entry = _gateway_pending_mirror_locked(session_key, run_id=run_id) - if entry: - approval_id = str(entry.get("approval_id") or "").strip() - return approval_id or None - return None - - def submit_gateway_pending_mirror(session_key: str, approval: dict) -> None: """Mirror the live gateway head into WebUI polling state under a typed tag.""" with _lock: run_id = str(approval.get("run_id") or "").strip() if run_id: live_gateway_queue = _gateway_queues.get(session_key) or [] - if not live_gateway_queue: + approval_id = str(approval.get("approval_id") or "").strip() + live_head_entry = live_gateway_queue[0] if live_gateway_queue else None + live_head_data = getattr(live_head_entry, "data", None) or {} + live_head_run_id = str(live_head_data.get("run_id") or "").strip() + live_head_approval_id = str(live_head_data.get("approval_id") or "").strip() + live_token = ( + _gateway_mirror_entry_token(live_head_entry) + if live_head_entry and live_head_data + else None + ) + if ( + live_token + and live_head_run_id == run_id + and ( + not approval_id + or not live_head_approval_id + or live_head_approval_id == approval_id + ) + ): + if approval_id: + live_head_data["approval_id"] = approval_id + else: + approval_id = live_head_approval_id + if not approval_id: + approval_id = f"gwrun:{run_id}:{live_token}" + live_head_data["approval_id"] = approval_id + approval["approval_id"] = approval_id + else: + if not approval_id: + approval_id = f"gwrun:{run_id}:{uuid.uuid4().hex}" + approval["approval_id"] = approval_id mirror_entry = dict(approval) mirror_entry["run_id"] = run_id - mirror_entry["approval_id"] = str(mirror_entry.get("approval_id") or f"gwrun:{run_id}").strip() + mirror_entry["approval_id"] = approval_id mirror_entry[_GATEWAY_MIRROR_FLAG] = True - if not _gateway_pending_mirror_locked(session_key, approval_id=mirror_entry["approval_id"], run_id=run_id): + if not _gateway_pending_mirror_locked(session_key, approval_id=approval_id, run_id=run_id): _normalize_pending_queue_locked(session_key).append(mirror_entry) head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) + approval_id = str(approval.get("approval_id") or "").strip() + if run_id and approval_id: + mirror_entry = _gateway_pending_mirror_locked( + session_key, approval_id=approval_id, run_id=run_id + ) + if mirror_entry: + approval["approval_id"] = str(mirror_entry.get("approval_id") or approval_id).strip() _approval_sse_notify_locked(session_key, head, total) publish_session_list_changed("attention_pending") diff --git a/api/routes.py b/api/routes.py index 5d72e0f8..9151abf4 100644 --- a/api/routes.py +++ b/api/routes.py @@ -13368,6 +13368,46 @@ def handle_get(handler, parsed) -> bool: return bad(handler, "stream_id required") if not _stream_id_visible_to_request_profile(handler, stream_id): return True + gateway_stop_blocked = False + try: + from api.gateway_chat import _STREAM_RUN_IDS, gateway_run_id_pending, stop_gateway_run + + run_id = _STREAM_RUN_IDS.get(stream_id) + pending_run_id = False + if not run_id: + import time as _time + + pending_run_id = gateway_run_id_pending(stream_id) + attempts_remaining = 10 + while not run_id and pending_run_id and attempts_remaining > 0: + _time.sleep(0.05) + run_id = _STREAM_RUN_IDS.get(stream_id) + pending_run_id = gateway_run_id_pending(stream_id) + attempts_remaining -= 1 + if run_id: + if stop_gateway_run(stream_id): + owner_sid = stream_owner_session_id(stream_id) + if owner_sid: + retire_gateway_pending_mirror(owner_sid, run_id=run_id) + else: + gateway_stop_blocked = True + elif pending_run_id: + gateway_stop_blocked = True + except Exception: + logger.debug("Failed to stop gateway run during chat cancellation", exc_info=True) + gateway_stop_blocked = True + if gateway_stop_blocked: + return j( + handler, + { + "ok": False, + "cancelled": False, + "stream_id": stream_id, + "error": "Gateway stop failed", + }, + status=502, + ) + from api.runtime_adapter import LegacyJournalRuntimeAdapter, runtime_adapter_enabled if runtime_adapter_enabled(): @@ -23749,7 +23789,7 @@ def _handle_workspace_reorder(handler, body): return j(handler, {"ok": True, "workspaces": reordered}) -def _resolve_approval_legacy(sid: str, approval_id: str, choice: str) -> bool: +def _resolve_approval_legacy(sid: str, approval_id: str, choice: str, run_id: str = "") -> bool: """Resolve an approval through the existing callback path. Slice 3b keeps the RuntimeAdapter as a protocol translator: it delegates to @@ -23760,6 +23800,7 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str) -> bool: pending = None found_target = False gateway_keys = [] + gateway_head_matches_target = False with _lock: reconcile_gateway_pending_mirror_locked(sid) queue = _pending.get(sid) @@ -23767,10 +23808,13 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str) -> bool: if approval_id: # Find and remove the specific entry by approval_id. for i, entry in enumerate(queue): - if entry.get("approval_id") == approval_id: - pending = queue.pop(i) - found_target = True - break + if entry.get("approval_id") != approval_id: + continue + if run_id and str(entry.get("run_id") or "").strip() != run_id: + continue + pending = queue.pop(i) + found_target = True + break else: # A stale explicit id must not accidentally approve the # oldest queued command; duplicate/stale responses are @@ -23783,7 +23827,13 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str) -> bool: _pending.pop(sid, None) elif queue: # Legacy single-dict value. - if not approval_id or queue.get("approval_id") == approval_id: + if ( + not approval_id + or ( + queue.get("approval_id") == approval_id + and (not run_id or str(queue.get("run_id") or "").strip() == run_id) + ) + ): pending = _pending.pop(sid, None) found_target = pending is not None # When no _pending entry found AND no explicit approval_id was @@ -23806,6 +23856,17 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str) -> bool: # idempotent over the session key set so the outcome is # the same regardless of which entry wins the race. found_target = True + elif approval_id: + gw_queue = _gateway_queues.get(sid) + if gw_queue and len(gw_queue) > 0: + gw_entry = gw_queue[0] + gw_data = getattr(gw_entry, "data", None) or {} + gw_approval_id = str(gw_data.get("approval_id") or "").strip() + gw_run_id = str(gw_data.get("run_id") or "").strip() + if run_id: + gateway_head_matches_target = gw_approval_id == approval_id and gw_run_id == run_id + else: + gateway_head_matches_target = gw_approval_id == approval_id # Notify SSE subscribers of the new head (or empty state) so the UI # surfaces any trailing approvals that were queued behind this one # without waiting for the next submit_pending. Without this, a parallel @@ -23837,7 +23898,17 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str) -> bool: # This is the primary signal when streaming is active — the agent # thread is parked in entry.event.wait() and needs to be woken up. gateway_resolved = 0 - if found_target or not approval_id: + should_resolve_gateway = ( + not approval_id + or ( + found_target + and ( + not run_id + or gateway_head_matches_target + ) + ) + ) + if should_resolve_gateway: gateway_resolved = resolve_gateway_approval(sid, choice, resolve_all=False) or 0 # Keep the historical no-id response path truthy for old clients/tests while # making stale explicit ids bounded as not-active for Slice 3b. @@ -23920,20 +23991,50 @@ def _handle_approval_respond(handler, body): active_sid = getattr(s, "active_stream_id", None) if active_sid: _candidate_run_id = _STREAM_RUN_IDS.get(active_sid) - matched_mirror = gateway_pending_mirror(sid, approval_id=approval_id, run_id=_candidate_run_id) - _run_id = matched_mirror["run_id"] if matched_mirror else None - if not matched_mirror and approval_id: + local_match = False + run_backed_gateway_matches = 0 + if approval_id: with _lock: queue = _pending.get(sid) entries = queue if isinstance(queue, list) else [queue] if queue else [] local_match = any( isinstance(entry, dict) and entry.get("approval_id") == approval_id - and not entry.get(_GATEWAY_MIRROR_FLAG) + and ( + not entry.get(_GATEWAY_MIRROR_FLAG) + or not str(entry.get("run_id") or "").strip() + ) for entry in entries ) + run_backed_gateway_matches = sum( + 1 + for entry in entries + if isinstance(entry, dict) + and entry.get("approval_id") == approval_id + and entry.get(_GATEWAY_MIRROR_FLAG) + and str(entry.get("run_id") or "").strip() + ) if local_match: _candidate_run_id = None + matched_mirror = gateway_pending_mirror( + sid, approval_id=approval_id, run_id=_candidate_run_id + ) if approval_id else None + _run_id = matched_mirror["run_id"] if matched_mirror else None + if not matched_mirror and approval_id: + if local_match: + _candidate_run_id = None + elif run_backed_gateway_matches > 1 and not _candidate_run_id: + return j( + handler, + { + "ok": False, + "choice": choice, + "relayed": False, + "code": "gateway_run_unavailable", + "error": _GATEWAY_APPROVAL_RELAY_UNAVAILABLE, + }, + status=409, + ) if _run_id: from api.runner_client import HttpRunnerClient, RunnerClientError _cfg = _get_config() @@ -23949,10 +24050,10 @@ def _handle_approval_respond(handler, body): # still needs the same cleanup path so the parked entry, mirrored # card, and agent signal all settle here too. cleanup_approval_id = matched_mirror["approval_id"] - _resolve_approval_legacy(sid, cleanup_approval_id, choice) + _resolve_approval_legacy(sid, cleanup_approval_id, choice, run_id=_run_id) retire_gateway_pending_mirror(sid, approval_id=cleanup_approval_id, run_id=_run_id) return j(handler, {"ok": True, "choice": choice, "relayed": True}) - if _candidate_run_id or approval_id.startswith("gwrun:"): + if _candidate_run_id: return j(handler, {"ok": False, "choice": choice, "relayed": False, "code": "gateway_run_unavailable", "error": _GATEWAY_APPROVAL_RELAY_UNAVAILABLE}, status=409) diff --git a/static/boot.js b/static/boot.js index 1e79ac4e..a45a33a4 100644 --- a/static/boot.js +++ b/static/boot.js @@ -17,7 +17,7 @@ async function cancelStream(reason){ const sid = S.session && S.session.session_id; const streamId = S.activeStreamId; - if(!streamId) return; + if(!streamId) return false; // Interrupt provenance: log WHY the active run is being cancelled so operators // can tell an explicit Stop / interrupt from any other trigger when they see a // SIGINT/exit-code-130 in the backend logs. Only explicit user paths reach @@ -30,8 +30,10 @@ async function cancelStream(reason){ console.info('[stream] cancel requested', {reason:_reason, streamId, sessionId:sid}); } let respBody=null; + let respOk=false; try{ const r=await fetch(new URL(`api/chat/cancel?stream_id=${encodeURIComponent(streamId)}`,document.baseURI||location.href).href,{credentials:'include'}); + respOk=!!(r&&r.ok); try{respBody=await r.json();}catch(_){} }catch(e){ if(typeof console !== 'undefined' && console.warn){ @@ -50,7 +52,7 @@ async function cancelStream(reason){ // `cancel` event can clear INFLIGHT, render "Task cancelled", and refresh // the sidebar. Only clear locally when the backend says there is no active // stream left to settle. - if(respBody && respBody.cancelled===false && S.activeStreamId===streamId){ + if(respOk && respBody && respBody.cancelled===false && S.activeStreamId===streamId){ S.activeStreamId=null; setBusy(false); if(typeof setComposerStatus==='function') setComposerStatus(''); @@ -59,20 +61,24 @@ async function cancelStream(reason){ // distinguish reasons — keep the toast generic and short. if(typeof showToast==='function') showToast('Stream is no longer active',2000); } + return respOk; } async function cancelSessionStream(session){ const streamId = session&&session.active_stream_id; const sid = session&&session.session_id; - if(!streamId||!sid) return; + if(!streamId||!sid) return false; // Explicit sidebar "Stop response" — log provenance for the same reason as // cancelStream(). (#5345) if(typeof console !== 'undefined' && console.info){ console.info('[stream] cancel requested', {reason:'sidebar-stop', streamId, sessionId:sid}); } + let respOk=false; try{ - await fetch(new URL(`api/chat/cancel?stream_id=${encodeURIComponent(streamId)}`,document.baseURI||location.href).href,{credentials:'include'}); + const r=await fetch(new URL(`api/chat/cancel?stream_id=${encodeURIComponent(streamId)}`,document.baseURI||location.href).href,{credentials:'include'}); + respOk=!!(r&&r.ok); }catch(e){/* close local stream; keep UI state honest below */} + if(!respOk) return false; if(typeof closeLiveStream==='function') closeLiveStream(sid, streamId); session.active_stream_id=null; delete INFLIGHT[sid]; @@ -94,6 +100,7 @@ async function cancelSessionStream(session){ hideClarifyCard(true, 'cancelled'); } if(typeof renderSessionList==='function') renderSessionList(); + return true; } async function _savedSessionShouldStaySidebarOnly(sid){ diff --git a/static/commands.js b/static/commands.js index dc6a0d43..a2377655 100644 --- a/static/commands.js +++ b/static/commands.js @@ -1213,7 +1213,10 @@ async function cmdPersonality(args){ async function cmdStop(){ if(!S.session){showToast(t('no_active_session'));return;} if(!S.activeStreamId){showToast(t('no_active_task'));return;} - if(typeof cancelStream==='function'){await cancelStream('slash-stop');showToast(t('stream_stopped'));} + if(typeof cancelStream==='function'){ + if(await cancelStream('slash-stop')) showToast(t('stream_stopped')); + else showToast(t('cancel_failed')); + } else showToast(t('cancel_unavailable')); } @@ -1319,8 +1322,10 @@ async function cmdInterrupt(args){ updateQueueBadge(S.session.session_id); S.pendingFiles=[];renderTray(); // Cancel the active stream; setBusy(false) will drain the queue - if(typeof cancelStream==='function'){await cancelStream('slash-interrupt');} - showToast(t('cmd_interrupt_confirm'),2000); + if(typeof cancelStream==='function'){ + if(await cancelStream('slash-interrupt')) showToast(t('cmd_interrupt_confirm'),2000); + else showToast(t('cancel_failed')); + } } /** diff --git a/static/messages.js b/static/messages.js index b67a95e8..ea19f451 100644 --- a/static/messages.js +++ b/static/messages.js @@ -1395,8 +1395,8 @@ async function send(){ _clearComposerAfterQueuedSelectionSend(S.session&&S.session.session_id); S.pendingFiles=[];renderTray(); if(S.activeStreamId&&typeof cancelStream==='function'){ - showToast(t('busy_interrupt_confirm'),2000); - await cancelStream('busy-interrupt'); + if(await cancelStream('busy-interrupt')) showToast(t('busy_interrupt_confirm'),2000); + else showToast(t('cancel_failed')); } else { showToast(`Queued: "${text.slice(0,40)}${text.length>40?'…':''}"`,2000); } diff --git a/static/sessions.js b/static/sessions.js index ed41c6d8..1100794a 100644 --- a/static/sessions.js +++ b/static/sessions.js @@ -4905,8 +4905,8 @@ function _openSessionActionMenu(session, anchorEl){ ICONS.stop, async()=>{ closeSessionActionMenu(); - await cancelSessionStream(session); - showToast(t('stream_stopped')); + if(await cancelSessionStream(session)) showToast(t('stream_stopped')); + else showToast(t('cancel_failed')); } )); } diff --git a/tests/test_cancel_stream_owner_guard.py b/tests/test_cancel_stream_owner_guard.py index e149dd42..bc5e3333 100644 --- a/tests/test_cancel_stream_owner_guard.py +++ b/tests/test_cancel_stream_owner_guard.py @@ -135,6 +135,10 @@ class TestCancelStreamOwnerGuardStructural: """When the backend reports ``cancelled:false``, the fix should show a lightweight toast (the backend may have already finalized the turn, or a newer turn may hold the session lock).""" + assert "r.ok" in CANCEL_STREAM_SRC, ( + "cancelStream() must gate cancelled:false cleanup on an HTTP-success " + "response so failed gateway stops do not settle the UI locally" + ) assert "cancelled" in CANCEL_STREAM_SRC and "false" in CANCEL_STREAM_SRC, ( "cancelStream() must check respBody.cancelled === false and surface " "a toast (e.g. 'Stream is no longer active')" @@ -254,6 +258,25 @@ async function runAll() { toastCalls: [...M.toastCalls], }; + // T3b — failed HTTP response with cancelled:false body: keep the active + // owner path intact because the cancel did not succeed. + reset(); + globalThis.S = { activeStreamId: 'stream-1', session: { session_id: 'sid-1' } }; + _fetchResponse = { + ok: false, + json: () => Promise.resolve({ ok: false, cancelled: false, stream_id: 'stream-1' }), + }; + _fetchThrows = false; + await cancelStream(); + out.t3b_failed_http = { + finalActiveStreamId: globalThis.S.activeStreamId, + fetchCalls: M.fetchCalls.length, + closeCalls: [...M.closeCalls], + busyCalls: [...M.busyCalls], + composerCalls: [...M.composerCalls], + toastCalls: [...M.toastCalls], + }; + // T4 — network error: no throw, active owner kept because we do not know // whether the cancel landed. reset(); @@ -466,6 +489,26 @@ class TestCancelStreamOwnerGuardRuntime: f"got {r['toastCalls']}" ) + def test_t3b_failed_http_keeps_owner_state(self, runtime_results): + r = runtime_results["t3b_failed_http"] + assert r["fetchCalls"] == 1, ( + f"cancelStream() must still issue the cancel request on failed HTTP, " + f"got {r['fetchCalls']}" + ) + assert r["finalActiveStreamId"] == "stream-1", ( + f"cancelStream() must keep S.activeStreamId when /api/chat/cancel " + f"fails, got {r['finalActiveStreamId']!r}" + ) + assert r["closeCalls"] == [], ( + f"cancelStream() must keep owned SSE open on failed HTTP, got {r['closeCalls']}" + ) + assert r["busyCalls"] == [], ( + f"cancelStream() must not call setBusy(false) on failed HTTP, got {r['busyCalls']}" + ) + assert r["toastCalls"] == [], ( + f"cancelStream() should not show a toast on failed HTTP, got {r['toastCalls']}" + ) + def test_t5_owner_guard_preserves_new_turn(self, runtime_results): r = runtime_results["t5_owner_guard"] # The fetch still happened for the OLD stream. @@ -512,12 +555,14 @@ class TestCancelStreamOwnerGuardRuntime: ) assert runtime_results["t3_cancelled_false"]["finalActiveStreamId"] is None assert runtime_results["t3_cancelled_false"]["busyCalls"] == [False] + assert runtime_results["t3b_failed_http"]["finalActiveStreamId"] == "stream-1" + assert runtime_results["t3b_failed_http"]["busyCalls"] == [] # T5 is the owner-rotation case and must preserve the new turn. assert runtime_results["t5_owner_guard"]["finalActiveStreamId"] == "stream-2" assert runtime_results["t5_owner_guard"]["busyCalls"] == [] # Only the stale-owner path closes the old SSE; active owned paths # keep it open so the backend terminal event can settle/render. - for key in ("t2_happy_path", "t3_cancelled_false", "t4_network_error", "t5_owner_guard"): + for key in ("t2_happy_path", "t3_cancelled_false", "t3b_failed_http", "t4_network_error", "t5_owner_guard"): if key == "t5_owner_guard": assert runtime_results[key]["closeCalls"] == [["sid-1", "stream-1"]], ( f"{key} must close stale old SSE for ('sid-1', 'stream-1'), " diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index 3b6cb1eb..19cd3094 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -303,7 +303,7 @@ def test_gateway_approval_event_translation(): assert result["run_id"] == "run-999" assert result["approval_id"] == "appr-1" empty_id = _gateway_runs_approval_event({**payload, "approval_id": "", "id": ""}) - assert empty_id["approval_id"] == "gwrun:run-999" + assert empty_id["approval_id"] == "" downgraded = _gateway_runs_approval_event({ "command": "rm -rf /tmp/x", @@ -467,7 +467,7 @@ def test_empty_id_runs_approval_reaches_real_response_lifecycle(): cancel_event=threading.Event(), ) approval_id = events[0][1]["approval_id"] - assert approval_id == f"gwrun:run-real-{choice}" + assert approval_id.startswith(f"gwrun:run-real-{choice}:") assert approvals.gateway_pending_mirror(sid, approval_id=approval_id)["run_id"] == f"run-real-{choice}" routes._handle_approval_respond(handler, { "session_id": sid, "choice": choice, "approval_id": approval_id, @@ -488,10 +488,10 @@ def test_gateway_mirror_match_is_session_scoped_and_retires_one_run(): try: approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-a", "command": "a"}) approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-b", "command": "b"}) - first = approvals.gateway_pending_mirror(sid, approval_id="gwrun:run-a") + first = approvals.gateway_pending_mirror(sid, run_id="run-a") second = approvals.gateway_pending_mirror(sid, run_id="run-b") - assert first["approval_id"] == "gwrun:run-a" - assert second["approval_id"] == "gwrun:run-b" + assert first["approval_id"].startswith("gwrun:run-a:") + assert second["approval_id"].startswith("gwrun:run-b:") assert approvals.gateway_pending_mirror(other_sid, run_id="run-a") is None assert approvals.retire_gateway_pending_mirror(sid, run_id="run-a") is True assert approvals.gateway_pending_mirror(sid, run_id="run-a") is None @@ -501,6 +501,904 @@ def test_gateway_mirror_match_is_session_scoped_and_retires_one_run(): approvals._pending.pop(other_sid, None) +def test_gateway_mirror_exact_pair_lookup_skips_same_id_other_run(): + import api.route_approvals as approvals + + sid = "sess-same-id-other-run" + approvals._pending.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, { + "run_id": "run-old", "approval_id": "same-id", "command": "old" + }) + approvals.submit_gateway_pending_mirror(sid, { + "run_id": "run-live", "approval_id": "same-id", "command": "live" + }) + assert approvals.gateway_pending_mirror(sid, approval_id="same-id") is None + live = approvals.gateway_pending_mirror(sid, approval_id="same-id", run_id="run-live") + assert live is not None + assert live["run_id"] == "run-live" + assert approvals.retire_gateway_pending_mirror(sid, approval_id="same-id", run_id="run-live") + assert approvals.gateway_pending_mirror(sid, approval_id="same-id", run_id="run-live") is None + assert approvals.gateway_pending_mirror(sid, approval_id="same-id", run_id="run-old") is not None + finally: + approvals._pending.pop(sid, None) + + +def test_gateway_approval_response_relay_cleans_only_selected_shared_id_run(): + from api.gateway_chat import _STREAM_RUN_IDS + from api import routes + import api.route_approvals as approvals + + sid = "sess-shared-id-relay" + stream_id = "sid-shared-id-relay" + _STREAM_RUN_IDS[stream_id] = "run-live" + approvals._pending.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, { + "run_id": "run-old", "approval_id": "same-id", "command": "old" + }) + approvals.submit_gateway_pending_mirror(sid, { + "run_id": "run-live", "approval_id": "same-id", "command": "live" + }) + + mock_session = MagicMock() + mock_session.active_stream_id = stream_id + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond_approval, \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""): + routes._handle_approval_respond(handler, { + "session_id": sid, "choice": "once", "approval_id": "same-id", + }) + + handler.send_response.assert_called_with(200) + respond_approval.assert_called_once_with("run-live", "same-id", "once") + assert approvals.gateway_pending_mirror(sid, approval_id="same-id", run_id="run-live") is None + assert approvals.gateway_pending_mirror(sid, approval_id="same-id", run_id="run-old") is not None + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + + +def test_gateway_approval_response_without_active_run_rejects_shared_upstream_id(): + from api import routes + import api.route_approvals as approvals + + sid = "sess-shared-id-no-active-run" + approvals._pending.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, { + "run_id": "run-a", "approval_id": "shared-upstream-id", "command": "a" + }) + approvals.submit_gateway_pending_mirror(sid, { + "run_id": "run-b", "approval_id": "shared-upstream-id", "command": "b" + }) + + mock_session = MagicMock() + mock_session.active_stream_id = None + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.gateway_chat.webui_gateway_chat_enabled", return_value=True), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond_approval: + routes._handle_approval_respond(handler, { + "session_id": sid, "choice": "once", "approval_id": "shared-upstream-id", + }) + + payload = json.loads(handler.wfile.getvalue().decode("utf-8")) + assert handler.send_response.call_args.args[0] == 409 + assert payload["code"] == "gateway_run_unavailable" + respond_approval.assert_not_called() + assert approvals.gateway_pending_mirror(sid, approval_id="shared-upstream-id", run_id="run-a") is not None + assert approvals.gateway_pending_mirror(sid, approval_id="shared-upstream-id", run_id="run-b") is not None + finally: + approvals._pending.pop(sid, None) + + +def test_gateway_approval_relay_does_not_unblock_unrelated_local_head(): + from api.gateway_chat import _STREAM_RUN_IDS + from api import routes + import api.route_approvals as approvals + import tools.approval as ta + + sid = "sess-remote-relay-local-head" + stream_id = "sid-remote-relay-local-head" + _STREAM_RUN_IDS[stream_id] = "run-remote" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, { + "run_id": "run-remote", "approval_id": "remote-a", "command": "remote" + }) + local_entry = ta._ApprovalEntry({ + "command": "local", + "description": "local approval", + "approval_id": "local-b", + }) + with approvals._lock: + approvals._gateway_queues[sid] = [local_entry] + + mock_session = MagicMock() + mock_session.active_stream_id = stream_id + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond_approval, \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""): + routes._handle_approval_respond(handler, { + "session_id": sid, "choice": "once", "approval_id": "remote-a", + }) + + handler.send_response.assert_called_with(200) + respond_approval.assert_called_once_with("run-remote", "remote-a", "once") + assert local_entry.event.is_set() is False + assert local_entry.result is None + assert approvals.gateway_pending_mirror(sid, approval_id="remote-a", run_id="run-remote") is None + with approvals._lock: + assert sid in approvals._gateway_queues + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_empty_id_approvals_same_run_keep_distinct_fifo_cards(): + import api.route_approvals as approvals + + sid = "sess-same-run" + approvals._pending.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-a", "command": "first"}) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-a", "command": "second"}) + queue = approvals._pending[sid] + assert [entry["command"] for entry in queue] == ["first", "second"] + assert queue[0]["approval_id"] != queue[1]["approval_id"] + assert approvals.retire_gateway_pending_mirror(sid, approval_id=queue[0]["approval_id"], run_id="run-a") + assert approvals._pending[sid][0]["command"] == "second" + finally: + approvals._pending.pop(sid, None) + + +def test_live_gateway_head_gets_token_scoped_approval_id(): + import api.route_approvals as approvals + + sid = "sess-live-head" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._gateway_queues[sid] = [SimpleNamespace(data={"run_id": "run-live", "command": "head"})] + with approvals._lock: + head, total, changed = approvals.reconcile_gateway_pending_mirror_locked(sid) + assert changed is True + assert total == 1 + assert head["approval_id"].startswith("gwrun:run-live:") + assert approvals._gateway_queues[sid][0].data["approval_id"] == head["approval_id"] + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_live_gateway_head_emits_the_stored_token_scoped_identity(): + import api.route_approvals as approvals + + sid = "sess-live-head-emitted-id" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._gateway_queues[sid] = [SimpleNamespace(data={"run_id": "run-live", "command": "head"})] + approval_data = {"run_id": "run-live", "command": "head"} + approvals.submit_gateway_pending_mirror(sid, approval_data) + mirror = approvals.gateway_pending_mirror(sid, run_id="run-live") + assert mirror is not None + assert approval_data["approval_id"] == mirror["approval_id"] + assert approvals._gateway_queues[sid][0].data["approval_id"] == mirror["approval_id"] + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_distinct_same_run_approval_does_not_overwrite_live_head_identity(): + import api.route_approvals as approvals + + sid = "sess-same-run-distinct-approval" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._gateway_queues[sid] = [ + SimpleNamespace(data={"run_id": "run-live", "approval_id": "appr-a", "command": "head"}) + ] + approvals.submit_gateway_pending_mirror( + sid, + {"run_id": "run-live", "approval_id": "appr-b", "command": "second"}, + ) + assert approvals._gateway_queues[sid][0].data["approval_id"] == "appr-a" + assert approvals.gateway_pending_mirror(sid, approval_id="appr-a", run_id="run-live") is not None + assert approvals.gateway_pending_mirror(sid, approval_id="appr-b", run_id="run-live") is not None + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_direct_remote_approval_still_mirrors_with_local_gateway_head(): + import api.route_approvals as approvals + + sid = "sess-local-head-remote-mirror" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._gateway_queues[sid] = [SimpleNamespace(data={"command": "local-head"})] + approvals.submit_gateway_pending_mirror( + sid, + {"run_id": "run-remote", "approval_id": "remote-a", "command": "remote"}, + ) + mirror = approvals.gateway_pending_mirror(sid, approval_id="remote-a", run_id="run-remote") + assert mirror is not None + assert mirror["run_id"] == "run-remote" + assert mirror["approval_id"] == "remote-a" + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_reconcile_does_not_bind_live_token_by_approval_id_alone(): + import api.route_approvals as approvals + + sid = "sess-cross-run-shared-id" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._pending[sid] = [{ + approvals._GATEWAY_MIRROR_FLAG: True, + "run_id": "run-stale", + "approval_id": "appr-shared", + "command": "stale", + }] + approvals._gateway_queues[sid] = [SimpleNamespace(data={ + "run_id": "run-live", + "approval_id": "appr-shared", + "command": "live", + })] + with approvals._lock: + head, total, _changed = approvals.reconcile_gateway_pending_mirror_locked(sid) + queue = list(approvals._pending[sid]) + assert total == 2 + assert head["run_id"] == "run-live" + assert queue[0]["run_id"] == "run-live" + assert queue[1]["run_id"] == "run-stale" + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_terminal_run_retirement_removes_all_same_run_mirrors(): + import api.route_approvals as approvals + + sid = "sess-terminal-run" + approvals._pending.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-a", "command": "first"}) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-a", "command": "second"}) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-b", "command": "other"}) + assert approvals.retire_gateway_pending_mirror(sid, run_id="run-a") + assert approvals.gateway_pending_mirror(sid, run_id="run-a") is None + assert approvals.gateway_pending_mirror(sid, run_id="run-b") is not None + finally: + approvals._pending.pop(sid, None) + + +def test_terminal_run_retirement_clears_same_run_gateway_queue_state(): + import api.route_approvals as approvals + + sid = "sess-terminal-run-queue" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._gateway_queues[sid] = [ + SimpleNamespace(data={"run_id": "run-a", "approval_id": "appr-a", "command": "a"}), + SimpleNamespace(data={"command": "local-head"}), + ] + approvals.submit_gateway_pending_mirror( + sid, + {"run_id": "run-a", "approval_id": "appr-a", "command": "a"}, + ) + assert approvals.retire_gateway_pending_mirror(sid, run_id="run-a") + assert approvals.gateway_pending_mirror(sid, run_id="run-a") is None + assert not any( + str((getattr(entry, "data", None) or {}).get("run_id") or "").strip() == "run-a" + for entry in approvals._gateway_queues.get(sid, []) + ) + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_terminal_run_retirement_clears_non_head_same_run_gateway_queue_state(): + import api.route_approvals as approvals + + sid = "sess-terminal-run-non-head-queue" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._gateway_queues[sid] = [ + SimpleNamespace(data={"command": "local-head"}), + SimpleNamespace(data={"run_id": "run-a", "approval_id": "appr-a", "command": "a"}), + ] + assert approvals.retire_gateway_pending_mirror(sid, run_id="run-a") + assert approvals.gateway_pending_mirror(sid, run_id="run-a") is None + assert not any( + str((getattr(entry, "data", None) or {}).get("run_id") or "").strip() == "run-a" + for entry in approvals._gateway_queues.get(sid, []) + ) + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_exact_one_retirement_keeps_same_run_gateway_queue_state(): + import api.route_approvals as approvals + + sid = "sess-exact-one-queue" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._pending[sid] = [ + { + approvals._GATEWAY_MIRROR_FLAG: True, + "run_id": "run-a", + "approval_id": "appr-a", + "command": "a", + }, + { + approvals._GATEWAY_MIRROR_FLAG: True, + "run_id": "run-a", + "approval_id": "appr-b", + "command": "b", + }, + ] + approvals._gateway_queues[sid] = [ + SimpleNamespace(data={"run_id": "run-a", "approval_id": "appr-b", "command": "b"}) + ] + assert approvals.retire_gateway_pending_mirror(sid, approval_id="appr-a", run_id="run-a") + assert approvals.gateway_pending_mirror(sid, approval_id="appr-b", run_id="run-a") is not None + queue = approvals._gateway_queues.get(sid) or [] + assert len(queue) == 1 + assert queue[0].data["approval_id"] == "appr-b" + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_local_stop_uses_runs_api_stop_endpoint(): + from api.gateway_chat import _STREAM_RUN_IDS, stop_gateway_run + + stream_id = "stream-stop" + _STREAM_RUN_IDS[stream_id] = "run-stop" + + class _Response: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def geturl(self): + return "http://gw:8642/v1/runs/run-stop/stop" + + requests = [] + + class _Opener: + def open(self, req, *, timeout=None): + requests.append((req, timeout)) + return _Response() + + def fake_build_opener(*handlers): + assert handlers, "stop must install a no-redirect handler" + return _Opener() + + try: + with patch("urllib.request.build_opener", side_effect=fake_build_opener), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value="secret"): + assert stop_gateway_run(stream_id) is True + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + + request, timeout = requests[0] + assert request.full_url == "http://gw:8642/v1/runs/run-stop/stop" + assert request.get_method() == "POST" + assert request.get_header("Authorization") == "Bearer secret" + assert timeout == 10 + + +def test_runs_api_cancel_after_run_creation_stops_remote_run_before_events(): + from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming + + stream_id = "sid-run-create-cancel" + cancel_event = threading.Event() + requests = [] + + class _JsonResponse: + def __init__(self, payload): + self._payload = json.dumps(payload).encode("utf-8") + + def read(self, _limit=None): + return self._payload + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def fake_urlopen(req, *, timeout=None): + requests.append(req.full_url) + if req.full_url.endswith("/v1/runs"): + cancel_event.set() + return _JsonResponse({"run_id": "run-pre-events-cancel"}) + raise AssertionError(f"unexpected urlopen after pre-stream cancel: {req.full_url}") + + try: + with patch("urllib.request.urlopen", side_effect=fake_urlopen), \ + patch("api.gateway_chat.stop_gateway_run", return_value=True) as stop_run: + final_text, usage = _run_gateway_runs_api_streaming( + session_id="sess-run-create-cancel", + msg_text="hi", + model="test-model", + workspace="/tmp", + stream_id=stream_id, + base_url="http://gw:8642", + api_key="secret", + prefill_messages=[], + body_extras={}, + put_gateway_event=lambda *_args, **_kwargs: None, + cancel_event=cancel_event, + cfg={}, + ) + assert final_text is None + assert usage == {} + stop_run.assert_called_once_with(stream_id) + assert requests == ["http://gw:8642/v1/runs"] + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + + +def test_runs_api_cancel_after_run_creation_keeps_stream_open_when_stop_fails(): + from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming + + stream_id = "sid-run-create-cancel-stop-failed" + cancel_event = threading.Event() + requests = [] + events = [] + + class _JsonResponse: + def __init__(self, payload): + self._payload = json.dumps(payload).encode("utf-8") + + def read(self, _limit=None): + return self._payload + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + class _SseResponse: + def __init__(self, lines): + self._lines = lines + + def __iter__(self): + return iter(self._lines) + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def fake_urlopen(req, *, timeout=None): + requests.append(req.full_url) + if req.full_url.endswith("/v1/runs"): + cancel_event.set() + return _JsonResponse({"run_id": "run-pre-events-stop-failed"}) + if req.full_url.endswith("/events"): + return _SseResponse([ + b'data: {"event":"message.delta","delta":"Hello"}\n', + b'\n', + b'data: {"event":"run.completed","output":"Hello"}\n', + b'\n', + ]) + raise AssertionError(f"unexpected urlopen: {req.full_url}") + + try: + with patch("urllib.request.urlopen", side_effect=fake_urlopen), \ + patch("api.gateway_chat.stop_gateway_run", return_value=False) as stop_run: + final_text, usage = _run_gateway_runs_api_streaming( + session_id="sess-run-create-cancel-stop-failed", + msg_text="hi", + model="test-model", + workspace="/tmp", + stream_id=stream_id, + base_url="http://gw:8642", + api_key="secret", + prefill_messages=[], + body_extras={}, + put_gateway_event=lambda event, data: events.append((event, data)), + cancel_event=cancel_event, + cfg={}, + ) + assert final_text == "Hello" + assert usage == {} + stop_run.assert_called_once_with(stream_id) + assert requests == [ + "http://gw:8642/v1/runs", + "http://gw:8642/v1/runs/run-pre-events-stop-failed/events", + ] + assert [event for event, _data in events] == ["token"] + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + + +def test_runs_api_marks_run_pending_before_request_preparation(): + from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming, gateway_run_id_pending + + stream_id = "sid-run-starting-prep" + cancel_event = threading.Event() + observed = {"pending_during_prepare": False, "pending_during_post": False} + + class _JsonResponse: + def __init__(self, payload): + self._payload = json.dumps(payload).encode("utf-8") + + def read(self, _limit=None): + return self._payload + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + class _SseResponse: + def __iter__(self): + return iter([ + b'data: {"event":"run.completed","output":"Hello"}\n', + b'\n', + ]) + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def fake_strip(text): + observed["pending_during_prepare"] = gateway_run_id_pending(stream_id) + return text + + def fake_urlopen(req, *, timeout=None): + if req.full_url.endswith("/v1/runs"): + observed["pending_during_post"] = gateway_run_id_pending(stream_id) + return _JsonResponse({"run_id": "run-starting"}) + if req.full_url.endswith("/events"): + return _SseResponse() + raise AssertionError(f"unexpected urlopen: {req.full_url}") + + try: + with patch("urllib.request.urlopen", side_effect=fake_urlopen), \ + patch("api.streaming._strip_oob_blocks", side_effect=fake_strip): + final_text, usage = _run_gateway_runs_api_streaming( + session_id="sess-run-starting-prep", + msg_text="hi", + model="test-model", + workspace="/tmp", + stream_id=stream_id, + base_url="http://gw:8642", + api_key="secret", + prefill_messages=[], + body_extras={}, + put_gateway_event=lambda *_args, **_kwargs: None, + cancel_event=cancel_event, + cfg={}, + session=SimpleNamespace(context_messages=[{"role": "user", "content": "ctx"}]), + ) + assert final_text == "Hello" + assert usage == {} + assert observed["pending_during_prepare"] is True + assert observed["pending_during_post"] is True + assert gateway_run_id_pending(stream_id) is False + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + + +def test_gateway_worker_marks_run_pending_before_runs_api_prelude(): + from api.gateway_chat import STREAMS, _run_gateway_chat_streaming, gateway_run_id_pending + + stream_id = "sid-worker-run-starting" + observed = {"pending_during_support_check": False, "pending_during_runs_call": False} + STREAMS[stream_id] = SimpleNamespace(put_nowait=lambda *_args, **_kwargs: None) + session = SimpleNamespace( + profile=None, + workspace="/tmp", + context_messages=[], + messages=[], + active_stream_id=stream_id, + pending_user_source="webui", + process_wakeup_pause={}, + save=lambda: None, + ) + + def fake_supports(_base_url, _api_key): + observed["pending_during_support_check"] = gateway_run_id_pending(stream_id) + return True + + def fake_runs(*_args, **_kwargs): + observed["pending_during_runs_call"] = gateway_run_id_pending(stream_id) + return None, {} + + with patch("api.gateway_chat.RunJournalWriter", return_value=SimpleNamespace(append_sse_event=lambda *_a, **_k: None)), \ + patch("api.gateway_chat.get_session", return_value=session), \ + patch("api.config.get_config", return_value={}), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.gateway_chat._gateway_use_runs_api_enabled", return_value=True), \ + patch("api.gateway_chat.gateway_supports_approval", side_effect=fake_supports), \ + patch("api.gateway_chat._gateway_reasoning_effort_for_request", return_value=None), \ + patch("api.gateway_chat._run_gateway_runs_api_streaming", side_effect=fake_runs), \ + patch("api.streaming._load_webui_prefill_context", return_value={}), \ + patch("api.streaming._prefill_messages_with_webui_context", return_value=[]), \ + patch("api.streaming._normalize_prefill_messages_before_user_turn", side_effect=lambda messages: messages), \ + patch("api.streaming._public_prefill_context_status", return_value={}), \ + patch("api.streaming._webui_ephemeral_system_prompt", return_value="sys"): + _run_gateway_chat_streaming( + session_id="sess-worker-run-starting", + msg_text="hi", + model="test-model", + workspace="/tmp", + stream_id=stream_id, + ) + + assert observed["pending_during_support_check"] is True + assert observed["pending_during_runs_call"] is True + assert gateway_run_id_pending(stream_id) is False + + +def test_local_stop_rejects_redirected_success(): + from api.gateway_chat import _STREAM_RUN_IDS, stop_gateway_run + + stream_id = "stream-stop-redirect" + _STREAM_RUN_IDS[stream_id] = "run-stop" + + class _Response: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def geturl(self): + return "http://gw:8642/login" + + class _Opener: + def open(self, req, *, timeout=None): + return _Response() + + try: + with patch("urllib.request.build_opener", return_value=_Opener()), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=None): + assert stop_gateway_run(stream_id) is False + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + + +def test_chat_cancel_surfaces_redirected_gateway_stop(monkeypatch): + import urllib.parse + + from api import routes + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + sid = "sess-cancel-stop-redirect" + stream_id = "stream-cancel-stop-redirect" + _STREAM_RUN_IDS[stream_id] = "run-stop" + approvals._pending.pop(sid, None) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-stop", "command": "first"}) + captured = {} + called = {"cancel": False} + + monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) + monkeypatch.setattr(routes, "webui_gateway_chat_enabled", lambda *_args, **_kwargs: True) + monkeypatch.setattr(routes, "stream_owner_session_id", lambda _stream_id: sid) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True)) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _stream_id: False) + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(routes, "j", fake_j) + parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") + try: + routes.handle_get(object(), parsed) + assert captured["status"] == 502 + assert captured["payload"] == { + "ok": False, + "cancelled": False, + "stream_id": stream_id, + "error": "Gateway stop failed", + } + assert called["cancel"] is False + assert sid in approvals._pending + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + + +def test_chat_cancel_retires_same_run_gateway_mirrors_after_stop(monkeypatch): + import urllib.parse + + from api import routes + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + sid = "sess-cancel-stop" + stream_id = "stream-cancel-stop" + _STREAM_RUN_IDS[stream_id] = "run-stop" + approvals._pending.pop(sid, None) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-stop", "command": "first"}) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-stop", "command": "second"}) + captured = {} + + monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) + monkeypatch.setattr(routes, "webui_gateway_chat_enabled", lambda *_args, **_kwargs: True) + monkeypatch.setattr(routes, "stream_owner_session_id", lambda _stream_id: sid) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: True) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _stream_id: True) + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(routes, "j", fake_j) + parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") + try: + routes.handle_get(object(), parsed) + assert captured["status"] == 200 + assert captured["payload"] == {"ok": True, "cancelled": True, "stream_id": stream_id} + assert sid not in approvals._pending + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + + +def test_chat_cancel_surfaces_gateway_stop_failure(monkeypatch): + import urllib.parse + + from api import routes + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + sid = "sess-cancel-stop-fail" + stream_id = "stream-cancel-stop-fail" + _STREAM_RUN_IDS[stream_id] = "run-stop" + approvals._pending.pop(sid, None) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-stop", "command": "first"}) + captured = {} + called = {"cancel": False} + + monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) + monkeypatch.setattr(routes, "webui_gateway_chat_enabled", lambda *_args, **_kwargs: True) + monkeypatch.setattr(routes, "stream_owner_session_id", lambda _stream_id: sid) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True)) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _stream_id: False) + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(routes, "j", fake_j) + parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") + try: + routes.handle_get(object(), parsed) + assert captured["status"] == 502 + assert captured["payload"] == { + "ok": False, + "cancelled": False, + "stream_id": stream_id, + "error": "Gateway stop failed", + } + assert called["cancel"] is False + assert sid in approvals._pending + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + + +def test_chat_cancel_blocks_while_gateway_run_id_is_pending(monkeypatch): + import urllib.parse + + from api import routes + + stream_id = "stream-cancel-pending-run-id" + captured = {} + called = {"cancel": False, "stop": False} + + monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True)) + monkeypatch.setattr("api.gateway_chat.gateway_run_id_pending", lambda _stream_id: True) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _stream_id: called.__setitem__("stop", True)) + monkeypatch.setattr("time.sleep", lambda _seconds: None) + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(routes, "j", fake_j) + parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") + routes.handle_get(object(), parsed) + assert captured["status"] == 502 + assert captured["payload"] == { + "ok": False, + "cancelled": False, + "stream_id": stream_id, + "error": "Gateway stop failed", + } + assert called["cancel"] is False + assert called["stop"] is False + + +def test_chat_cancel_waits_for_pending_gateway_run_id_then_stops_remote_run(monkeypatch): + import urllib.parse + + from api import routes + from api.gateway_chat import _STREAM_RUN_IDS + + stream_id = "stream-cancel-pending-wait" + captured = {} + called = {"cancel": False, "stop": False, "pending": True} + + def fake_sleep(_seconds): + _STREAM_RUN_IDS[stream_id] = "run-stop" + called["pending"] = False + + monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) + monkeypatch.setattr("api.gateway_chat.gateway_run_id_pending", lambda _stream_id: called["pending"]) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _stream_id: called.__setitem__("stop", True) or True) + monkeypatch.setattr("time.sleep", fake_sleep) + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(routes, "j", fake_j) + parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") + try: + routes.handle_get(object(), parsed) + assert captured["status"] == 200 + assert captured["payload"] == {"ok": True, "cancelled": True, "stream_id": stream_id} + assert called["stop"] is True + assert called["cancel"] is True + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + + def test_stale_gateway_card_does_not_relay_live_run(): from api.gateway_chat import _STREAM_RUN_IDS import api.route_approvals as approvals @@ -581,6 +1479,101 @@ def test_retained_gateway_card_does_not_relay_different_active_run(): approvals._pending.pop(sid, None) +def test_stale_same_run_gateway_card_does_not_relay_live_head(): + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + sid = "sess-stale-same-run" + stream_id = "sid-stale-same-run" + _STREAM_RUN_IDS[stream_id] = "run-shared" + approvals._pending.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-shared", "command": "first"}) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-shared", "command": "second"}) + queue = approvals._pending[sid] + stale_id = queue[0]["approval_id"] + live_id = queue[1]["approval_id"] + assert approvals.retire_gateway_pending_mirror(sid, approval_id=stale_id, run_id="run-shared") + + mock_session = MagicMock() + mock_session.active_stream_id = stream_id + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.runner_client.HttpRunnerClient._request_json") as request_json: + from api.routes import _handle_approval_respond + _handle_approval_respond(handler, { + "session_id": sid, + "choice": "once", + "approval_id": stale_id, + }) + + payload = json.loads(handler.wfile.getvalue().decode("utf-8")) + assert handler.send_response.call_args.args[0] == 409 + assert payload["code"] == "gateway_run_unavailable" + request_json.assert_not_called() + assert approvals.gateway_pending_mirror(sid, approval_id=live_id) is not None + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + + +def test_token_stale_same_run_gateway_card_does_not_relay_advanced_live_head(): + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + sid = "sess-token-stale-same-run" + stream_id = "sid-token-stale-same-run" + _STREAM_RUN_IDS[stream_id] = "run-shared" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._gateway_queues[sid] = [SimpleNamespace(data={ + "run_id": "run-shared", + "approval_id": "gwrun:run-shared:a", + "command": "first", + })] + approvals.submit_gateway_pending_mirror(sid, dict(approvals._gateway_queues[sid][0].data)) + stale_id = approvals._pending[sid][0]["approval_id"] + + approvals._gateway_queues[sid] = [SimpleNamespace(data={ + "run_id": "run-shared", + "approval_id": "gwrun:run-shared:b", + "command": "second", + })] + approvals.submit_gateway_pending_mirror(sid, dict(approvals._gateway_queues[sid][0].data)) + + mock_session = MagicMock() + mock_session.active_stream_id = stream_id + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.runner_client.HttpRunnerClient._request_json") as request_json: + from api.routes import _handle_approval_respond + _handle_approval_respond(handler, { + "session_id": sid, + "choice": "once", + "approval_id": stale_id, + }) + + payload = json.loads(handler.wfile.getvalue().decode("utf-8")) + assert handler.send_response.call_args.args[0] == 409 + assert payload["code"] == "gateway_run_unavailable" + request_json.assert_not_called() + assert approvals.gateway_pending_mirror(sid, approval_id="gwrun:run-shared:a") is None + assert approvals.gateway_pending_mirror(sid, approval_id="gwrun:run-shared:b") is not None + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + def test_gateway_runs_api_streaming_preserves_multimodal_input(): """Attachment-backed runs requests must keep multimodal content lists.""" from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming @@ -752,8 +1745,8 @@ def test_gateway_approval_response_relay(): approvals._pending.pop("sess-relay", None) -def test_gateway_approval_response_without_approval_id_routes_by_run_id(): - """Runs API approvals resolve by the server-owned run ID.""" +def test_gateway_approval_response_without_approval_id_409s(): + """Gateway relay requires the emitted per-approval id, not bare run state.""" from api.gateway_chat import _STREAM_RUN_IDS import api.route_approvals as approvals @@ -764,20 +1757,13 @@ def test_gateway_approval_response_without_approval_id_routes_by_run_id(): mock_session = MagicMock() mock_session.active_stream_id = "sid-relay-empty-id" - captured = {} - - def fake_request_json(self, req): - captured["url"] = req.full_url - captured["body"] = json.loads(req.data) - return {"ok": True} - handler = MagicMock() handler.wfile = io.BytesIO() with patch("api.routes.get_session", return_value=mock_session), \ - patch("api.runner_client.HttpRunnerClient._request_json", new=fake_request_json), \ patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.runner_client.HttpRunnerClient._request_json") as request_json, \ patch("api.routes._resolve_approval_legacy", return_value=True) as cleanup: from api.routes import _handle_approval_respond _handle_approval_respond( @@ -787,41 +1773,17 @@ def test_gateway_approval_response_without_approval_id_routes_by_run_id(): payload = json.loads(handler.wfile.getvalue().decode("utf-8")) status = handler.send_response.call_args.args[0] - assert status == 200, f"status={status} payload={payload} outbound={captured}" - assert captured["url"] == "http://gw:8642/v1/runs/run%20abc%2F1/approval" - assert captured["body"] == {"choice": "once", "approval_id": "gwrun:run abc/1"} - assert payload["ok"] is True - assert payload["relayed"] is True - cleanup.assert_called_once_with("sess-relay", "gwrun:run abc/1", "once") + assert status == 409, f"status={status} payload={payload}" + assert payload["ok"] is False + assert payload["relayed"] is False + assert payload["code"] == "gateway_run_unavailable" + request_json.assert_not_called() + cleanup.assert_not_called() _STREAM_RUN_IDS.pop("sid-relay-empty-id", None) approvals._pending.pop("sess-relay", None) -def test_gateway_mirrored_pending_approval_id_by_run_id_skips_older_local_entries(): - """Run-scoped cleanup must target the mirrored gateway card, not queue head.""" - import api.route_approvals as approvals - - sid = "sess-relay-mirror" - approvals._pending.pop(sid, None) - approvals._gateway_queues.pop(sid, None) - try: - approvals._pending[sid] = [{"approval_id": "local-1", "command": "echo local"}] - approvals._gateway_queues[sid] = [SimpleNamespace(data={ - "approval_id": "gateway-real", - "command": "echo gateway", - "run_id": "run-xyz", - })] - approvals.submit_gateway_pending_mirror(sid, {}) - queue = approvals._pending[sid] - assert queue[0]["approval_id"] == "local-1" - assert queue[1]["approval_id"] == "gateway-real" - assert approvals._gateway_mirrored_pending_approval_id_by_run_id(sid, "run-xyz") == "gateway-real" - finally: - approvals._pending.pop(sid, None) - approvals._gateway_queues.pop(sid, None) - - def test_gateway_approval_response_relay_failure_returns_502(): """Gateway relay failures must surface as HTTP errors to the frontend.""" from api.gateway_chat import _STREAM_RUN_IDS diff --git a/tests/test_issue4771_local_approval_regression.py b/tests/test_issue4771_local_approval_regression.py index 2718c18e..7b62795a 100644 --- a/tests/test_issue4771_local_approval_regression.py +++ b/tests/test_issue4771_local_approval_regression.py @@ -40,6 +40,7 @@ import pytest # Import order matters: tools.* will not resolve until api.config has run. from api import routes from api import models +from api.gateway_chat import _STREAM_RUN_IDS try: import tools.approval as ta @@ -182,6 +183,101 @@ def test_local_mirrored_approval_deny_resolves(): _cleanup(sid) +def test_local_mirrored_approval_with_gwrun_prefix_still_resolves(): + """A local approval id that happens to start with gwrun: stays local.""" + sid = f"local-approval-prefixed-{uuid.uuid4().hex[:8]}" + _register_session(sid) + try: + entry, _approval_id = _seed_local_pending_approval(sid) + with ta._lock: + head = ta._pending[sid][0] + head["approval_id"] = "gwrun:local-test" + handler = _FakeHandler() + with patch("api.gateway_chat.webui_gateway_chat_enabled", return_value=False): + routes._handle_approval_respond( + handler, + {"session_id": sid, "choice": "once", "approval_id": "gwrun:local-test"}, + ) + resp = handler.json() + assert handler.status == 200, f"expected 200, got {handler.status}: {resp}" + assert resp.get("ok") is True + assert resp.get("code") != "gateway_run_unavailable" + assert entry.event.is_set() + assert entry.result == "once" + finally: + _cleanup(sid) + + +def test_local_live_head_does_not_inherit_stale_remote_run(): + """A stale remote mirror cannot seize a live local approval with the same id.""" + sid = f"local-approval-collision-{uuid.uuid4().hex[:8]}" + _register_session(sid) + try: + entry, approval_id = _seed_local_pending_approval(sid) + with ta._lock: + local_queue = ta._pending.get(sid) + local_entries = list(local_queue) if isinstance(local_queue, list) else [local_queue] + ta._pending[sid] = [{ + ra._GATEWAY_MIRROR_FLAG: True, + "run_id": "remote-run", + "approval_id": approval_id, + "command": "stale-remote", + }, *local_entries] + handler = _FakeHandler() + with patch("api.gateway_chat.webui_gateway_chat_enabled", return_value=False), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond_approval: + routes._handle_approval_respond( + handler, + {"session_id": sid, "choice": "once", "approval_id": approval_id}, + ) + resp = handler.json() + assert handler.status == 200, f"expected 200, got {handler.status}: {resp}" + assert resp.get("ok") is True + assert resp.get("relayed") is not True + respond_approval.assert_not_called() + assert entry.event.is_set() + assert entry.result == "once" + finally: + _cleanup(sid) + + +def test_local_live_head_beats_active_remote_candidate_run(): + """A local approval still wins when an active remote run shares its id.""" + sid = f"local-approval-active-remote-{uuid.uuid4().hex[:8]}" + session = _register_session(sid) + stream_id = f"remote-stream-{uuid.uuid4().hex[:8]}" + session.active_stream_id = stream_id + _STREAM_RUN_IDS[stream_id] = "remote-run" + try: + entry, approval_id = _seed_local_pending_approval(sid) + with ta._lock: + local_queue = ta._pending.get(sid) + local_entries = list(local_queue) if isinstance(local_queue, list) else [local_queue] + ta._pending[sid] = [{ + ra._GATEWAY_MIRROR_FLAG: True, + "run_id": "remote-run", + "approval_id": approval_id, + "command": "remote-collision", + }, *local_entries] + handler = _FakeHandler() + with patch("api.gateway_chat.webui_gateway_chat_enabled", return_value=False), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond_approval: + routes._handle_approval_respond( + handler, + {"session_id": sid, "choice": "once", "approval_id": approval_id}, + ) + resp = handler.json() + assert handler.status == 200, f"expected 200, got {handler.status}: {resp}" + assert resp.get("ok") is True + assert resp.get("relayed") is not True + respond_approval.assert_not_called() + assert entry.event.is_set() + assert entry.result == "once" + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + _cleanup(sid) + + def test_gateway_mirrored_approval_without_run_still_409s(): """#4771 behaviour preserved: on the gateway backend, a mirrored approval whose run is gone must still surface the actionable 409 so the card stays diff --git a/tests/test_issue6008_sidebar_stop_failure_ui.py b/tests/test_issue6008_sidebar_stop_failure_ui.py new file mode 100644 index 00000000..6965ab3b --- /dev/null +++ b/tests/test_issue6008_sidebar_stop_failure_ui.py @@ -0,0 +1,144 @@ +"""Sidebar stop must not settle the UI when /api/chat/cancel fails.""" +from __future__ import annotations + +import json +import pathlib +import re +import subprocess + + +REPO = pathlib.Path(__file__).parent.parent +BOOT_JS = (REPO / "static" / "boot.js").read_text(encoding="utf-8") +COMMANDS_JS = (REPO / "static" / "commands.js").read_text(encoding="utf-8") +MESSAGES_JS = (REPO / "static" / "messages.js").read_text(encoding="utf-8") +SESSIONS_JS = (REPO / "static" / "sessions.js").read_text(encoding="utf-8") + + +def _extract_function(src: str, name: str) -> str: + m = re.search(rf"async function {name}\s*\(", src) + assert m, f"{name} not found in static/boot.js" + brace_pos = src.index("{", m.end()) + depth = 1 + pos = brace_pos + 1 + while pos < len(src) and depth > 0: + ch = src[pos] + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + pos += 1 + return src[m.start():pos] + + +CANCEL_SESSION_STREAM_SRC = _extract_function(BOOT_JS, "cancelSessionStream") + + +def test_source_gates_sidebar_settle_on_http_success(): + assert "return false" in CANCEL_SESSION_STREAM_SRC + assert "return true" in CANCEL_SESSION_STREAM_SRC + assert "r.ok" in CANCEL_SESSION_STREAM_SRC, ( + "cancelSessionStream() must check the /api/chat/cancel HTTP status before " + "closing local UI state" + ) + assert "if(!respOk)returnfalse;" in "".join(CANCEL_SESSION_STREAM_SRC.split()), ( + "cancelSessionStream() must bail out on failed stop responses" + ) + + +def test_stop_callers_gate_success_toasts_on_cancel_result(): + compact_commands = "".join(COMMANDS_JS.split()) + compact_messages = "".join(MESSAGES_JS.split()) + compact_sessions = "".join(SESSIONS_JS.split()) + assert "if(awaitcancelStream('slash-stop'))showToast(t('stream_stopped'))" in compact_commands + assert "elseshowToast(t('cancel_failed'))" in compact_commands + assert "if(awaitcancelStream('slash-interrupt'))showToast(t('cmd_interrupt_confirm'),2000)" in compact_commands + assert "if(awaitcancelStream('busy-interrupt'))showToast(t('busy_interrupt_confirm'),2000)" in compact_messages + assert "if(awaitcancelSessionStream(session))showToast(t('stream_stopped'))" in compact_sessions + + +_NODE_SCRIPT = r''' +const M = { + closeCalls: [], + busyCalls: [], + composerCalls: [], + statusCalls: [], + renderCalls: 0, + clearCalls: [], + approvalStops: 0, + approvalHides: 0, + clarifyStops: 0, + clarifyHides: 0, + fetchCalls: [], +}; + +globalThis.INFLIGHT = { 'sid-1': { streamId: 'stream-1' } }; +globalThis.S = { activeStreamId: 'stream-1', session: { session_id: 'sid-1', active_stream_id: 'stream-1' } }; +globalThis.closeLiveStream = (...a) => M.closeCalls.push(a); +globalThis.clearInflightState = (sid) => M.clearCalls.push(['clearInflightState', sid]); +globalThis.clearInflight = () => M.clearCalls.push(['clearInflight']); +globalThis.setBusy = (v) => M.busyCalls.push(v); +globalThis.setComposerStatus = (v) => M.composerCalls.push(v); +globalThis.setStatus = (v) => M.statusCalls.push(v); +globalThis.stopApprovalPolling = () => M.approvalStops += 1; +globalThis.hideApprovalCard = () => M.approvalHides += 1; +globalThis.stopClarifyPolling = () => M.clarifyStops += 1; +globalThis.hideClarifyCard = () => M.clarifyHides += 1; +globalThis.renderSessionList = () => M.renderCalls += 1; +globalThis._approvalSessionId = 'sid-1'; +globalThis._clarifySessionId = 'sid-1'; +globalThis.document = { baseURI: 'http://localhost:8787/' }; +globalThis.location = { href: 'http://localhost:8787/' }; +globalThis.fetch = (url, opts) => { + M.fetchCalls.push({ url: String(url), opts }); + return Promise.resolve({ + ok: false, + json: () => Promise.resolve({ ok: false, cancelled: false, stream_id: 'stream-1' }), + }); +}; + +__CANCEL_SESSION_STREAM_SRC__ + +const session = { session_id: 'sid-1', active_stream_id: 'stream-1' }; +await cancelSessionStream(session); +console.log(JSON.stringify({ + sessionActiveStreamId: session.active_stream_id, + activeStreamId: globalThis.S.activeStreamId, + closeCalls: M.closeCalls, + busyCalls: M.busyCalls, + composerCalls: M.composerCalls, + renderCalls: M.renderCalls, + clearCalls: M.clearCalls, + approvalStops: M.approvalStops, + approvalHides: M.approvalHides, + clarifyStops: M.clarifyStops, + clarifyHides: M.clarifyHides, + fetchCalls: M.fetchCalls.length, +})); +''' + + +def test_failed_sidebar_stop_keeps_local_state(): + script = _NODE_SCRIPT.replace("__CANCEL_SESSION_STREAM_SRC__", CANCEL_SESSION_STREAM_SRC) + completed = subprocess.run( + ["node", "--input-type=module", "-e", script], + cwd=str(REPO), + text=True, + capture_output=True, + timeout=30, + ) + assert completed.returncode == 0, ( + f"node subprocess failed:\n--- stdout ---\n{completed.stdout}\n--- stderr ---\n{completed.stderr}" + ) + result = json.loads(completed.stdout.splitlines()[-1]) + assert result["fetchCalls"] == 1 + assert result["sessionActiveStreamId"] == "stream-1" + assert result["activeStreamId"] == "stream-1" + assert result["closeCalls"] == [] + assert result["busyCalls"] == [] + assert result["composerCalls"] == [] + assert result["renderCalls"] == 0 + assert result["clearCalls"] == [] + assert result["approvalStops"] == 0 + assert result["approvalHides"] == 0 + assert result["clarifyStops"] == 0 + assert result["clarifyHides"] == 0 From 6460c48ce69316a7ab4674a3ed7095e19ebbc203 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Thu, 16 Jul 2026 04:37:50 -0400 Subject: [PATCH 04/20] fix(#6008): keep gateway approvals attached to the right run --- api/route_approvals.py | 63 +++- api/routes.py | 85 ++++- tests/test_gateway_approval_runs_api.py | 313 +++++++++++++++++- ...est_issue4771_local_approval_regression.py | 103 ++++++ tests/test_runtime_adapter_seam.py | 2 +- 5 files changed, 546 insertions(+), 20 deletions(-) diff --git a/api/route_approvals.py b/api/route_approvals.py index 6ab739d8..f524e852 100644 --- a/api/route_approvals.py +++ b/api/route_approvals.py @@ -47,6 +47,7 @@ _approval_sse_subscribers: dict[str, list[queue.Queue]] = {} _GATEWAY_MIRROR_FLAG = "_gateway_mirror" _GATEWAY_MIRROR_TOKEN = "_gateway_mirror_token" _GATEWAY_ENTRY_DATA_TOKEN_KEY = "_webui_mirror_token" +_GATEWAY_ENTRY_DATA_LOCAL_HEAD_KEY = "_webui_local_head" def _approval_sse_subscribe(session_id: str) -> queue.Queue: @@ -106,7 +107,7 @@ def _approval_sse_notify(session_id: str, head: dict | None, total: int) -> None _approval_sse_notify_locked(session_id, head, total) -def _gateway_mirror_entry_token(entry) -> str: +def _gateway_mirror_entry_token(entry) -> str | None: """Return a stable token for the current process lifetime of a gateway head. Stamps a token key into the entry's `.data` dict so @@ -115,12 +116,14 @@ def _gateway_mirror_entry_token(entry) -> str: """ data = getattr(entry, "data", None) if isinstance(data, dict): + if data.get(_GATEWAY_ENTRY_DATA_LOCAL_HEAD_KEY): + return None token = data.get(_GATEWAY_ENTRY_DATA_TOKEN_KEY) if not token: token = uuid.uuid4().hex data[_GATEWAY_ENTRY_DATA_TOKEN_KEY] = token return token - return uuid.uuid4().hex + return None def _is_gateway_mirror_entry(entry: dict | None) -> bool: @@ -203,6 +206,13 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No live_mirror_present = True continue + if not live_token: + if entry_token: + changed = True + continue + rebuilt.append(entry) + continue + changed = True if live_token and not live_mirror_present: @@ -305,6 +315,9 @@ def retire_gateway_pending_mirror(session_key: str, approval_id: str = "", run_i continue retained_gateway_queue.append(entry) if not retired and not gateway_queue_changed: + if approval_id: + head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) + _approval_sse_notify_locked(session_key, head, total) return False for match in retired: entries.remove(match) @@ -339,11 +352,30 @@ def submit_gateway_pending_mirror(session_key: str, approval: dict) -> None: """Mirror the live gateway head into WebUI polling state under a typed tag.""" with _lock: run_id = str(approval.get("run_id") or "").strip() + approval_id = str(approval.get("approval_id") or "").strip() + live_gateway_queue = _gateway_queues.get(session_key) or [] + live_head_entry = live_gateway_queue[0] if live_gateway_queue else None + live_head_data = getattr(live_head_entry, "data", None) or {} + matching_no_run_live_head = ( + not run_id + and live_head_data + and str(live_head_data.get("command") or "") == str(approval.get("command") or "") + and str(live_head_data.get("description") or "") == str(approval.get("description") or "") + and str(live_head_data.get("pattern_key") or "") == str(approval.get("pattern_key") or "") + and tuple(str(k) for k in (live_head_data.get("pattern_keys") or [])) + == tuple(str(k) for k in (approval.get("pattern_keys") or [])) + ) + if matching_no_run_live_head and not live_head_data.get(_GATEWAY_ENTRY_DATA_TOKEN_KEY): + live_head_data.pop(_GATEWAY_ENTRY_DATA_LOCAL_HEAD_KEY, None) + live_head_data[_GATEWAY_ENTRY_DATA_TOKEN_KEY] = uuid.uuid4().hex + elif not run_id and live_head_data: + live_head_data[_GATEWAY_ENTRY_DATA_LOCAL_HEAD_KEY] = True + if not run_id and not approval_id: + approval_id = uuid.uuid4().hex + approval["approval_id"] = approval_id + if matching_no_run_live_head and approval_id and not str(live_head_data.get("approval_id") or "").strip(): + live_head_data["approval_id"] = approval_id if run_id: - live_gateway_queue = _gateway_queues.get(session_key) or [] - approval_id = str(approval.get("approval_id") or "").strip() - live_head_entry = live_gateway_queue[0] if live_gateway_queue else None - live_head_data = getattr(live_head_entry, "data", None) or {} live_head_run_id = str(live_head_data.get("run_id") or "").strip() live_head_approval_id = str(live_head_data.get("approval_id") or "").strip() live_token = ( @@ -386,6 +418,25 @@ def submit_gateway_pending_mirror(session_key: str, approval: dict) -> None: ) if mirror_entry: approval["approval_id"] = str(mirror_entry.get("approval_id") or approval_id).strip() + elif approval_id: + queue = _pending.get(session_key) + entries = queue if isinstance(queue, list) else [queue] if queue else [] + no_run_mirror = next( + ( + entry for entry in reversed(entries) + if _is_gateway_mirror_entry(entry) + and not str(entry.get("run_id") or "").strip() + and str(entry.get("approval_id") or "").strip() == approval_id + ), + None, + ) + if no_run_mirror: + approval["approval_id"] = str(no_run_mirror.get("approval_id") or approval_id).strip() + elif not _gateway_pending_mirror_locked(session_key, approval_id=approval_id): + mirror_entry = dict(approval) + mirror_entry["approval_id"] = approval_id + mirror_entry[_GATEWAY_MIRROR_FLAG] = True + _normalize_pending_queue_locked(session_key).append(mirror_entry) _approval_sse_notify_locked(session_key, head, total) publish_session_list_changed("attention_pending") diff --git a/api/routes.py b/api/routes.py index 9151abf4..016df3a6 100644 --- a/api/routes.py +++ b/api/routes.py @@ -9715,6 +9715,8 @@ from api.route_approvals import ( # noqa: F401 — re-exports for backward comp _approval_sse_notify_locked, _approval_sse_notify, _GATEWAY_MIRROR_FLAG, + _GATEWAY_MIRROR_TOKEN, + _gateway_mirror_entry_token, gateway_pending_mirror, retire_gateway_pending_mirror, reconcile_gateway_pending_mirror_locked, @@ -23806,15 +23808,24 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str, run_id: st queue = _pending.get(sid) if isinstance(queue, list): if approval_id: - # Find and remove the specific entry by approval_id. + # Prefer a local exact-id match over a mirrored one, so a + # preceding remote mirror cannot consume the user's local choice. + preferred_index = None + fallback_index = None for i, entry in enumerate(queue): if entry.get("approval_id") != approval_id: continue if run_id and str(entry.get("run_id") or "").strip() != run_id: continue - pending = queue.pop(i) + if not entry.get(_GATEWAY_MIRROR_FLAG) or not str(entry.get("run_id") or "").strip(): + preferred_index = i + break + if fallback_index is None: + fallback_index = i + match_index = preferred_index if preferred_index is not None else fallback_index + if match_index is not None: + pending = queue.pop(match_index) found_target = True - break else: # A stale explicit id must not accidentally approve the # oldest queued command; duplicate/stale responses are @@ -23993,10 +24004,11 @@ def _handle_approval_respond(handler, body): _candidate_run_id = _STREAM_RUN_IDS.get(active_sid) local_match = False run_backed_gateway_matches = 0 - if approval_id: - with _lock: - queue = _pending.get(sid) - entries = queue if isinstance(queue, list) else [queue] if queue else [] + same_run_stale_without_token = False + with _lock: + queue = _pending.get(sid) + entries = queue if isinstance(queue, list) else [queue] if queue else [] + if approval_id: local_match = any( isinstance(entry, dict) and entry.get("approval_id") == approval_id @@ -24014,11 +24026,60 @@ def _handle_approval_respond(handler, body): and entry.get(_GATEWAY_MIRROR_FLAG) and str(entry.get("run_id") or "").strip() ) - if local_match: - _candidate_run_id = None - matched_mirror = gateway_pending_mirror( - sid, approval_id=approval_id, run_id=_candidate_run_id - ) if approval_id else None + gateway_queue = _gateway_queues.get(sid) or [] + live_head_data = getattr(gateway_queue[0], "data", None) or {} if gateway_queue else {} + live_head_run_id = str(live_head_data.get("run_id") or "").strip() + live_head_token = ( + _gateway_mirror_entry_token(gateway_queue[0]) + if gateway_queue and live_head_data + else None + ) + live_head_approval_id = str(live_head_data.get("approval_id") or "").strip() + if not live_head_approval_id and live_head_token and live_head_run_id: + live_head_approval_id = f"gwrun:{live_head_run_id}:{live_head_token}" + stale_same_run_id = _candidate_run_id or live_head_run_id + if ( + stale_same_run_id + and live_head_run_id == stale_same_run_id + and live_head_approval_id + and live_head_approval_id != approval_id + ): + same_run_stale_without_token = any( + isinstance(entry, dict) + and entry.get("approval_id") == approval_id + and entry.get(_GATEWAY_MIRROR_FLAG) + and str(entry.get("run_id") or "").strip() == stale_same_run_id + and not str(entry.get(_GATEWAY_MIRROR_TOKEN) or "").strip() + for entry in entries + ) + else: + local_match = any( + isinstance(entry, dict) + and ( + not entry.get(_GATEWAY_MIRROR_FLAG) + or not str(entry.get("run_id") or "").strip() + ) + for entry in entries + ) + if local_match: + _candidate_run_id = None + if approval_id and not local_match and same_run_stale_without_token: + return j( + handler, + { + "ok": False, + "choice": choice, + "relayed": False, + "code": "gateway_run_unavailable", + "error": _GATEWAY_APPROVAL_RELAY_UNAVAILABLE, + }, + status=409, + ) + matched_mirror = ( + gateway_pending_mirror(sid, approval_id=approval_id, run_id=_candidate_run_id) + if approval_id and not local_match + else None + ) _run_id = matched_mirror["run_id"] if matched_mirror else None if not matched_mirror and approval_id: if local_match: diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index 19cd3094..3ba15e9a 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -9,6 +9,8 @@ import urllib.error from types import SimpleNamespace from unittest.mock import MagicMock, patch +import pytest + # --------------------------------------------------------------------------- # 1. Capability detection @@ -603,7 +605,10 @@ def test_gateway_approval_relay_does_not_unblock_unrelated_local_head(): from api.gateway_chat import _STREAM_RUN_IDS from api import routes import api.route_approvals as approvals - import tools.approval as ta + ta = pytest.importorskip( + "tools.approval", + reason="tools.approval not available in this environment", + ) sid = "sess-remote-relay-local-head" stream_id = "sid-remote-relay-local-head" @@ -746,6 +751,132 @@ def test_direct_remote_approval_still_mirrors_with_local_gateway_head(): approvals._gateway_queues.pop(sid, None) +def test_legacy_gateway_approval_without_run_gets_browser_visible_id(): + import api.route_approvals as approvals + + sid = "sess-legacy-no-run-id" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._gateway_queues[sid] = [SimpleNamespace(data={"command": "legacy"})] + approval = {"command": "legacy"} + approvals.submit_gateway_pending_mirror(sid, approval) + assert approval.get("approval_id") + queue = approvals._pending.get(sid) or [] + assert queue[0]["approval_id"] == approval["approval_id"] + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_legacy_gateway_approval_without_run_keeps_its_own_id_beside_local_head(): + import api.route_approvals as approvals + + sid = "sess-legacy-no-run-local-head" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._pending[sid] = [{"approval_id": "local-id", "command": "local"}] + approvals._gateway_queues[sid] = [SimpleNamespace(data={"command": "legacy"})] + approval = {"command": "legacy"} + approvals.submit_gateway_pending_mirror(sid, approval) + assert approval.get("approval_id") + assert approval["approval_id"] != "local-id" + queue = approvals._pending.get(sid) or [] + mirrors = [ + entry for entry in queue + if entry.get(approvals._GATEWAY_MIRROR_FLAG) + and not str(entry.get("run_id") or "").strip() + ] + assert mirrors + assert mirrors[-1]["approval_id"] == approval["approval_id"] + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_legacy_gateway_approval_without_run_keeps_its_own_id_beside_local_gateway_head(): + import api.route_approvals as approvals + ta = pytest.importorskip( + "tools.approval", + reason="tools.approval not available in this environment", + ) + + sid = "sess-legacy-no-run-local-gateway-head" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._gateway_queues[sid] = [ta._ApprovalEntry({ + "approval_id": "local-id", + "command": "local-head", + })] + approval = {"approval_id": "remote-id", "command": "legacy"} + approvals.submit_gateway_pending_mirror(sid, approval) + assert approval["approval_id"] == "remote-id" + queue = approvals._pending.get(sid) or [] + mirrors = [ + entry for entry in queue + if entry.get(approvals._GATEWAY_MIRROR_FLAG) + and not str(entry.get("run_id") or "").strip() + and entry.get("approval_id") == "remote-id" + ] + assert mirrors + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_identical_legacy_gateway_approvals_without_run_keep_distinct_ids(): + import api.route_approvals as approvals + + sid = "sess-legacy-no-run-identical" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + first = {"command": "legacy"} + second = {"command": "legacy"} + approvals.submit_gateway_pending_mirror(sid, first) + approvals.submit_gateway_pending_mirror(sid, second) + assert first["approval_id"] != second["approval_id"] + queue = approvals._pending.get(sid) or [] + mirrored_ids = [ + entry.get("approval_id") + for entry in queue + if entry.get(approvals._GATEWAY_MIRROR_FLAG) + and not str(entry.get("run_id") or "").strip() + ] + assert mirrored_ids == [first["approval_id"], second["approval_id"]] + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_identical_legacy_gateway_approvals_with_explicit_ids_keep_distinct_ids(): + import api.route_approvals as approvals + + sid = "sess-legacy-no-run-identical-explicit" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + first = {"approval_id": "upstream-a", "command": "legacy"} + second = {"approval_id": "upstream-b", "command": "legacy"} + approvals.submit_gateway_pending_mirror(sid, first) + approvals.submit_gateway_pending_mirror(sid, second) + assert first["approval_id"] == "upstream-a" + assert second["approval_id"] == "upstream-b" + queue = approvals._pending.get(sid) or [] + mirrored_ids = [ + entry.get("approval_id") + for entry in queue + if entry.get(approvals._GATEWAY_MIRROR_FLAG) + and not str(entry.get("run_id") or "").strip() + ] + assert mirrored_ids == ["upstream-a", "upstream-b"] + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + def test_reconcile_does_not_bind_live_token_by_approval_id_alone(): import api.route_approvals as approvals @@ -874,6 +1005,30 @@ def test_exact_one_retirement_keeps_same_run_gateway_queue_state(): approvals._gateway_queues.pop(sid, None) +def test_retire_gateway_pending_mirror_notifies_reconciled_successor_when_target_missing(): + import api.route_approvals as approvals + + sid = "sess-retire-notify-successor" + approvals._pending.pop(sid, None) + try: + approvals._pending[sid] = [{ + approvals._GATEWAY_MIRROR_FLAG: True, + "run_id": "run-a", + "approval_id": "appr-b", + "command": "b", + }] + notifications = [] + + def fake_notify(_sid, head, total): + notifications.append((head["approval_id"] if head else None, total)) + + with patch.object(approvals, "_approval_sse_notify_locked", side_effect=fake_notify): + assert approvals.retire_gateway_pending_mirror(sid, approval_id="appr-a", run_id="run-a") is False + assert notifications[-1] == ("appr-b", 1) + finally: + approvals._pending.pop(sid, None) + + def test_local_stop_uses_runs_api_stop_endpoint(): from api.gateway_chat import _STREAM_RUN_IDS, stop_gateway_run @@ -1521,6 +1676,162 @@ def test_stale_same_run_gateway_card_does_not_relay_live_head(): approvals._pending.pop(sid, None) +def test_stale_same_run_gateway_card_without_token_does_not_relay_live_head(): + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + ta = pytest.importorskip( + "tools.approval", + reason="tools.approval not available in this environment", + ) + + sid = "sess-stale-same-run-no-token" + stream_id = "sid-stale-same-run-no-token" + _STREAM_RUN_IDS[stream_id] = "run-shared" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._pending[sid] = [{ + approvals._GATEWAY_MIRROR_FLAG: True, + "run_id": "run-shared", + "approval_id": "stale-a", + "command": "stale", + }] + approvals._gateway_queues[sid] = [ta._ApprovalEntry({ + "run_id": "run-shared", + "approval_id": "live-b", + "command": "live", + })] + + mock_session = MagicMock() + mock_session.active_stream_id = stream_id + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.runner_client.HttpRunnerClient._request_json") as request_json: + from api.routes import _handle_approval_respond + _handle_approval_respond(handler, { + "session_id": sid, + "choice": "once", + "approval_id": "stale-a", + }) + + payload = json.loads(handler.wfile.getvalue().decode("utf-8")) + assert handler.send_response.call_args.args[0] == 409 + assert payload["code"] == "gateway_run_unavailable" + request_json.assert_not_called() + assert approvals.gateway_pending_mirror(sid, approval_id="live-b", run_id="run-shared") is not None + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_stale_same_run_gateway_card_without_token_after_teardown_does_not_relay_live_head(): + import api.route_approvals as approvals + ta = pytest.importorskip( + "tools.approval", + reason="tools.approval not available in this environment", + ) + + sid = "sess-stale-same-run-no-token-teardown" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._pending[sid] = [{ + approvals._GATEWAY_MIRROR_FLAG: True, + "run_id": "run-shared", + "approval_id": "stale-a", + "command": "stale", + }] + approvals._gateway_queues[sid] = [ta._ApprovalEntry({ + "run_id": "run-shared", + "approval_id": "live-b", + "command": "live", + })] + + mock_session = MagicMock() + mock_session.active_stream_id = None + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.runner_client.HttpRunnerClient._request_json") as request_json: + from api.routes import _handle_approval_respond + _handle_approval_respond(handler, { + "session_id": sid, + "choice": "once", + "approval_id": "stale-a", + }) + + payload = json.loads(handler.wfile.getvalue().decode("utf-8")) + assert handler.send_response.call_args.args[0] == 409 + assert payload["code"] == "gateway_run_unavailable" + request_json.assert_not_called() + assert approvals.gateway_pending_mirror(sid, approval_id="live-b", run_id="run-shared") is not None + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_stale_same_run_gateway_card_without_token_does_not_relay_live_head_when_live_id_is_synthesized(): + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + ta = pytest.importorskip( + "tools.approval", + reason="tools.approval not available in this environment", + ) + + sid = "sess-stale-same-run-no-token-synthesized-live-id" + stream_id = "sid-stale-same-run-no-token-synthesized-live-id" + _STREAM_RUN_IDS[stream_id] = "run-shared" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals._pending[sid] = [{ + approvals._GATEWAY_MIRROR_FLAG: True, + "run_id": "run-shared", + "approval_id": "stale-a", + "command": "stale", + }] + approvals._gateway_queues[sid] = [ta._ApprovalEntry({ + "run_id": "run-shared", + "command": "live", + })] + + mock_session = MagicMock() + mock_session.active_stream_id = stream_id + handler = MagicMock() + handler.wfile = io.BytesIO() + + with patch("api.routes.get_session", return_value=mock_session), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.runner_client.HttpRunnerClient._request_json") as request_json: + from api.routes import _handle_approval_respond + _handle_approval_respond(handler, { + "session_id": sid, + "choice": "once", + "approval_id": "stale-a", + }) + + payload = json.loads(handler.wfile.getvalue().decode("utf-8")) + assert handler.send_response.call_args.args[0] == 409 + assert payload["code"] == "gateway_run_unavailable" + request_json.assert_not_called() + live_mirror = approvals.gateway_pending_mirror(sid, run_id="run-shared") + assert live_mirror is not None + assert live_mirror["approval_id"] != "stale-a" + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + def test_token_stale_same_run_gateway_card_does_not_relay_advanced_live_head(): from api.gateway_chat import _STREAM_RUN_IDS import api.route_approvals as approvals diff --git a/tests/test_issue4771_local_approval_regression.py b/tests/test_issue4771_local_approval_regression.py index 7b62795a..5a3da490 100644 --- a/tests/test_issue4771_local_approval_regression.py +++ b/tests/test_issue4771_local_approval_regression.py @@ -278,6 +278,109 @@ def test_local_live_head_beats_active_remote_candidate_run(): _cleanup(sid) +def test_local_plain_pending_exact_id_beats_remote_mirror(): + sid = f"local-plain-pending-id-{uuid.uuid4().hex[:8]}" + session = _register_session(sid) + stream_id = f"remote-stream-{uuid.uuid4().hex[:8]}" + session.active_stream_id = stream_id + _STREAM_RUN_IDS[stream_id] = "remote-run" + try: + with ta._lock: + ta._pending[sid] = [ + {"approval_id": "shared-id", "command": "local"}, + { + ra._GATEWAY_MIRROR_FLAG: True, + "run_id": "remote-run", + "approval_id": "shared-id", + "command": "remote", + }, + ] + handler = _FakeHandler() + with patch("api.gateway_chat.webui_gateway_chat_enabled", return_value=False), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond_approval, \ + patch.object(routes, "_resolve_approval_legacy", return_value=True) as resolve_legacy: + routes._handle_approval_respond( + handler, + {"session_id": sid, "choice": "once", "approval_id": "shared-id"}, + ) + resp = handler.json() + assert handler.status == 200, f"expected 200, got {handler.status}: {resp}" + assert resp.get("ok") is True + assert resp.get("relayed") is not True + resolve_legacy.assert_called_once_with(sid, "shared-id", "once") + respond_approval.assert_not_called() + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + _cleanup(sid) + + +def test_local_plain_pending_exact_id_beats_remote_first_queue_order(): + sid = f"local-plain-pending-remote-first-{uuid.uuid4().hex[:8]}" + session = _register_session(sid) + stream_id = f"remote-stream-{uuid.uuid4().hex[:8]}" + session.active_stream_id = stream_id + _STREAM_RUN_IDS[stream_id] = "remote-run" + try: + with ta._lock: + ta._pending[sid] = [ + { + ra._GATEWAY_MIRROR_FLAG: True, + "run_id": "remote-run", + "approval_id": "shared-id", + "command": "remote", + }, + {"approval_id": "shared-id", "command": "local"}, + ] + handler = _FakeHandler() + with patch("api.gateway_chat.webui_gateway_chat_enabled", return_value=False), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond_approval: + routes._handle_approval_respond( + handler, + {"session_id": sid, "choice": "once", "approval_id": "shared-id"}, + ) + resp = handler.json() + assert handler.status == 200, f"expected 200, got {handler.status}: {resp}" + assert resp.get("ok") is True + assert resp.get("relayed") is not True + respond_approval.assert_not_called() + with ta._lock: + queue = ta._pending.get(sid) or [] + assert len(queue) == 1 + assert queue[0].get("command") == "remote" + assert queue[0].get(ra._GATEWAY_MIRROR_FLAG) is True + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + _cleanup(sid) + + +def test_local_plain_pending_without_id_beats_active_remote_candidate_run(): + sid = f"local-plain-pending-no-id-{uuid.uuid4().hex[:8]}" + session = _register_session(sid) + stream_id = f"remote-stream-{uuid.uuid4().hex[:8]}" + session.active_stream_id = stream_id + _STREAM_RUN_IDS[stream_id] = "remote-run" + try: + with ta._lock: + ta._pending[sid] = [{"command": "local"}] + handler = _FakeHandler() + with patch("api.gateway_chat.webui_gateway_chat_enabled", return_value=False), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond_approval, \ + patch.object(routes, "_resolve_approval_legacy", return_value=True) as resolve_legacy: + routes._handle_approval_respond( + handler, + {"session_id": sid, "choice": "once"}, + ) + resp = handler.json() + assert handler.status == 200, f"expected 200, got {handler.status}: {resp}" + assert resp.get("ok") is True + assert resp.get("relayed") is not True + resolve_legacy.assert_called_once_with(sid, "", "once") + respond_approval.assert_not_called() + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + _cleanup(sid) + + def test_gateway_mirrored_approval_without_run_still_409s(): """#4771 behaviour preserved: on the gateway backend, a mirrored approval whose run is gone must still surface the actionable 409 so the card stays diff --git a/tests/test_runtime_adapter_seam.py b/tests/test_runtime_adapter_seam.py index ee3a5208..54ca5017 100644 --- a/tests/test_runtime_adapter_seam.py +++ b/tests/test_runtime_adapter_seam.py @@ -325,7 +325,7 @@ def test_approval_respond_does_not_fallback_to_oldest_when_explicit_id_is_stale( helper_body = src[helper_idx:src.index("def _handle_approval_respond", helper_idx)] assert "A stale explicit id must not accidentally approve" in helper_body - assert "if found_target or not approval_id:" in helper_body + assert "if not pending and not approval_id:" in helper_body stale_branch = helper_body[helper_body.index("else:", helper_body.index("for i, entry")):helper_body.index("else:\n pending = queue.pop(0)")] assert "pending = None" in stale_branch assert "queue.pop(0)" not in stale_branch From 5826d76da04a69a509408b43033c76d52c302344 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 19 Jul 2026 10:26:58 -0400 Subject: [PATCH 05/20] fix(#6008): publish the authoritative approval head --- api/gateway_chat.py | 14 +- api/route_approvals.py | 142 ++++++++++++++++----- api/routes.py | 75 ++++++++--- api/streaming.py | 20 +-- static/messages.js | 2 +- tests/test_approval_queue.py | 35 +++++ tests/test_approval_sse.py | 4 +- tests/test_approval_unblock.py | 48 ++++++- tests/test_gateway_approval_legacy_path.py | 26 ++-- tests/test_session_attention_badges.py | 19 +++ 10 files changed, 301 insertions(+), 84 deletions(-) diff --git a/api/gateway_chat.py b/api/gateway_chat.py index db157b68..f4e5158a 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -507,8 +507,8 @@ def _run_gateway_runs_api_streaming( if not approval_data.get("approval_id"): approval_data["approval_id"] = f"gwrun:{run_id}:{uuid.uuid4().hex}" from api.route_approvals import submit_gateway_pending_mirror - submit_gateway_pending_mirror(session_id, approval_data) - put_gateway_event("approval", approval_data) + head, total = submit_gateway_pending_mirror(session_id, approval_data) + put_gateway_event("approval", {**(head or approval_data), "pending_count": total}) sse_event = "message" continue if payload_event in {"tool.started", "tool.completed", "reasoning.available"}: @@ -989,7 +989,8 @@ def _run_gateway_chat_streaming( _STREAM_RUN_IDS[stream_id] = _approval_run_id try: from api.route_approvals import submit_gateway_pending_mirror - submit_gateway_pending_mirror(session_id, approval_data) + head, total = submit_gateway_pending_mirror(session_id, approval_data) + approval_data = {**(head or approval_data), "pending_count": total} except Exception: logger.debug("submit_gateway_pending_mirror failed", exc_info=True) put_gateway_event("approval", approval_data) @@ -1259,6 +1260,13 @@ def _run_gateway_chat_streaming( "hint": "Check HERMES_WEBUI_GATEWAY_BASE_URL and Gateway API server health.", }) finally: + mapped_run_id = str(_STREAM_RUN_IDS.get(stream_id) or "").strip() + if mapped_run_id: + try: + from api.route_approvals import retire_gateway_pending_mirror + retire_gateway_pending_mirror(session_id, run_id=mapped_run_id) + except Exception: + logger.debug("Failed to retire gateway pending mirrors during teardown", exc_info=True) if s is not None: try: with _get_session_agent_lock(session_id): diff --git a/api/route_approvals.py b/api/route_approvals.py index f524e852..41bb6e5c 100644 --- a/api/route_approvals.py +++ b/api/route_approvals.py @@ -47,7 +47,6 @@ _approval_sse_subscribers: dict[str, list[queue.Queue]] = {} _GATEWAY_MIRROR_FLAG = "_gateway_mirror" _GATEWAY_MIRROR_TOKEN = "_gateway_mirror_token" _GATEWAY_ENTRY_DATA_TOKEN_KEY = "_webui_mirror_token" -_GATEWAY_ENTRY_DATA_LOCAL_HEAD_KEY = "_webui_local_head" def _approval_sse_subscribe(session_id: str) -> queue.Queue: @@ -116,8 +115,6 @@ def _gateway_mirror_entry_token(entry) -> str | None: """ data = getattr(entry, "data", None) if isinstance(data, dict): - if data.get(_GATEWAY_ENTRY_DATA_LOCAL_HEAD_KEY): - return None token = data.get(_GATEWAY_ENTRY_DATA_TOKEN_KEY) if not token: token = uuid.uuid4().hex @@ -150,8 +147,31 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No live_head_entry = live_gateway_queue[0] if live_gateway_queue else None live_head_data = getattr(live_head_entry, "data", None) or {} - live_token = _gateway_mirror_entry_token(live_head_entry) if live_head_entry and live_head_data else None + has_no_run_mirror = any( + _is_gateway_mirror_entry(entry) + and not str(entry.get("run_id") or "").strip() + for entry in queue_list + ) live_run_id = str(live_head_data.get("run_id") or "").strip() + live_has_token = bool(live_head_data.get(_GATEWAY_ENTRY_DATA_TOKEN_KEY)) + live_local_tokens: set[str] = set() + for live_entry in live_gateway_queue: + live_data = getattr(live_entry, "data", None) or {} + if str(live_data.get("run_id") or "").strip(): + continue + live_entry_token = str(live_data.get(_GATEWAY_ENTRY_DATA_TOKEN_KEY) or "").strip() + if live_entry_token or not has_no_run_mirror: + live_entry_token = _gateway_mirror_entry_token(live_entry) or "" + if live_entry_token: + live_local_tokens.add(live_entry_token) + if not str(live_data.get("approval_id") or "").strip(): + live_data["approval_id"] = f"gwlocal:{live_entry_token}" + live_token = ( + _gateway_mirror_entry_token(live_head_entry) + if live_head_entry and live_head_data + and (live_run_id or live_has_token or not has_no_run_mirror) + else None + ) if live_token and live_run_id and not str(live_head_data.get("approval_id") or "").strip(): live_head_data["approval_id"] = f"gwrun:{live_run_id}:{live_token}" live_approval_id = str(live_head_data.get("approval_id") or "").strip() @@ -206,6 +226,10 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No live_mirror_present = True continue + if entry_token and entry_token in live_local_tokens: + rebuilt.append(entry) + continue + if not live_token: if entry_token: changed = True @@ -348,34 +372,56 @@ def _gateway_mirrored_pending_run_id(session_key: str, approval_id: str) -> str return None -def submit_gateway_pending_mirror(session_key: str, approval: dict) -> None: +def submit_gateway_pending_mirror(session_key: str, approval: dict) -> tuple[dict | None, int]: """Mirror the live gateway head into WebUI polling state under a typed tag.""" with _lock: run_id = str(approval.get("run_id") or "").strip() approval_id = str(approval.get("approval_id") or "").strip() live_gateway_queue = _gateway_queues.get(session_key) or [] - live_head_entry = live_gateway_queue[0] if live_gateway_queue else None - live_head_data = getattr(live_head_entry, "data", None) or {} - matching_no_run_live_head = ( - not run_id - and live_head_data - and str(live_head_data.get("command") or "") == str(approval.get("command") or "") - and str(live_head_data.get("description") or "") == str(approval.get("description") or "") - and str(live_head_data.get("pattern_key") or "") == str(approval.get("pattern_key") or "") - and tuple(str(k) for k in (live_head_data.get("pattern_keys") or [])) - == tuple(str(k) for k in (approval.get("pattern_keys") or [])) - ) - if matching_no_run_live_head and not live_head_data.get(_GATEWAY_ENTRY_DATA_TOKEN_KEY): - live_head_data.pop(_GATEWAY_ENTRY_DATA_LOCAL_HEAD_KEY, None) - live_head_data[_GATEWAY_ENTRY_DATA_TOKEN_KEY] = uuid.uuid4().hex - elif not run_id and live_head_data: - live_head_data[_GATEWAY_ENTRY_DATA_LOCAL_HEAD_KEY] = True - if not run_id and not approval_id: - approval_id = uuid.uuid4().hex - approval["approval_id"] = approval_id - if matching_no_run_live_head and approval_id and not str(live_head_data.get("approval_id") or "").strip(): - live_head_data["approval_id"] = approval_id + exact_local_entry = next( + ( + entry for entry in live_gateway_queue + if getattr(entry, "data", None) is approval + ), + None, + ) if not run_id else None + if exact_local_entry is None and not run_id and approval_id: + exact_local_entry = next( + ( + entry for entry in live_gateway_queue + if str(((getattr(entry, "data", None) or {}).get("approval_id") or "")).strip() == approval_id + ), + None, + ) + if exact_local_entry is not None: + mirror_entries = _normalize_pending_queue_locked(session_key) + entries_to_mirror = [live_gateway_queue[0]] if live_gateway_queue else [] + if exact_local_entry not in entries_to_mirror: + entries_to_mirror.append(exact_local_entry) + for entry in entries_to_mirror: + local_data = entry.data + token = _gateway_mirror_entry_token(entry) + entry_approval_id = str(local_data.get("approval_id") or "").strip() + if entry is exact_local_entry: + entry_approval_id = approval_id or entry_approval_id or f"gwlocal:{token}" + approval_id = entry_approval_id + approval["approval_id"] = entry_approval_id + elif not entry_approval_id: + entry_approval_id = f"gwlocal:{token}" + local_data["approval_id"] = entry_approval_id + if not any( + _is_gateway_mirror_entry(mirror) + and str(mirror.get(_GATEWAY_MIRROR_TOKEN) or "") == token + for mirror in mirror_entries + ): + mirror_entry = dict(local_data) + mirror_entry["approval_id"] = entry_approval_id + mirror_entry[_GATEWAY_MIRROR_FLAG] = True + mirror_entry[_GATEWAY_MIRROR_TOKEN] = token + mirror_entries.append(mirror_entry) if run_id: + live_head_entry = live_gateway_queue[0] if live_gateway_queue else None + live_head_data = getattr(live_head_entry, "data", None) or {} live_head_run_id = str(live_head_data.get("run_id") or "").strip() live_head_approval_id = str(live_head_data.get("approval_id") or "").strip() live_token = ( @@ -410,15 +456,10 @@ def submit_gateway_pending_mirror(session_key: str, approval: dict) -> None: mirror_entry[_GATEWAY_MIRROR_FLAG] = True if not _gateway_pending_mirror_locked(session_key, approval_id=approval_id, run_id=run_id): _normalize_pending_queue_locked(session_key).append(mirror_entry) - head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) - approval_id = str(approval.get("approval_id") or "").strip() - if run_id and approval_id: - mirror_entry = _gateway_pending_mirror_locked( - session_key, approval_id=approval_id, run_id=run_id - ) - if mirror_entry: - approval["approval_id"] = str(mirror_entry.get("approval_id") or approval_id).strip() - elif approval_id: + elif not exact_local_entry: + if not approval_id: + approval_id = uuid.uuid4().hex + approval["approval_id"] = approval_id queue = _pending.get(session_key) entries = queue if isinstance(queue, list) else [queue] if queue else [] no_run_mirror = next( @@ -437,8 +478,39 @@ def submit_gateway_pending_mirror(session_key: str, approval: dict) -> None: mirror_entry["approval_id"] = approval_id mirror_entry[_GATEWAY_MIRROR_FLAG] = True _normalize_pending_queue_locked(session_key).append(mirror_entry) + head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) _approval_sse_notify_locked(session_key, head, total) publish_session_list_changed("attention_pending") + return (dict(head) if head else None), total + + +def resolve_gateway_pending_local( + session_key: str, approval_id: str, choice: str, reason: str | None = None +) -> tuple[int, dict | None, int]: + """Resolve the exact parked local entry bound to an approval mirror.""" + target = None + with _lock: + approval_id = str(approval_id or "").strip() + gateway_queue = _gateway_queues.get(session_key) or [] + for index, entry in enumerate(gateway_queue): + data = getattr(entry, "data", None) or {} + if str(data.get("approval_id") or "").strip() == approval_id: + target = gateway_queue.pop(index) + break + if gateway_queue: + _gateway_queues[session_key] = gateway_queue + else: + _gateway_queues.pop(session_key, None) + head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) + _approval_sse_notify_locked(session_key, head, total) + if target is None: + return 0, head, total + target.result = choice + if reason: + target.reason = reason + target.event.set() + publish_session_list_changed("attention_resolved") + return 1, head, total def submit_pending(session_key: str, approval: dict) -> None: diff --git a/api/routes.py b/api/routes.py index 016df3a6..76dfa066 100644 --- a/api/routes.py +++ b/api/routes.py @@ -9720,6 +9720,7 @@ from api.route_approvals import ( # noqa: F401 — re-exports for backward comp gateway_pending_mirror, retire_gateway_pending_mirror, reconcile_gateway_pending_mirror_locked, + resolve_gateway_pending_local, submit_gateway_pending_mirror, submit_pending, ) @@ -9746,12 +9747,40 @@ except ImportError: def _session_attention_summary(session_id: str) -> dict | None: """Return sidebar attention metadata for pending approval/clarify work.""" + active_run_id = "" + try: + from api.gateway_chat import _STREAM_RUN_IDS + + session = get_session(session_id) + active_stream_id = getattr(session, "active_stream_id", None) if session is not None else None + if active_stream_id: + active_run_id = str(_STREAM_RUN_IDS.get(active_stream_id) or "").strip() + except Exception: + active_run_id = "" + approval_count = 0 with _lock: reconcile_gateway_pending_mirror_locked(session_id) queue_list = _pending.get(session_id) if isinstance(queue_list, list): - approval_count = len(queue_list) + gateway_queue = _gateway_queues.get(session_id) or [] + live_gateway_run_ids = { + str((getattr(entry, "data", None) or {}).get("run_id") or "").strip() + for entry in gateway_queue + if str((getattr(entry, "data", None) or {}).get("run_id") or "").strip() + } + approval_count = sum( + 1 + for entry in queue_list + if not ( + isinstance(entry, dict) + and entry.get(_GATEWAY_MIRROR_FLAG) + and str(entry.get("run_id") or "").strip() + and not str(entry.get(_GATEWAY_MIRROR_TOKEN) or "").strip() + and str(entry.get("run_id") or "").strip() not in live_gateway_run_ids + and str(entry.get("run_id") or "").strip() != active_run_id + ) + ) elif queue_list: approval_count = 1 if approval_count > 0: @@ -23802,6 +23831,7 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str, run_id: st pending = None found_target = False gateway_keys = [] + local_gateway_approval_id = "" gateway_head_matches_target = False with _lock: reconcile_gateway_pending_mirror_locked(sid) @@ -23874,20 +23904,28 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str, run_id: st gw_data = getattr(gw_entry, "data", None) or {} gw_approval_id = str(gw_data.get("approval_id") or "").strip() gw_run_id = str(gw_data.get("run_id") or "").strip() - if run_id: - gateway_head_matches_target = gw_approval_id == approval_id and gw_run_id == run_id - else: - gateway_head_matches_target = gw_approval_id == approval_id + gateway_head_matches_target = bool( + run_id and gw_approval_id == approval_id and gw_run_id == run_id + ) + if gw_approval_id == approval_id and (not run_id or gw_run_id == run_id): + local_gateway_approval_id = approval_id + elif not run_id and found_target and pending: + pending_token = str(pending.get(_GATEWAY_MIRROR_TOKEN) or "").strip() + gateway_token = str(gw_data.get("_webui_mirror_token") or "").strip() + if pending_token and pending_token == gateway_token: + gw_data["approval_id"] = approval_id + local_gateway_approval_id = approval_id # Notify SSE subscribers of the new head (or empty state) so the UI # surfaces any trailing approvals that were queued behind this one # without waiting for the next submit_pending. Without this, a parallel # tool-call scenario (#527) would leave the second approval invisible # in the SSE path until the next event ever fired (the agent thread # would be parked indefinitely from the user's perspective). - if isinstance(_pending.get(sid), list) and _pending[sid]: - _approval_sse_notify_locked(sid, _pending[sid][0], len(_pending[sid])) - else: - _approval_sse_notify_locked(sid, None, 0) + if not local_gateway_approval_id: + if isinstance(_pending.get(sid), list) and _pending[sid]: + _approval_sse_notify_locked(sid, _pending[sid][0], len(_pending[sid])) + else: + _approval_sse_notify_locked(sid, None, 0) # Collect keys from both _pending and _gateway_queues keys_from_pending = pending.get("pattern_keys") or [pending.get("pattern_key", "")] if pending else [] @@ -23909,21 +23947,18 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str, run_id: st # This is the primary signal when streaming is active — the agent # thread is parked in entry.event.wait() and needs to be woken up. gateway_resolved = 0 - should_resolve_gateway = ( - not approval_id - or ( - found_target - and ( - not run_id - or gateway_head_matches_target - ) + local_gateway_resolved = 0 + if approval_id and found_target and not run_id and local_gateway_approval_id: + local_gateway_resolved, _head, _total = resolve_gateway_pending_local( + sid, local_gateway_approval_id, choice ) - ) - if should_resolve_gateway: + elif approval_id and found_target and run_id and gateway_head_matches_target: + gateway_resolved = resolve_gateway_approval(sid, choice, resolve_all=False) or 0 + elif not approval_id: gateway_resolved = resolve_gateway_approval(sid, choice, resolve_all=False) or 0 # Keep the historical no-id response path truthy for old clients/tests while # making stale explicit ids bounded as not-active for Slice 3b. - resolved = bool(pending) or bool(gateway_resolved) or not bool(approval_id) + resolved = bool(pending) or bool(gateway_resolved) or bool(local_gateway_resolved) or not bool(approval_id) if resolved: publish_session_list_changed("attention_resolved") return resolved diff --git a/api/streaming.py b/api/streaming.py index 2528baf4..bda92823 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -7826,7 +7826,8 @@ def _run_agent_streaming( def _approval_notify_cb(approval_data): if _submit_pending_for_polling is not None: try: - _submit_pending_for_polling(session_id, approval_data) + head, total = _submit_pending_for_polling(session_id, approval_data) + approval_data = {**(head or approval_data), "pending_count": total} except Exception: logger.warning("Failed to mirror approval into WebUI polling state", exc_info=True) put('approval', approval_data) @@ -8241,20 +8242,9 @@ def _run_agent_streaming( if _has_blocking_approval(session_id): p = None with _approval_lock: - _reconcile_gateway_pending_mirror_locked(session_id) - queue = _approval_pending.get(session_id) - if isinstance(queue, list): - p = dict(queue[0]) if queue else None - elif queue: - p = dict(queue) - if p is None: - gw_queue = _approval_gateway_queues.get(session_id) or [] - if gw_queue: - raw = getattr(gw_queue[0], 'data', None) or {} - if raw: - p = dict(raw) - else: - logger.warning("Gateway queue entry for %s has no .data attribute", session_id) + p, pending_count, _changed = _reconcile_gateway_pending_mirror_locked(session_id) + if p: + p = {**p, "pending_count": pending_count} if p: put('approval', p) except ImportError: diff --git a/static/messages.js b/static/messages.js index ea19f451..c45ff129 100644 --- a/static/messages.js +++ b/static/messages.js @@ -5589,7 +5589,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ source.addEventListener('approval',e=>{ const d=JSON.parse(e.data); _applyToAnchor('approval',d,e); - showApprovalForSession(activeSid, d, 1); + showApprovalForSession(activeSid, d, d.pending_count || 1); playAttentionSound(_attentionSoundKey(activeSid,'approval',1)); sendBrowserNotification('Approval required',d.description||'Tool approval needed',{sid:activeSid}); }); diff --git a/tests/test_approval_queue.py b/tests/test_approval_queue.py index 2ca10f58..4de98b72 100644 --- a/tests/test_approval_queue.py +++ b/tests/test_approval_queue.py @@ -7,6 +7,8 @@ approval_id so /api/approval/respond can target a specific entry. import json import pathlib import re +import shutil +import subprocess import sys REPO_ROOT = pathlib.Path(__file__).parent.parent.resolve() @@ -108,6 +110,39 @@ def test_polling_passes_count_to_show(): "Poll loop must pass data.pending_count through showApprovalForSession" +def test_chat_stream_approval_listener_renders_received_count(): + """A chat-stream approval event with count two renders the head and count.""" + node = shutil.which("node") + if node is None: + import pytest + pytest.skip("node is required for the approval listener harness") + start = MESSAGES_JS.index("source.addEventListener('approval',e=>{") + end_marker = "\n });" + end = MESSAGES_JS.index(end_marker, start) + len(end_marker) + listener = MESSAGES_JS[start:end] + script = f""" + const rendered = []; + let activeSid = 'sid-browser'; + const source = {{ listeners: {{}}, addEventListener(name, cb) {{ this.listeners[name] = cb; }} }}; + function _applyToAnchor() {{}} + function showApprovalForSession(sid, data, count) {{ rendered.push({{sid, data, count}}); }} + function playAttentionSound() {{}} + function _attentionSoundKey() {{ return 'approval'; }} + function sendBrowserNotification() {{}} + {listener} + source.listeners.approval({{ data: JSON.stringify({{command: 'head', approval_id: 'a', pending_count: 2}}) }}); + process.stdout.write(JSON.stringify(rendered)); + """ + result = subprocess.run([node, "-e", script], capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stderr + rendered = json.loads(result.stdout) + assert rendered == [{ + "sid": "sid-browser", + "data": {"command": "head", "approval_id": "a", "pending_count": 2}, + "count": 2, + }] + + # --------------------------------------------------------------------------- # HTML: counter element present # --------------------------------------------------------------------------- diff --git a/tests/test_approval_sse.py b/tests/test_approval_sse.py index f10ea608..dfc4dfc6 100644 --- a/tests/test_approval_sse.py +++ b/tests/test_approval_sse.py @@ -113,8 +113,10 @@ class TestSSEStaticAnalysis: cb_start = streaming_src.find("def _approval_notify_cb(approval_data):") cb_end = streaming_src.find("_reg_notify(session_id, _approval_notify_cb)", cb_start) cb_body = streaming_src[cb_start:cb_end] - assert "_submit_pending_for_polling(session_id, approval_data)" in cb_body, \ + assert "head, total = _submit_pending_for_polling(session_id, approval_data)" in cb_body, \ "_approval_notify_cb must mirror approval data into polling state before SSE" + assert '"pending_count": total' in cb_body, \ + "_approval_notify_cb must emit the reconciled pending count" assert "put('approval', approval_data)" in cb_body, \ "_approval_notify_cb must still emit the approval SSE event" diff --git a/tests/test_approval_unblock.py b/tests/test_approval_unblock.py index d9040d95..757ab5e6 100644 --- a/tests/test_approval_unblock.py +++ b/tests/test_approval_unblock.py @@ -232,11 +232,57 @@ class TestApprovalModuleExports: assert cb_start != -1, "_approval_notify_cb must exist" cb_end = STREAMING_SRC.find("_reg_notify(session_id, _approval_notify_cb)", cb_start) cb_body = STREAMING_SRC[cb_start:cb_end] - assert "_submit_pending_for_polling(session_id, approval_data)" in cb_body, \ + assert "head, total = _submit_pending_for_polling(session_id, approval_data)" in cb_body, \ "approval notify callback must mirror approval data into polling state" + assert '"pending_count": total' in cb_body, \ + "approval notify callback must publish the reconciled pending count" assert "put('approval', approval_data)" in cb_body, \ "approval notify callback must still push the SSE event" + def test_reversed_local_callbacks_publish_all_parked_entries(self): + """The local producer must count every exact parked entry, not only its head.""" + import api.route_approvals as ra + from api import routes + + sid = f"unit-reversed-local-{uuid.uuid4().hex[:8]}" + entry_a = _ApprovalEntry({ + "command": "same-command", + "description": "same-description", + "pattern_key": "same-pattern", + "pattern_keys": ["same-pattern"], + }) + entry_b = _ApprovalEntry(dict(entry_a.data)) + events = [] + with _lock: + _gateway_queues[sid] = [entry_a, entry_b] + ra._pending.pop(sid, None) + + def local_producer(approval_data): + head, total = ra.submit_gateway_pending_mirror(sid, approval_data) + events.append((head, total)) + + try: + local_producer(entry_b.data) + local_producer(entry_a.data) + + assert [total for _head, total in events] == [2, 2] + assert all(head["command"] == "same-command" for head, _total in events) + head_id = events[0][0]["approval_id"] + assert head_id == events[1][0]["approval_id"] + assert routes._resolve_approval_legacy(sid, head_id, "once") is True + assert entry_a.event.is_set() + assert not entry_b.event.is_set() + with _lock: + successor = ra._pending[sid][0] + assert successor["approval_id"] != head_id + with _lock: + assert ra.reconcile_gateway_pending_mirror_locked(sid)[1] == 1 + finally: + with _lock: + ra._pending.pop(sid, None) + _gateway_queues.pop(sid, None) + ra._pending.pop(sid, None) + # ── HTTP regression tests (test server, port 8788) ─────────────────────────── diff --git a/tests/test_gateway_approval_legacy_path.py b/tests/test_gateway_approval_legacy_path.py index a170bb72..d9a1c7d1 100644 --- a/tests/test_gateway_approval_legacy_path.py +++ b/tests/test_gateway_approval_legacy_path.py @@ -336,6 +336,19 @@ def test_legacy_approval_records_run_id_for_response_relay(): # (connection still open, run parked) — this test's stream already ran to # completion, which cleared active_stream_id and popped the mapping. _STREAM_RUN_IDS[stream_id] = "run-legacy-1" + from types import SimpleNamespace + from api import route_approvals as ra + with ra._lock: + ra._gateway_queues["sess-legacy-runid"] = [SimpleNamespace(data={ + "command": "rm -rf /tmp/test", + "description": "Delete temporary files", + "pattern_key": "dangerous_command", + "pattern_keys": ["dangerous_command"], + "approval_id": "appr-legacy-runid", + "run_id": "run-legacy-1", + })] + live_entry = ra._gateway_queues["sess-legacy-runid"][0] + ra.submit_gateway_pending_mirror("sess-legacy-runid", live_entry.data) relay_session = MagicMock() relay_session.active_stream_id = stream_id captured = {} @@ -447,8 +460,8 @@ def test_legacy_teardown_clears_stale_gateway_mirror_and_notifies_empty_state(): ra._gateway_queues.pop(session_id, None) -def test_legacy_teardown_preserves_live_gateway_head_mirror(): - """A live gateway head must still be mirrored after legacy teardown runs.""" +def test_legacy_teardown_retires_live_gateway_head_mirror(): + """A mapped legacy run retires its live gateway mirror during outer teardown.""" from types import SimpleNamespace from api import route_approvals as ra from api.config import STREAMS, STREAMS_LOCK @@ -507,14 +520,11 @@ def test_legacy_teardown_preserves_live_gateway_head_mirror(): payloads = _drain_queue(subscriber) assert payloads, "Expected mirrored approval notifications" - assert payloads[-1]["pending"]["_gateway_mirror"] is True - assert payloads[-1]["pending_count"] == 1 + assert payloads[-1]["pending"] is None + assert payloads[-1]["pending_count"] == 0 with ra._lock: pending = ra._pending.get(session_id) - assert isinstance(pending, list) - assert len(pending) == 1 - assert pending[0]["approval_id"] == approval_data["approval_id"] - assert pending[0]["_gateway_mirror"] is True + assert pending is None finally: ra._approval_sse_unsubscribe(session_id, subscriber) with STREAMS_LOCK: diff --git a/tests/test_session_attention_badges.py b/tests/test_session_attention_badges.py index 8955d000..91bf7c13 100644 --- a/tests/test_session_attention_badges.py +++ b/tests/test_session_attention_badges.py @@ -115,6 +115,25 @@ def test_attention_summary_prefers_pending_approvals_over_clarify_questions(): _clear_attention_state(sid) +def test_attention_summary_ignores_stale_gateway_mirror(): + sid = "attention-stale-gateway-session" + _clear_attention_state(sid) + try: + with routes._lock: + routes._pending[sid] = [{ + "_gateway_mirror": True, + "approval_id": "appr-stale-gateway", + "run_id": "run-stale-gateway", + "command": "rm -rf /tmp/stale", + "description": "Stale approval", + }] + routes._gateway_queues[sid] = [] + + assert routes._session_attention_summary(sid) is None + finally: + _clear_attention_state(sid) + + def test_attention_summary_reports_clarify_when_no_approval_is_pending(): sid = "attention-clarify-session" _clear_attention_state(sid) From b481dfea13fdc7e3eefd512f6afa6a6fea945b8b Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 19 Jul 2026 12:41:24 -0400 Subject: [PATCH 06/20] fix(#6008): keep retained approvals visible in session attention --- api/routes.py | 30 +-------- tests/test_gateway_approval_runs_api.py | 82 +++++++++++++++++++++++++ tests/test_session_attention_badges.py | 67 ++++++++++++++++---- 3 files changed, 138 insertions(+), 41 deletions(-) diff --git a/api/routes.py b/api/routes.py index 76dfa066..b893d7e9 100644 --- a/api/routes.py +++ b/api/routes.py @@ -9747,40 +9747,12 @@ except ImportError: def _session_attention_summary(session_id: str) -> dict | None: """Return sidebar attention metadata for pending approval/clarify work.""" - active_run_id = "" - try: - from api.gateway_chat import _STREAM_RUN_IDS - - session = get_session(session_id) - active_stream_id = getattr(session, "active_stream_id", None) if session is not None else None - if active_stream_id: - active_run_id = str(_STREAM_RUN_IDS.get(active_stream_id) or "").strip() - except Exception: - active_run_id = "" - approval_count = 0 with _lock: reconcile_gateway_pending_mirror_locked(session_id) queue_list = _pending.get(session_id) if isinstance(queue_list, list): - gateway_queue = _gateway_queues.get(session_id) or [] - live_gateway_run_ids = { - str((getattr(entry, "data", None) or {}).get("run_id") or "").strip() - for entry in gateway_queue - if str((getattr(entry, "data", None) or {}).get("run_id") or "").strip() - } - approval_count = sum( - 1 - for entry in queue_list - if not ( - isinstance(entry, dict) - and entry.get(_GATEWAY_MIRROR_FLAG) - and str(entry.get("run_id") or "").strip() - and not str(entry.get(_GATEWAY_MIRROR_TOKEN) or "").strip() - and str(entry.get("run_id") or "").strip() not in live_gateway_run_ids - and str(entry.get("run_id") or "").strip() != active_run_id - ) - ) + approval_count = len(queue_list) elif queue_list: approval_count = 1 if approval_count > 0: diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index 3ba15e9a..3b06d611 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -421,6 +421,88 @@ def test_gateway_runs_api_streaming_parses_real_run_events(): assert approvals.gateway_pending_mirror("sess1", run_id="run-abc") is None +def test_gateway_runs_api_streaming_same_run_fifo_emits_head_and_promotes_successor(): + """Runs API approval events publish the reconciled FIFO head and count.""" + from api.config import STREAM_PARTIAL_TEXT, STREAM_REASONING_TEXT + from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming + from api import routes + import api.route_approvals as approvals + + sid = "sess-same-run-producer" + stream_id = "stream-same-run-producer" + events = [] + STREAM_PARTIAL_TEXT[stream_id] = "" + STREAM_REASONING_TEXT[stream_id] = "" + + class _JsonResponse: + def read(self, _limit=None): + return b'{"run_id":"run-fifo"}' + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + class _SseResponse: + def __iter__(self): + return iter([ + b'data: {"event":"approval.request","command":"first","description":"First","run_id":"run-fifo","approval_id":"approval-first"}\n', + b'\n', + b'data: {"event":"approval.request","command":"second","description":"Second","run_id":"run-fifo","approval_id":"approval-second"}\n', + b'\n', + b'data: [DONE]\n', + b'\n', + ]) + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def fake_urlopen(req, *, timeout=None): + return _JsonResponse() if req.full_url.endswith("/v1/runs") else _SseResponse() + + try: + with patch("urllib.request.urlopen", side_effect=fake_urlopen): + _run_gateway_runs_api_streaming( + sid, "hi", "test", "/tmp", stream_id, "http://gw:8642", "", [], {}, + put_gateway_event=lambda event, data: events.append((event, data)), + cancel_event=threading.Event(), + ) + + approval_events = [data for event, data in events if event == "approval"] + assert [(data["approval_id"], data["pending_count"]) for data in approval_events] == [ + ("approval-first", 1), + ("approval-first", 2), + ] + + handler = MagicMock() + handler.wfile = io.BytesIO() + with patch("api.routes.get_session", return_value=SimpleNamespace(active_stream_id=stream_id)), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond_approval: + routes._handle_approval_respond(handler, { + "session_id": sid, + "choice": "deny", + "approval_id": "approval-first", + }) + + handler.send_response.assert_called_with(200) + respond_approval.assert_called_once_with("run-fifo", "approval-first", "deny") + promoted = approvals.gateway_pending_mirror(sid, run_id="run-fifo") + assert promoted is not None + assert promoted["approval_id"] == "approval-second" + with approvals._lock: + assert len(approvals._pending[sid]) == 1 + finally: + STREAM_PARTIAL_TEXT.pop(stream_id, None) + STREAM_REASONING_TEXT.pop(stream_id, None) + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + def test_empty_id_runs_approval_reaches_real_response_lifecycle(): """Runs API empty IDs are carried and resolved through the live mirror.""" from api.gateway_chat import _run_gateway_runs_api_streaming diff --git a/tests/test_session_attention_badges.py b/tests/test_session_attention_badges.py index 91bf7c13..e67f2328 100644 --- a/tests/test_session_attention_badges.py +++ b/tests/test_session_attention_badges.py @@ -115,22 +115,65 @@ def test_attention_summary_prefers_pending_approvals_over_clarify_questions(): _clear_attention_state(sid) -def test_attention_summary_ignores_stale_gateway_mirror(): - sid = "attention-stale-gateway-session" +def test_attention_summary_keeps_direct_runs_mirror_after_stream_pointer_clears(): + from types import SimpleNamespace + from unittest.mock import MagicMock, patch + + import api.route_approvals as approvals + + sid = "attention-retained-gateway-session" _clear_attention_state(sid) try: - with routes._lock: - routes._pending[sid] = [{ - "_gateway_mirror": True, - "approval_id": "appr-stale-gateway", - "run_id": "run-stale-gateway", - "command": "rm -rf /tmp/stale", - "description": "Stale approval", - }] - routes._gateway_queues[sid] = [] + approvals.submit_gateway_pending_mirror(sid, { + "_gateway_mirror": True, + "approval_id": "approval-retained", + "run_id": "run-retained", + "command": "rm -rf /tmp/retained", + "description": "Retained approval", + }) + handler = MagicMock() + handler.wfile = io.BytesIO() - assert routes._session_attention_summary(sid) is None + with patch("api.routes.get_session", return_value=SimpleNamespace(active_stream_id=None)), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond_approval: + assert routes._session_attention_summary(sid) == { + "kind": "approval", + "count": 1, + "severity": "critical", + } + routes._handle_approval_respond(handler, { + "session_id": sid, + "choice": "deny", + "approval_id": "approval-retained", + }) + + handler.send_response.assert_called_with(200) + respond_approval.assert_called_once_with("run-retained", "approval-retained", "deny") + assert json.loads(handler.wfile.getvalue().decode("utf-8"))["ok"] is True finally: + approvals._pending.pop(sid, None) + _clear_attention_state(sid) + + +def test_attention_summary_hides_retired_direct_runs_mirror(): + import api.route_approvals as approvals + + sid = "attention-retired-gateway-session" + _clear_attention_state(sid) + try: + approvals.submit_gateway_pending_mirror(sid, { + "_gateway_mirror": True, + "approval_id": "approval-retired", + "run_id": "run-retired", + "command": "rm -rf /tmp/retired", + "description": "Retired approval", + }) + + assert approvals.retire_gateway_pending_mirror(sid, run_id="run-retired") is True + assert routes._session_attention_summary(sid) is None + assert approvals.gateway_pending_mirror(sid, run_id="run-retired") is None + finally: + approvals._pending.pop(sid, None) _clear_attention_state(sid) From f30178e18aa99bb1059ff62983b33eef507cbc12 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 19 Jul 2026 15:30:00 -0400 Subject: [PATCH 07/20] fix(gateway): fall back safely without approval identities --- api/config.py | 7 +++ api/gateway_chat.py | 5 +-- api/routes.py | 42 +++++++----------- static/i18n.js | 30 ++++++------- static/sessions.js | 1 + tests/test_gateway_approval_runs_api.py | 43 +++++++++---------- ...est_issue4771_local_approval_regression.py | 12 ++---- tests/test_session_attention_badges.py | 2 +- 8 files changed, 65 insertions(+), 77 deletions(-) diff --git a/api/config.py b/api/config.py index 0647b945..dd3569bc 100644 --- a/api/config.py +++ b/api/config.py @@ -8892,6 +8892,7 @@ def get_gateway_caps(base_url: str, api_key: str = "") -> dict: caps = { "approval_events": False, "run_approval_response": False, + "approval_identity_v1": False, "capabilities_reachable": False, "probe_error": None, "fetched_at": 0.0, @@ -8909,6 +8910,7 @@ def get_gateway_caps(base_url: str, api_key: str = "") -> dict: features = {} caps["approval_events"] = bool(features.get("approval_events")) caps["run_approval_response"] = bool(features.get("run_approval_response")) + caps["approval_identity_v1"] = bool(features.get("approval_identity_v1")) except urllib.error.HTTPError as exc: caps["capabilities_reachable"] = True caps["probe_error"] = f"{type(exc).__name__}: {exc}" @@ -8948,6 +8950,11 @@ def gateway_supports_approval(base_url: str, api_key: str = "") -> bool: return bool(caps.get("approval_events") and caps.get("run_approval_response")) +def gateway_supports_approval_identity_v1(base_url: str, api_key: str = "") -> bool: + """True only when Gateway advertises authoritative approval identities.""" + return bool(get_gateway_caps(base_url, api_key).get("approval_identity_v1")) + + def invalidate_gateway_caps(base_url: str | None = None) -> None: """Evict capability cache for base_url, or all entries when base_url is None.""" with _GATEWAY_CAPS_LOCK: diff --git a/api/gateway_chat.py b/api/gateway_chat.py index f4e5158a..c0746074 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -470,7 +470,8 @@ def _run_gateway_runs_api_streaming( if stop_confirmed: put_gateway_event("cancel", {"message": "Cancelled by user"}) return None, usage - cancel_event.clear() + put_gateway_event("apperror", {"message": "Gateway stop failed"}) + return None, usage url_events = f"{base_url.rstrip('/')}/v1/runs/{run_id}/events" headers_sse = dict(headers) @@ -504,8 +505,6 @@ def _run_gateway_runs_api_streaming( approval_data = _gateway_runs_approval_event(payload) if approval_data: approval_data["run_id"] = run_id - if not approval_data.get("approval_id"): - approval_data["approval_id"] = f"gwrun:{run_id}:{uuid.uuid4().hex}" from api.route_approvals import submit_gateway_pending_mirror head, total = submit_gateway_pending_mirror(session_id, approval_data) put_gateway_event("approval", {**(head or approval_data), "pending_count": total}) diff --git a/api/routes.py b/api/routes.py index b893d7e9..8b8854f1 100644 --- a/api/routes.py +++ b/api/routes.py @@ -13378,15 +13378,13 @@ def handle_get(handler, parsed) -> bool: run_id = _STREAM_RUN_IDS.get(stream_id) pending_run_id = False if not run_id: - import time as _time - pending_run_id = gateway_run_id_pending(stream_id) - attempts_remaining = 10 - while not run_id and pending_run_id and attempts_remaining > 0: - _time.sleep(0.05) - run_id = _STREAM_RUN_IDS.get(stream_id) - pending_run_id = gateway_run_id_pending(stream_id) - attempts_remaining -= 1 + if pending_run_id: + cancel_flag = CANCEL_FLAGS.get(stream_id) + if cancel_flag: + cancel_flag.set() + cancelled = cancel_stream(stream_id) + return j(handler, {"ok": True, "cancelled": cancelled, "deferred": True, "stream_id": stream_id}) if run_id: if stop_gateway_run(stream_id): owner_sid = stream_owner_session_id(stream_id) @@ -13394,8 +13392,6 @@ def handle_get(handler, parsed) -> bool: retire_gateway_pending_mirror(owner_sid, run_id=run_id) else: gateway_stop_blocked = True - elif pending_run_id: - gateway_stop_blocked = True except Exception: logger.debug("Failed to stop gateway run during chat cancellation", exc_info=True) gateway_stop_blocked = True @@ -23955,11 +23951,11 @@ def _gateway_pending_approval_without_run_id(sid: str, approval_id: str) -> bool if approval_id: for entry in entries: if isinstance(entry, dict) and entry.get("approval_id") == approval_id: - return bool(entry.get(_GATEWAY_MIRROR_FLAG)) + return bool(entry.get(_GATEWAY_MIRROR_FLAG)) and not str(entry.get("run_id") or "").strip() return False if not entries or not isinstance(entries[0], dict): return False - return bool(entries[0].get(_GATEWAY_MIRROR_FLAG)) + return bool(entries[0].get(_GATEWAY_MIRROR_FLAG)) and not str(entries[0].get("run_id") or "").strip() def _session_has_pending_approval(sid: str) -> bool: @@ -24002,7 +23998,7 @@ def _handle_approval_respond(handler, body): _gateway_api_key, webui_gateway_chat_enabled, ) - from api.config import get_config as _get_config + from api.config import get_config as _get_config, gateway_supports_approval_identity_v1 s = get_session(sid) _candidate_run_id = None if s is not None: @@ -24109,8 +24105,9 @@ def _handle_approval_respond(handler, body): _base = _gateway_base_url(_cfg) _key = _gateway_api_key() try: + identity_v1 = gateway_supports_approval_identity_v1(_base, _key) HttpRunnerClient(base_url=_base, api_key=_key).respond_approval( - _run_id, matched_mirror["approval_id"], choice + _run_id, matched_mirror["approval_id"] if identity_v1 else "", choice ) except (RunnerClientError, ValueError) as exc: return j(handler, {"ok": False, "choice": choice, "relayed": True, "error": str(exc)}, status=502) @@ -24125,22 +24122,13 @@ def _handle_approval_respond(handler, body): return j(handler, {"ok": False, "choice": choice, "relayed": False, "code": "gateway_run_unavailable", "error": _GATEWAY_APPROVAL_RELAY_UNAVAILABLE}, status=409) - # Only a still-mirrored gateway approval with a missing run should 409; - # stale or empty gateway clicks fall through to local resolution. + # A no-run mirror is local visibility state only. Retire it instead of + # claiming that a remote approval can still be resolved. if webui_gateway_chat_enabled(_get_config()) and _gateway_pending_approval_without_run_id( sid, approval_id ): - return j( - handler, - { - "ok": False, - "choice": choice, - "relayed": False, - "code": "gateway_run_unavailable", - "error": _GATEWAY_APPROVAL_RELAY_UNAVAILABLE, - }, - status=409, - ) + retire_gateway_pending_mirror(sid, approval_id=approval_id) + return j(handler, {"ok": True, "choice": choice, "local_retired": True}) except Exception: pass # fall through to local approval path diff --git a/static/i18n.js b/static/i18n.js index 3c55bf51..48a926c1 100644 --- a/static/i18n.js +++ b/static/i18n.js @@ -17,7 +17,7 @@ const LOCALES = { _speech: 'en-US', // boot.js cancelling: 'Cancelling\u2026', - cancel_failed: 'Cancel failed: ', + cancel_failed: 'Cancel failed.', mic_denied: 'Microphone access denied. Check browser permissions.', mic_insecure_origin: 'Voice input needs a secure connection. Open Hermes over HTTPS or from localhost to use the microphone.', mic_no_speech: 'No speech detected. Try again.', @@ -1777,7 +1777,7 @@ const LOCALES = { _speech: 'it-IT', // boot.js cancelling: 'Annullamento\u2026', - cancel_failed: 'Annullamento fallito: ', + cancel_failed: 'Annullamento fallito.', mic_denied: 'Accesso al microfono negato. Controlla i permessi del browser.', mic_insecure_origin: 'L\'input vocale richiede una connessione sicura. Apri Hermes tramite HTTPS o da localhost per usare il microfono.', mic_no_speech: 'Nessuna voce rilevata. Riprova.', @@ -3519,7 +3519,7 @@ const LOCALES = { _speech: 'ja-JP', // boot.js cancelling: 'キャンセル中…', - cancel_failed: 'キャンセル失敗: ', + cancel_failed: 'キャンセルに失敗しました。', mic_denied: 'マイクへのアクセスが拒否されました。ブラウザの権限を確認してください。', mic_insecure_origin: '音声入力にはセキュアな接続が必要です。マイクを使うには HTTPS または localhost 経由で Hermes を開いてください。', mic_no_speech: '音声が検出されませんでした。もう一度お試しください。', @@ -5265,7 +5265,7 @@ const LOCALES = { _label: 'Русский', _speech: 'ru-RU', cancelling: 'Отменяю…', - cancel_failed: 'Не удалось отменить: ', + cancel_failed: 'Не удалось отменить.', mic_denied: 'Доступ к микрофону запрещён. Проверьте разрешения браузера.', mic_insecure_origin: 'Для голосового ввода нужно защищённое соединение. Откройте Hermes по HTTPS или с localhost, чтобы использовать микрофон.', mic_no_speech: 'Речь не распознана. Попробуйте ещё раз.', @@ -6987,7 +6987,7 @@ const LOCALES = { _speech: 'es-ES', // boot.js cancelling: 'Cancelando…', - cancel_failed: 'Error al cancelar: ', + cancel_failed: 'Error al cancelar.', mic_denied: 'Acceso al micrófono denegado. Revisa los permisos del navegador.', mic_insecure_origin: 'La entrada de voz necesita una conexión segura. Abre Hermes con HTTPS o desde localhost para usar el micrófono.', mic_no_speech: 'No se detectó voz. Inténtalo de nuevo.', @@ -8675,7 +8675,7 @@ const LOCALES = { _speech: 'de-DE', // boot.js cancelling: 'Wird abgebrochen\u2026', - cancel_failed: 'Abbrechen fehlgeschlagen: ', + cancel_failed: 'Abbrechen fehlgeschlagen.', mic_denied: 'Mikrofonzugriff verweigert. Überprüfen Sie die Browserberechtigungen.', mic_insecure_origin: 'Für die Spracheingabe ist eine sichere Verbindung erforderlich. Öffne Hermes über HTTPS oder von localhost, um das Mikrofon zu verwenden.', mic_no_speech: 'Keine Sprache erkannt. Versuchen Sie es erneut.', @@ -10357,7 +10357,7 @@ const LOCALES = { _speech: 'zh-CN', // boot.js cancelling: '正在取消...', - cancel_failed: '取消失败：', + cancel_failed: '取消失败。', mic_denied: '麦克风访问被拒绝，请检查浏览器权限。', mic_insecure_origin: '语音输入需要安全连接。请通过 HTTPS 协议或从本地主机打开Hermes，以便使用麦克风。', mic_no_speech: '没有检测到语音，请再试一次。', @@ -12034,7 +12034,7 @@ const LOCALES = { _speech: 'zh-TW', // boot.js cancelling: '正在取消……', - cancel_failed: '取消失敗：', + cancel_failed: '取消失敗。', mic_denied: '麥克風存取遭拒。請檢查瀏覽器權限。', mic_insecure_origin: '語音輸入需要安全連線。請透過 HTTPS 或 localhost 開啟 Hermes 才能使用麥克風。', mic_no_speech: '未偵測到語音。請再試一次。', @@ -13778,7 +13778,7 @@ const LOCALES = { _speech: 'pt-BR', // boot.js cancelling: 'Cancelando…', - cancel_failed: 'Falha ao cancelar: ', + cancel_failed: 'Falha ao cancelar.', mic_denied: 'Acesso ao microfone negado. Verifique as permissões do navegador.', mic_insecure_origin: 'A entrada de voz precisa de uma conexão segura. Abra o Hermes por HTTPS ou a partir do localhost para usar o microfone.', mic_no_speech: 'Nenhuma fala detectada. Tente novamente.', @@ -15340,7 +15340,7 @@ const LOCALES = { _speech: 'ko-KR', // boot.js cancelling: '취소 중\u2026', - cancel_failed: '취소 실패: ', + cancel_failed: '취소에 실패했습니다.', mic_denied: '마이크 접근이 거부되었습니다. 브라우저 권한을 확인하세요.', mic_insecure_origin: '음성 입력에는 보안 연결이 필요합니다. 마이크를 사용하려면 HTTPS 또는 localhost를 통해 Hermes를 여세요.', mic_no_speech: '음성이 감지되지 않았습니다. 다시 시도하세요.', @@ -17071,7 +17071,7 @@ const LOCALES = { _label: 'Français', _speech: 'fr-FR', cancelling: 'Annulation\u2026', - cancel_failed: 'Échec de l\'annulation : ', + cancel_failed: 'Échec de l\'annulation.', mic_denied: 'Accès au microphone refusé. Vérifiez les autorisations du navigateur.', mic_insecure_origin: 'La saisie vocale nécessite une connexion sécurisée. Ouvrez Hermes en HTTPS ou depuis localhost pour utiliser le microphone.', mic_no_speech: 'Aucune parole détectée. Réessayez.', @@ -20000,7 +20000,7 @@ const LOCALES = { btw_no_answer: 'Nebyla přijata žádná odpověď.', busy_interrupt_confirm: 'Přerušeno — odeslání nové zprávy', busy_steer_fallback: 'Řízení není k dispozici — návrh byl obnoven', - cancel_failed: 'Zrušení selhalo: ', + cancel_failed: 'Zrušení selhalo.', cancel_unavailable: 'Zrušení není k dispozici.', cancelling: 'Rušení\u2026', clarify_heading: 'Zde je zapotřebí vyjasnění.', @@ -20492,7 +20492,7 @@ const LOCALES = { _speech: 'tr-TR', // boot.js cancelling: 'İptal ediliyor\u2026', - cancel_failed: 'İptal başarısız oldu:', + cancel_failed: 'İptal başarısız oldu.', mic_denied: 'Mikrofon erişimi reddedildi. Tarayıcı izinlerini kontrol edin.', mic_insecure_origin: 'Sesli giriş güvenli bir bağlantı gerektirir. Mikrofonu kullanmak için Hermes\'i HTTPS üzerinden veya localhost\'tan açın.', mic_no_speech: 'Konuşma algılanmadı. Tekrar deneyin.', @@ -22225,7 +22225,7 @@ const LOCALES = { _speech: 'pl-PL', // boot.js cancelling: 'Anulowanie…', - cancel_failed: 'Anulowanie nie powiodło się: ', + cancel_failed: 'Anulowanie nie powiodło się.', mic_denied: 'Odmowa dostępu do mikrofonu. Sprawdź uprawnienia przeglądarki.', mic_insecure_origin: 'Wprowadzanie głosowe wymaga bezpiecznego połączenia. Otwórz Hermes przez HTTPS lub z localhost, aby użyć mikrofonu.', mic_no_speech: 'Nie wykryto mowy. Spróbuj ponownie.', @@ -23962,7 +23962,7 @@ const LOCALES = { _speech: 'vi-VN', // boot.js cancelling: 'Đang hủy…', - cancel_failed: 'Hủy thất bại: ', + cancel_failed: 'Hủy thất bại.', mic_denied: 'Quyền truy cập microphone bị từ chối. Kiểm tra quyền trình duyệt.', mic_no_speech: 'Không phát hiện giọng nói. Hãy thử lại.', mic_network: 'Nhận diện giọng nói không khả dụng.', diff --git a/static/sessions.js b/static/sessions.js index 1100794a..db7ee1c0 100644 --- a/static/sessions.js +++ b/static/sessions.js @@ -2315,6 +2315,7 @@ async function loadSession(sid){ setComposerStatus(''); updateQueueBadge(sid); syncTopbar();renderMessages(sameSessionForceReload?{preserveScroll:true}:undefined); + startApprovalPolling(sid); if(typeof resumeManualCompressionForSession==='function') resumeManualCompressionForSession(sid); // Workspace refresh is guarded by session id inside loadDir(); keep it // after the transcript's first paint so chat switching is not competing diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index 3b06d611..7561490e 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -489,7 +489,7 @@ def test_gateway_runs_api_streaming_same_run_fifo_emits_head_and_promotes_succes }) handler.send_response.assert_called_with(200) - respond_approval.assert_called_once_with("run-fifo", "approval-first", "deny") + respond_approval.assert_called_once_with("run-fifo", "", "deny") promoted = approvals.gateway_pending_mirror(sid, run_id="run-fifo") assert promoted is not None assert promoted["approval_id"] == "approval-second" @@ -639,7 +639,7 @@ def test_gateway_approval_response_relay_cleans_only_selected_shared_id_run(): }) handler.send_response.assert_called_with(200) - respond_approval.assert_called_once_with("run-live", "same-id", "once") + respond_approval.assert_called_once_with("run-live", "", "once") assert approvals.gateway_pending_mirror(sid, approval_id="same-id", run_id="run-live") is None assert approvals.gateway_pending_mirror(sid, approval_id="same-id", run_id="run-old") is not None finally: @@ -723,7 +723,7 @@ def test_gateway_approval_relay_does_not_unblock_unrelated_local_head(): }) handler.send_response.assert_called_with(200) - respond_approval.assert_called_once_with("run-remote", "remote-a", "once") + respond_approval.assert_called_once_with("run-remote", "", "once") assert local_entry.event.is_set() is False assert local_entry.result is None assert approvals.gateway_pending_mirror(sid, approval_id="remote-a", run_id="run-remote") is None @@ -1207,7 +1207,7 @@ def test_runs_api_cancel_after_run_creation_stops_remote_run_before_events(): _STREAM_RUN_IDS.pop(stream_id, None) -def test_runs_api_cancel_after_run_creation_keeps_stream_open_when_stop_fails(): +def test_runs_api_cancel_after_run_creation_reports_stop_failure(): from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming stream_id = "sid-run-create-cancel-stop-failed" @@ -1272,14 +1272,11 @@ def test_runs_api_cancel_after_run_creation_keeps_stream_open_when_stop_fails(): cancel_event=cancel_event, cfg={}, ) - assert final_text == "Hello" + assert final_text is None assert usage == {} stop_run.assert_called_once_with(stream_id) - assert requests == [ - "http://gw:8642/v1/runs", - "http://gw:8642/v1/runs/run-pre-events-stop-failed/events", - ] - assert [event for event, _data in events] == ["token"] + assert requests == ["http://gw:8642/v1/runs"] + assert [event for event, _data in events] == ["apperror"] finally: _STREAM_RUN_IDS.pop(stream_id, None) @@ -1457,7 +1454,7 @@ def test_chat_cancel_surfaces_redirected_gateway_stop(monkeypatch): monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) monkeypatch.setattr(routes, "webui_gateway_chat_enabled", lambda *_args, **_kwargs: True) monkeypatch.setattr(routes, "stream_owner_session_id", lambda _stream_id: sid) - monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True)) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _stream_id: False) def fake_j(handler, data, status=200, extra_headers=None): @@ -1539,7 +1536,7 @@ def test_chat_cancel_surfaces_gateway_stop_failure(monkeypatch): monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) monkeypatch.setattr(routes, "webui_gateway_chat_enabled", lambda *_args, **_kwargs: True) monkeypatch.setattr(routes, "stream_owner_session_id", lambda _stream_id: sid) - monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True)) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _stream_id: False) def fake_j(handler, data, status=200, extra_headers=None): @@ -1565,7 +1562,7 @@ def test_chat_cancel_surfaces_gateway_stop_failure(monkeypatch): approvals._pending.pop(sid, None) -def test_chat_cancel_blocks_while_gateway_run_id_is_pending(monkeypatch): +def test_chat_cancel_defers_while_gateway_run_id_is_pending(monkeypatch): import urllib.parse from api import routes @@ -1575,7 +1572,7 @@ def test_chat_cancel_blocks_while_gateway_run_id_is_pending(monkeypatch): called = {"cancel": False, "stop": False} monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) - monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True)) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) monkeypatch.setattr("api.gateway_chat.gateway_run_id_pending", lambda _stream_id: True) monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _stream_id: called.__setitem__("stop", True)) monkeypatch.setattr("time.sleep", lambda _seconds: None) @@ -1588,18 +1585,18 @@ def test_chat_cancel_blocks_while_gateway_run_id_is_pending(monkeypatch): monkeypatch.setattr(routes, "j", fake_j) parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") routes.handle_get(object(), parsed) - assert captured["status"] == 502 + assert captured["status"] == 200 assert captured["payload"] == { - "ok": False, - "cancelled": False, + "ok": True, + "cancelled": True, + "deferred": True, "stream_id": stream_id, - "error": "Gateway stop failed", } - assert called["cancel"] is False + assert called["cancel"] is True assert called["stop"] is False -def test_chat_cancel_waits_for_pending_gateway_run_id_then_stops_remote_run(monkeypatch): +def test_chat_cancel_defers_stop_until_gateway_run_id_exists(monkeypatch): import urllib.parse from api import routes @@ -1629,8 +1626,8 @@ def test_chat_cancel_waits_for_pending_gateway_run_id_then_stops_remote_run(monk try: routes.handle_get(object(), parsed) assert captured["status"] == 200 - assert captured["payload"] == {"ok": True, "cancelled": True, "stream_id": stream_id} - assert called["stop"] is True + assert captured["payload"] == {"ok": True, "cancelled": True, "deferred": True, "stream_id": stream_id} + assert called["stop"] is False assert called["cancel"] is True finally: _STREAM_RUN_IDS.pop(stream_id, None) @@ -2130,7 +2127,7 @@ def test_gateway_approval_response_relay(): _handle_approval_respond(handler, body) assert captured.get("url", "") == "http://gw:8642/v1/runs/run%20abc%2F1/approval" - assert captured["body"] == {"choice": "once", "approval_id": "appr x/y"} + assert captured["body"] == {"choice": "once", "approval_id": ""} handler.send_response.assert_called_with(200) # Cleanup. diff --git a/tests/test_issue4771_local_approval_regression.py b/tests/test_issue4771_local_approval_regression.py index 5a3da490..3c45c978 100644 --- a/tests/test_issue4771_local_approval_regression.py +++ b/tests/test_issue4771_local_approval_regression.py @@ -381,10 +381,8 @@ def test_local_plain_pending_without_id_beats_active_remote_candidate_run(): _cleanup(sid) -def test_gateway_mirrored_approval_without_run_still_409s(): - """#4771 behaviour preserved: on the gateway backend, a mirrored approval - whose run is gone must still surface the actionable 409 so the card stays - visible instead of silently failing.""" +def test_gateway_mirrored_approval_without_run_retires_locally(): + """A gateway mirror without a run retires instead of leaving a ghost card.""" sid = f"gw-approval-{uuid.uuid4().hex[:8]}" _register_session(sid) # no active_stream_id -> no _STREAM_RUN_IDS mapping try: @@ -396,10 +394,8 @@ def test_gateway_mirrored_approval_without_run_still_409s(): {"session_id": sid, "choice": "once", "approval_id": approval_id}, ) resp = handler.json() - assert handler.status == 409, f"expected 409, got {handler.status}: {resp}" - assert resp.get("ok") is False - assert resp.get("relayed") is False - assert resp.get("code") == "gateway_run_unavailable" + assert handler.status == 200, f"expected 200, got {handler.status}: {resp}" + assert resp == {"ok": True, "choice": "once", "local_retired": True} finally: _cleanup(sid) diff --git a/tests/test_session_attention_badges.py b/tests/test_session_attention_badges.py index e67f2328..b9ff2b1e 100644 --- a/tests/test_session_attention_badges.py +++ b/tests/test_session_attention_badges.py @@ -148,7 +148,7 @@ def test_attention_summary_keeps_direct_runs_mirror_after_stream_pointer_clears( }) handler.send_response.assert_called_with(200) - respond_approval.assert_called_once_with("run-retained", "approval-retained", "deny") + respond_approval.assert_called_once_with("run-retained", "", "deny") assert json.loads(handler.wfile.getvalue().decode("utf-8"))["ok"] is True finally: approvals._pending.pop(sid, None) From a91bd88c70d37d138d321b5714ae44591f20bd38 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 19 Jul 2026 15:35:45 -0400 Subject: [PATCH 08/20] fix(gateway): preserve approval identity provenance --- api/gateway_chat.py | 2 ++ api/route_approvals.py | 5 ++++ api/routes.py | 4 +++- tests/test_gateway_approval_runs_api.py | 23 +++++++++++++++++++ ...est_issue4771_local_approval_regression.py | 5 ++++ 5 files changed, 38 insertions(+), 1 deletion(-) diff --git a/api/gateway_chat.py b/api/gateway_chat.py index c0746074..b58985bd 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -505,6 +505,8 @@ def _run_gateway_runs_api_streaming( approval_data = _gateway_runs_approval_event(payload) if approval_data: approval_data["run_id"] = run_id + from api.config import gateway_supports_approval_identity_v1 + approval_data["_gateway_agent_identity_v1"] = gateway_supports_approval_identity_v1(base_url, api_key) from api.route_approvals import submit_gateway_pending_mirror head, total = submit_gateway_pending_mirror(session_id, approval_data) put_gateway_event("approval", {**(head or approval_data), "pending_count": total}) diff --git a/api/route_approvals.py b/api/route_approvals.py index 41bb6e5c..0402360d 100644 --- a/api/route_approvals.py +++ b/api/route_approvals.py @@ -47,6 +47,7 @@ _approval_sse_subscribers: dict[str, list[queue.Queue]] = {} _GATEWAY_MIRROR_FLAG = "_gateway_mirror" _GATEWAY_MIRROR_TOKEN = "_gateway_mirror_token" _GATEWAY_ENTRY_DATA_TOKEN_KEY = "_webui_mirror_token" +_GATEWAY_AGENT_IDENTITY_V1 = "_gateway_agent_identity_v1" def _approval_sse_subscribe(session_id: str) -> queue.Queue: @@ -323,6 +324,10 @@ def retire_gateway_pending_mirror(session_key: str, approval_id: str = "", run_i gateway_queue_changed = False if approval_id: match = _gateway_pending_mirror_locked(session_key, approval_id, run_id) + if match is None and not normalized_run_id: + match = next((entry for entry in entries if _is_gateway_mirror_entry(entry) + and not str(entry.get("run_id") or "").strip() + and str(entry.get("approval_id") or "").strip() == approval_id), None) retired = [match] if match else [] else: retired = [ diff --git a/api/routes.py b/api/routes.py index 8b8854f1..db26c696 100644 --- a/api/routes.py +++ b/api/routes.py @@ -23999,6 +23999,7 @@ def _handle_approval_respond(handler, body): webui_gateway_chat_enabled, ) from api.config import get_config as _get_config, gateway_supports_approval_identity_v1 + from api.route_approvals import _GATEWAY_AGENT_IDENTITY_V1 s = get_session(sid) _candidate_run_id = None if s is not None: @@ -24105,7 +24106,7 @@ def _handle_approval_respond(handler, body): _base = _gateway_base_url(_cfg) _key = _gateway_api_key() try: - identity_v1 = gateway_supports_approval_identity_v1(_base, _key) + identity_v1 = bool(matched_mirror.get(_GATEWAY_AGENT_IDENTITY_V1)) and gateway_supports_approval_identity_v1(_base, _key) HttpRunnerClient(base_url=_base, api_key=_key).respond_approval( _run_id, matched_mirror["approval_id"] if identity_v1 else "", choice ) @@ -24127,6 +24128,7 @@ def _handle_approval_respond(handler, body): if webui_gateway_chat_enabled(_get_config()) and _gateway_pending_approval_without_run_id( sid, approval_id ): + resolve_gateway_pending_local(sid, approval_id, choice) retire_gateway_pending_mirror(sid, approval_id=approval_id) return j(handler, {"ok": True, "choice": choice, "local_retired": True}) except Exception: diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index 7561490e..9f4e2dd4 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -2135,6 +2135,29 @@ def test_gateway_approval_response_relay(): approvals._pending.pop("sess-relay", None) +def test_synthetic_gateway_identity_stays_fifo_after_capability_upgrade(): + """A local gwrun ID never becomes remote authority after a capability refresh.""" + from api import routes + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + _STREAM_RUN_IDS["sid-upgrade"] = "run-upgrade" + approvals.submit_gateway_pending_mirror("sess-upgrade", { + "run_id": "run-upgrade", "approval_id": "gwrun:run-upgrade:local", "command": "echo x" + }) + handler = MagicMock() + handler.wfile = io.BytesIO() + with patch("api.routes.get_session", return_value=SimpleNamespace(active_stream_id="sid-upgrade")), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=True), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond: + routes._handle_approval_respond(handler, { + "session_id": "sess-upgrade", "choice": "once", "approval_id": "gwrun:run-upgrade:local", + }) + respond.assert_called_once_with("run-upgrade", "", "once") + _STREAM_RUN_IDS.pop("sid-upgrade", None) + approvals._pending.pop("sess-upgrade", None) + + def test_gateway_approval_response_without_approval_id_409s(): """Gateway relay requires the emitted per-approval id, not bare run state.""" from api.gateway_chat import _STREAM_RUN_IDS diff --git a/tests/test_issue4771_local_approval_regression.py b/tests/test_issue4771_local_approval_regression.py index 3c45c978..68f5bd33 100644 --- a/tests/test_issue4771_local_approval_regression.py +++ b/tests/test_issue4771_local_approval_regression.py @@ -396,6 +396,11 @@ def test_gateway_mirrored_approval_without_run_retires_locally(): resp = handler.json() assert handler.status == 200, f"expected 200, got {handler.status}: {resp}" assert resp == {"ok": True, "choice": "once", "local_retired": True} + assert entry.event.is_set() + assert entry.result == "once" + with ta._lock: + assert sid not in ta._pending + assert sid not in ta._gateway_queues finally: _cleanup(sid) From fb880955303a374362013a173f5f4d8e24e3e897 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 19 Jul 2026 15:58:37 -0400 Subject: [PATCH 09/20] fix(gateway): require agent-issued approval identities --- api/gateway_chat.py | 2 +- tests/test_gateway_approval_runs_api.py | 53 +++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/api/gateway_chat.py b/api/gateway_chat.py index b58985bd..187f643c 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -506,7 +506,7 @@ def _run_gateway_runs_api_streaming( if approval_data: approval_data["run_id"] = run_id from api.config import gateway_supports_approval_identity_v1 - approval_data["_gateway_agent_identity_v1"] = gateway_supports_approval_identity_v1(base_url, api_key) + approval_data["_gateway_agent_identity_v1"] = bool(approval_data.get("approval_id")) and gateway_supports_approval_identity_v1(base_url, api_key) from api.route_approvals import submit_gateway_pending_mirror head, total = submit_gateway_pending_mirror(session_id, approval_data) put_gateway_event("approval", {**(head or approval_data), "pending_count": total}) diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index 9f4e2dd4..1f0d3ce6 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -2158,6 +2158,59 @@ def test_synthetic_gateway_identity_stays_fifo_after_capability_upgrade(): approvals._pending.pop("sess-upgrade", None) +def test_agent_identity_v1_relays_the_ingress_identity_exactly(): + """An Agent-issued v1 identity is the only targeted relay authority.""" + from api import routes + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + _STREAM_RUN_IDS["sid-v1"] = "run-v1" + approvals.submit_gateway_pending_mirror("sess-v1", { + "run_id": "run-v1", "approval_id": "agent-v1", "command": "echo x", + "_gateway_agent_identity_v1": True, + }) + mirror = approvals.gateway_pending_mirror("sess-v1", approval_id="agent-v1", run_id="run-v1") + assert mirror["_gateway_agent_identity_v1"] is True + handler = MagicMock() + handler.wfile = io.BytesIO() + with patch("api.routes.get_session", return_value=SimpleNamespace(active_stream_id="sid-v1")), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=True), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond: + routes._handle_approval_respond(handler, { + "session_id": "sess-v1", "choice": "once", "approval_id": "agent-v1", + }) + respond.assert_called_once_with("run-v1", "agent-v1", "once") + _STREAM_RUN_IDS.pop("sid-v1", None) + approvals._pending.pop("sess-v1", None) + + +def test_capability_v1_empty_ingress_identity_stays_fifo_only(): + """Capability alone cannot promote a locally synthesized gwrun identity.""" + from api import routes + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + _STREAM_RUN_IDS["sid-v1-empty"] = "run-v1-empty" + approvals.submit_gateway_pending_mirror("sess-v1-empty", { + "run_id": "run-v1-empty", "approval_id": "", "command": "echo x", + "_gateway_agent_identity_v1": False, + }) + mirror = approvals.gateway_pending_mirror("sess-v1-empty", run_id="run-v1-empty") + assert mirror["approval_id"].startswith("gwrun:run-v1-empty:") + assert mirror["_gateway_agent_identity_v1"] is False + handler = MagicMock() + handler.wfile = io.BytesIO() + with patch("api.routes.get_session", return_value=SimpleNamespace(active_stream_id="sid-v1-empty")), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=True), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond: + routes._handle_approval_respond(handler, { + "session_id": "sess-v1-empty", "choice": "once", "approval_id": mirror["approval_id"], + }) + respond.assert_called_once_with("run-v1-empty", "", "once") + _STREAM_RUN_IDS.pop("sid-v1-empty", None) + approvals._pending.pop("sess-v1-empty", None) + + def test_gateway_approval_response_without_approval_id_409s(): """Gateway relay requires the emitted per-approval id, not bare run state.""" from api.gateway_chat import _STREAM_RUN_IDS From 918b9ddb630dd8b0002483dd4da5934bb6d8b70c Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 19 Jul 2026 16:06:37 -0400 Subject: [PATCH 10/20] test(gateway): align approval lifecycle regressions --- api/gateway_chat.py | 1 - tests/test_gateway_approval_legacy_path.py | 27 ++++++++-------------- tests/test_gateway_approval_runs_api.py | 6 +++-- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/api/gateway_chat.py b/api/gateway_chat.py index 187f643c..32c4cb37 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -10,7 +10,6 @@ import uuid import urllib.error import urllib.parse import urllib.request -import uuid from typing import Any from api.config import ( diff --git a/tests/test_gateway_approval_legacy_path.py b/tests/test_gateway_approval_legacy_path.py index d9a1c7d1..c394224c 100644 --- a/tests/test_gateway_approval_legacy_path.py +++ b/tests/test_gateway_approval_legacy_path.py @@ -6,6 +6,7 @@ PR #4495 fixed the runs API path but left the legacy path without approval handl from __future__ import annotations import json +import threading from pathlib import Path from unittest.mock import MagicMock, patch @@ -762,8 +763,8 @@ def test_gateway_mode_no_pending_click_stays_non_409(): assert captured["payload"].get("code") != "gateway_run_unavailable" -def test_legacy_approval_without_run_id_stays_actionable(): - """Legacy approvals without a run_id must fail explicitly and keep the mirror live.""" +def test_legacy_approval_without_run_id_retires_locally(): + """A no-run legacy mirror retires locally instead of becoming a ghost card.""" from types import SimpleNamespace from api import routes as r from api import route_approvals as ra @@ -842,22 +843,15 @@ def test_legacy_approval_without_run_id_stays_actionable(): ] assert approval_events approval_data = approval_events[0][1] + local_entry = SimpleNamespace(data=dict(approval_data), event=threading.Event(), result=None) with ra._lock: - r._gateway_queues[session_id] = [SimpleNamespace(data=dict(approval_data))] + r._gateway_queues[session_id] = [local_entry] ra.submit_gateway_pending_mirror(session_id, approval_data) with ra._lock: pending_queue = r._pending.get(session_id) assert isinstance(pending_queue, list) approval_id = pending_queue[0]["approval_id"] - # The relay-unavailable 409 is only meaningful when the WebUI is - # actually running the gateway chat backend. On a gateway deployment - # the backend env is process-wide (not just during the stream), so - # assert the 409 with HERMES_WEBUI_CHAT_BACKEND=gateway active at - # respond time. Without this scope the handler now (correctly) treats - # a mirrored approval on the default LOCAL backend as locally - # resolvable and falls through instead of 409ing — see - # test_issue4771_local_approval_regression.py (#4771 follow-up). with patch.dict("os.environ", {"HERMES_WEBUI_CHAT_BACKEND": "gateway"}), \ patch("api.routes.get_session", return_value=mock_session), \ patch("api.routes.j", new=fake_j): @@ -866,13 +860,12 @@ def test_legacy_approval_without_run_id_stays_actionable(): {"session_id": session_id, "choice": "once", "approval_id": approval_id}, ) - assert captured["status"] == 409 - assert captured["payload"]["code"] == "gateway_run_unavailable" - assert captured["payload"]["error"] == r._GATEWAY_APPROVAL_RELAY_UNAVAILABLE + assert captured["status"] == 200 + assert captured["payload"] == {"ok": True, "choice": "once", "local_retired": True} + assert local_entry.event.is_set() + assert local_entry.result == "once" with ra._lock: - pending_queue = r._pending.get(session_id) - assert isinstance(pending_queue, list) - assert pending_queue[0]["approval_id"] == approval_id + assert session_id not in r._pending finally: with STREAMS_LOCK: STREAMS.pop(stream_id, None) diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index 1f0d3ce6..6806a1f7 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -551,8 +551,10 @@ def test_empty_id_runs_approval_reaches_real_response_lifecycle(): cancel_event=threading.Event(), ) approval_id = events[0][1]["approval_id"] - assert approval_id.startswith(f"gwrun:run-real-{choice}:") - assert approvals.gateway_pending_mirror(sid, approval_id=approval_id)["run_id"] == f"run-real-{choice}" + assert approval_id + mirror = approvals.gateway_pending_mirror(sid, approval_id=approval_id) + assert mirror["run_id"] == f"run-real-{choice}" + assert mirror["approval_id"] == approval_id routes._handle_approval_respond(handler, { "session_id": sid, "choice": choice, "approval_id": approval_id, }) From ecdd0af5731db47a80ab73e640689b28ca91c4e6 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 19 Jul 2026 16:12:55 -0400 Subject: [PATCH 11/20] test(gateway): cover capability-gated approval IDs --- tests/test_gateway_approval_legacy_path.py | 4 ++-- tests/test_gateway_approval_runs_api.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_gateway_approval_legacy_path.py b/tests/test_gateway_approval_legacy_path.py index c394224c..dabbf7ec 100644 --- a/tests/test_gateway_approval_legacy_path.py +++ b/tests/test_gateway_approval_legacy_path.py @@ -374,7 +374,7 @@ def test_legacy_approval_records_run_id_for_response_relay(): assert captured.get("url", "") == "http://gw:8642/v1/runs/run-legacy-1/approval", ( f"approval respond must relay to the gateway run; got {captured.get('url')!r}" ) - assert captured["body"] == {"choice": "once", "approval_id": "appr-legacy-runid"} + assert captured["body"] == {"choice": "once", "approval_id": ""} handler.send_response.assert_called_with(200) finally: with STREAMS_LOCK: @@ -697,7 +697,7 @@ def test_mirrored_run_id_survives_active_stream_loss(): assert captured.get("url", "") == f"http://gw:8642/v1/runs/{run_id}/approval", ( f"approval respond must relay to the mirrored gateway run; got {captured.get('url')!r}" ) - assert captured["body"] == {"choice": "once", "approval_id": approval_id} + assert captured["body"] == {"choice": "once", "approval_id": ""} handler.send_response.assert_called_with(200) assert entry.event.is_set(), "mirrored gateway approval was not resolved" assert entry.result == "once" diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index 6806a1f7..d2eaf917 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -305,7 +305,15 @@ def test_gateway_approval_event_translation(): assert result["run_id"] == "run-999" assert result["approval_id"] == "appr-1" empty_id = _gateway_runs_approval_event({**payload, "approval_id": "", "id": ""}) - assert empty_id["approval_id"] == "" + import api.route_approvals as approvals + sid = "translation-empty-id" + try: + approvals.submit_gateway_pending_mirror(sid, empty_id) + mirror = approvals.gateway_pending_mirror(sid, run_id="run-999") + assert mirror["approval_id"] + assert mirror.get("_gateway_agent_identity_v1") is not True + finally: + approvals._pending.pop(sid, None) downgraded = _gateway_runs_approval_event({ "command": "rm -rf /tmp/x", From 0a3a6308567f5ac4360ec4346662a70eb1004516 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 20 Jul 2026 08:55:14 -0400 Subject: [PATCH 12/20] fix(gateway): keep approval and stop acknowledgements honest --- api/gateway_chat.py | 6 +- api/route_approvals.py | 49 ++++++ api/routes.py | 22 ++- static/ui.js | 2 +- tests/test_approval_unblock.py | 108 +++++++++++- tests/test_gateway_approval_runs_api.py | 166 ++++++++++++++++++ .../test_issue6008_sidebar_stop_failure_ui.py | 34 ++++ 7 files changed, 371 insertions(+), 16 deletions(-) diff --git a/api/gateway_chat.py b/api/gateway_chat.py index 32c4cb37..261691d4 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -345,7 +345,8 @@ def _gateway_runs_approval_event(payload: dict) -> dict | None: pattern_key = str(payload.get("pattern_key") or "").strip() args = payload.get("args") if isinstance(payload.get("args"), (list, dict)) else [] run_id = str(payload.get("run_id") or "").strip() - approval_id = str(payload.get("approval_id") or payload.get("id") or "").strip() + raw_approval_id = str(payload.get("approval_id") or payload.get("id") or "").strip() + approval_id = raw_approval_id risk = str(payload.get("risk_level") or "high").strip() choices = payload.get("choices") if isinstance(payload.get("choices"), list) else [] allow_permanent = payload.get("allow_permanent") @@ -363,6 +364,7 @@ def _gateway_runs_approval_event(payload: dict) -> dict | None: "risk_level": risk, "run_id": run_id, "approval_id": approval_id, + "_gateway_raw_approval_id_present": bool(raw_approval_id), "choices": choices, "allow_permanent": bool(allow_permanent), } @@ -505,7 +507,7 @@ def _run_gateway_runs_api_streaming( if approval_data: approval_data["run_id"] = run_id from api.config import gateway_supports_approval_identity_v1 - approval_data["_gateway_agent_identity_v1"] = bool(approval_data.get("approval_id")) and gateway_supports_approval_identity_v1(base_url, api_key) + approval_data["_gateway_agent_identity_v1"] = bool(approval_data.get("_gateway_raw_approval_id_present")) and gateway_supports_approval_identity_v1(base_url, api_key) from api.route_approvals import submit_gateway_pending_mirror head, total = submit_gateway_pending_mirror(session_id, approval_data) put_gateway_event("approval", {**(head or approval_data), "pending_count": total}) diff --git a/api/route_approvals.py b/api/route_approvals.py index 0402360d..c3ba6a09 100644 --- a/api/route_approvals.py +++ b/api/route_approvals.py @@ -518,6 +518,55 @@ def resolve_gateway_pending_local( return 1, head, total +def resolve_gateway_pending_local_no_run_mirror( + session_key: str, approval_id: str, choice: str, reason: str | None = None +) -> tuple[bool, int, dict | None, int]: + """Resolve an exact no-run mirror only while its parked producer still exists.""" + target = None + with _lock: + approval_id = str(approval_id or "").strip() + queue = _pending.get(session_key) + entries = queue if isinstance(queue, list) else [queue] if queue else [] + matched_mirror = next( + ( + entry for entry in entries + if _is_gateway_mirror_entry(entry) + and not str(entry.get("run_id") or "").strip() + and str(entry.get("approval_id") or "").strip() == approval_id + ), + None, + ) + if matched_mirror is None: + return False, 0, entries[0] if entries else None, len(entries) + + gateway_queue = _gateway_queues.get(session_key) or [] + for index, entry in enumerate(gateway_queue): + data = getattr(entry, "data", None) or {} + if str(data.get("approval_id") or "").strip() == approval_id: + target = gateway_queue.pop(index) + break + if target is None: + return True, 0, entries[0] if entries else None, len(entries) + + if gateway_queue: + _gateway_queues[session_key] = gateway_queue + else: + _gateway_queues.pop(session_key, None) + entries.remove(matched_mirror) + if entries: + _pending[session_key] = entries + else: + _pending.pop(session_key, None) + head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) + _approval_sse_notify_locked(session_key, head, total) + target.result = choice + if reason: + target.reason = reason + target.event.set() + publish_session_list_changed("attention_resolved") + return True, 1, head, total + + def submit_pending(session_key: str, approval: dict) -> None: """Append a pending approval to the per-session queue. diff --git a/api/routes.py b/api/routes.py index db26c696..82f241f5 100644 --- a/api/routes.py +++ b/api/routes.py @@ -9721,6 +9721,7 @@ from api.route_approvals import ( # noqa: F401 — re-exports for backward comp retire_gateway_pending_mirror, reconcile_gateway_pending_mirror_locked, resolve_gateway_pending_local, + resolve_gateway_pending_local_no_run_mirror, submit_gateway_pending_mirror, submit_pending, ) @@ -24123,14 +24124,19 @@ def _handle_approval_respond(handler, body): return j(handler, {"ok": False, "choice": choice, "relayed": False, "code": "gateway_run_unavailable", "error": _GATEWAY_APPROVAL_RELAY_UNAVAILABLE}, status=409) - # A no-run mirror is local visibility state only. Retire it instead of - # claiming that a remote approval can still be resolved. - if webui_gateway_chat_enabled(_get_config()) and _gateway_pending_approval_without_run_id( - sid, approval_id - ): - resolve_gateway_pending_local(sid, approval_id, choice) - retire_gateway_pending_mirror(sid, approval_id=approval_id) - return j(handler, {"ok": True, "choice": choice, "local_retired": True}) + # A no-run mirror is local visibility state only. Resolve it only while + # the exact parked producer still exists; otherwise keep the card live + # and fail closed instead of claiming success. + if webui_gateway_chat_enabled(_get_config()): + handled_no_run_mirror, resolved_count, _, _ = resolve_gateway_pending_local_no_run_mirror( + sid, approval_id, choice + ) + if handled_no_run_mirror and resolved_count == 1: + return j(handler, {"ok": True, "choice": choice, "local_retired": True}) + if handled_no_run_mirror: + return j(handler, {"ok": False, "choice": choice, "relayed": False, + "code": "gateway_run_unavailable", + "error": _GATEWAY_APPROVAL_RELAY_UNAVAILABLE}, status=409) except Exception: pass # fall through to local approval path diff --git a/static/ui.js b/static/ui.js index 79a0e837..81e30e4c 100644 --- a/static/ui.js +++ b/static/ui.js @@ -7901,7 +7901,7 @@ async function handleComposerPrimaryAction(){ const action=typeof getComposerPrimaryAction==='function'?getComposerPrimaryAction():'send'; if(action==='disabled') return; if(action==='stop'){ - if(typeof cancelStream==='function') await cancelStream('composer-stop'); + if(typeof cancelStream==='function' && !await cancelStream('composer-stop')) showToast(t('cancel_failed')); return; } await send(); diff --git a/tests/test_approval_unblock.py b/tests/test_approval_unblock.py index 757ab5e6..05579f13 100644 --- a/tests/test_approval_unblock.py +++ b/tests/test_approval_unblock.py @@ -527,8 +527,8 @@ class TestApprovalHTTPEndpoints: r._gateway_queues.pop(sid, None) pass # no external token state to clean - def test_gateway_mirror_without_run_id_returns_explicit_conflict(self, monkeypatch): - """Gateway-mirrored approvals without relayable run state must stay actionable.""" + def test_gateway_mirror_without_run_id_and_no_producer_returns_explicit_conflict(self, monkeypatch): + """A no-run mirror stays actionable when no local producer is parked.""" from api import routes as r from api import route_approvals as ra from api.gateway_chat import _STREAM_RUN_IDS @@ -561,7 +561,6 @@ class TestApprovalHTTPEndpoints: r._pending.pop(sid, None) r._gateway_queues.pop(sid, None) _STREAM_RUN_IDS.pop(stream_id, None) - r._gateway_queues[sid] = [_ApprovalEntry(approval)] try: ra.submit_gateway_pending_mirror(sid, approval) with _lock: @@ -585,8 +584,107 @@ class TestApprovalHTTPEndpoints: assert isinstance(pending_queue, list) assert len(pending_queue) == 1 assert pending_queue[0]["approval_id"] == approval_id - assert sid in r._gateway_queues - assert not r._gateway_queues[sid][0].event.is_set() + assert sid not in r._gateway_queues + finally: + with _lock: + r._pending.pop(sid, None) + r._gateway_queues.pop(sid, None) + _STREAM_RUN_IDS.pop(stream_id, None) + + def test_gateway_mirror_without_run_id_with_one_producer_resolves_exactly(self, monkeypatch): + """A no-run mirror retires only after its exact local producer resolves.""" + from api import routes as r + from api import route_approvals as ra + from api.gateway_chat import _STREAM_RUN_IDS + + sid = f"http-gateway-local-{uuid.uuid4().hex[:8]}" + stream_id = f"stream-local-{uuid.uuid4().hex[:8]}" + approval = {"command": "rm -rf /tmp/local", "description": "local"} + sibling = {"command": "rm -rf /tmp/sibling", "description": "sibling"} + captured = {} + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(r, "j", fake_j) + monkeypatch.setattr(r, "get_session", lambda _sid: SimpleNamespace(active_stream_id=stream_id)) + monkeypatch.setenv("HERMES_WEBUI_CHAT_BACKEND", "gateway") + entry = _ApprovalEntry(approval) + sibling_entry = _ApprovalEntry(sibling) + with _lock: + r._pending.pop(sid, None) + r._gateway_queues[sid] = [entry, sibling_entry] + try: + ra.submit_gateway_pending_mirror(sid, approval) + with _lock: + approval_id = r._pending[sid][0]["approval_id"] + + r._handle_approval_respond( + object(), {"session_id": sid, "choice": "once", "approval_id": approval_id} + ) + + assert captured["status"] == 200 + assert captured["payload"] == {"ok": True, "choice": "once", "local_retired": True} + assert entry.event.is_set() + assert entry.result == "once" + assert not sibling_entry.event.is_set() + with _lock: + assert r._pending[sid][0]["command"] == sibling["command"] + finally: + with _lock: + r._pending.pop(sid, None) + r._gateway_queues.pop(sid, None) + _STREAM_RUN_IDS.pop(stream_id, None) + + def test_gateway_no_run_mirror_stays_visible_when_exact_local_producer_is_gone(self, monkeypatch): + """A tokenized no-run mirror stays live when its parked producer already vanished.""" + from api import routes as r + from api import route_approvals as ra + from api.gateway_chat import _STREAM_RUN_IDS + + sid = f"http-gateway-race-{uuid.uuid4().hex[:8]}" + stream_id = f"stream-race-{uuid.uuid4().hex[:8]}" + approval = {"command": "rm -rf /tmp/race", "description": "race"} + captured = {} + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(r, "j", fake_j) + monkeypatch.setattr(r, "get_session", lambda _sid: SimpleNamespace(active_stream_id=stream_id)) + monkeypatch.setenv("HERMES_WEBUI_CHAT_BACKEND", "gateway") + entry = _ApprovalEntry(approval) + with _lock: + r._pending.pop(sid, None) + r._gateway_queues[sid] = [entry] + try: + ra.submit_gateway_pending_mirror(sid, approval) + with _lock: + approval_id = r._pending[sid][0]["approval_id"] + r._gateway_queues.pop(sid, None) + + r._handle_approval_respond( + object(), {"session_id": sid, "choice": "once", "approval_id": approval_id} + ) + + assert captured["status"] == 409 + assert captured["payload"] == { + "ok": False, + "choice": "once", + "relayed": False, + "code": "gateway_run_unavailable", + "error": r._GATEWAY_APPROVAL_RELAY_UNAVAILABLE, + } + assert not entry.event.is_set() + with _lock: + pending_queue = r._pending.get(sid) + assert isinstance(pending_queue, list) + assert len(pending_queue) == 1 + assert pending_queue[0]["approval_id"] == approval_id finally: with _lock: r._pending.pop(sid, None) diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index d2eaf917..b988a9d4 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -429,6 +429,172 @@ def test_gateway_runs_api_streaming_parses_real_run_events(): assert approvals.gateway_pending_mirror("sess1", run_id="run-abc") is None +def test_live_empty_ingress_id_stays_fifo_under_capability_v1(): + """A normalized fallback browser ID must not become authoritative Agent identity.""" + from api.config import STREAM_PARTIAL_TEXT, STREAM_REASONING_TEXT + from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming + from api import routes + import api.route_approvals as approvals + + sid = "sid-live-empty-ingress" + stream_id = "stream-live-empty-ingress" + events = [] + STREAM_PARTIAL_TEXT[stream_id] = "" + STREAM_REASONING_TEXT[stream_id] = "" + + class _JsonResponse: + def read(self, _limit=None): + return b'{"run_id":"run-empty-ingress"}' + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + class _SseResponse: + def __iter__(self): + return iter([ + b'data: {"event":"approval.request","command":"rm -rf /tmp/x","description":"Dangerous","run_id":"run-empty-ingress","approval_id":"","id":""}\n', + b'\n', + b'data: [DONE]\n', + b'\n', + ]) + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def fake_urlopen(req, *, timeout=None): + if req.full_url.endswith("/v1/runs"): + return _JsonResponse() + return _SseResponse() + + def fake_mapped_approval(payload): + assert payload["approval_id"] == "" + assert payload["id"] == "" + return { + "command": "rm -rf /tmp/x", + "description": "Dangerous", + "run_id": "run-empty-ingress", + "approval_id": "gwrun:normalized-fallback", + "_gateway_raw_approval_id_present": False, + } + + handler = MagicMock() + handler.wfile = io.BytesIO() + try: + with patch("urllib.request.urlopen", side_effect=fake_urlopen), \ + patch("api.gateway_chat._gateway_runs_approval_event", side_effect=fake_mapped_approval), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=True): + _run_gateway_runs_api_streaming( + session_id=sid, msg_text="hi", model="test-model", workspace="/tmp", + stream_id=stream_id, base_url="http://gw:8642", api_key="secret", + prefill_messages=[], body_extras={}, put_gateway_event=lambda event, data: events.append((event, data)), + cancel_event=threading.Event(), + ) + mirror = approvals.gateway_pending_mirror(sid, run_id="run-empty-ingress") + with patch("api.routes.get_session", return_value=SimpleNamespace(active_stream_id=stream_id)), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=True), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond: + routes._handle_approval_respond(handler, { + "session_id": sid, "choice": "once", "approval_id": "gwrun:normalized-fallback", + }) + respond.assert_called_once_with("run-empty-ingress", "", "once") + finally: + STREAM_PARTIAL_TEXT.pop(stream_id, None) + STREAM_REASONING_TEXT.pop(stream_id, None) + _STREAM_RUN_IDS.pop(stream_id, None) + + assert events[0][0] == "approval" + assert events[0][1]["approval_id"] == "gwrun:normalized-fallback" + assert events[0][1]["_gateway_agent_identity_v1"] is False + assert mirror["approval_id"] == "gwrun:normalized-fallback" + assert mirror["_gateway_agent_identity_v1"] is False + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_live_authoritative_ingress_id_relays_exactly_under_capability_v1(): + """A raw Agent-issued approval ID becomes authoritative only through live ingress.""" + from api.config import STREAM_PARTIAL_TEXT, STREAM_REASONING_TEXT + from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming + from api import routes + import api.route_approvals as approvals + + sid = "sid-live-authoritative-ingress" + stream_id = "stream-live-authoritative-ingress" + events = [] + STREAM_PARTIAL_TEXT[stream_id] = "" + STREAM_REASONING_TEXT[stream_id] = "" + + class _JsonResponse: + def read(self, _limit=None): + return b'{"run_id":"run-authoritative-ingress"}' + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + class _SseResponse: + def __iter__(self): + return iter([ + b'data: {"event":"approval.request","command":"echo hi","description":"Safe","run_id":"run-authoritative-ingress","approval_id":"agent-approval-1","id":"agent-approval-legacy"}\n', + b'\n', + b'data: [DONE]\n', + b'\n', + ]) + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def fake_urlopen(req, *, timeout=None): + if req.full_url.endswith("/v1/runs"): + return _JsonResponse() + return _SseResponse() + + handler = MagicMock() + handler.wfile = io.BytesIO() + try: + with patch("urllib.request.urlopen", side_effect=fake_urlopen), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=True): + _run_gateway_runs_api_streaming( + session_id=sid, msg_text="hi", model="test-model", workspace="/tmp", + stream_id=stream_id, base_url="http://gw:8642", api_key="secret", + prefill_messages=[], body_extras={}, put_gateway_event=lambda event, data: events.append((event, data)), + cancel_event=threading.Event(), + ) + mirror = approvals.gateway_pending_mirror( + sid, approval_id="agent-approval-1", run_id="run-authoritative-ingress" + ) + with patch("api.routes.get_session", return_value=SimpleNamespace(active_stream_id=stream_id)), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=True), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond: + routes._handle_approval_respond(handler, { + "session_id": sid, "choice": "once", "approval_id": "agent-approval-1", + }) + respond.assert_called_once_with("run-authoritative-ingress", "agent-approval-1", "once") + finally: + STREAM_PARTIAL_TEXT.pop(stream_id, None) + STREAM_REASONING_TEXT.pop(stream_id, None) + _STREAM_RUN_IDS.pop(stream_id, None) + + assert events[0][0] == "approval" + assert events[0][1]["approval_id"] == "agent-approval-1" + assert events[0][1]["_gateway_agent_identity_v1"] is True + assert mirror["approval_id"] == "agent-approval-1" + assert mirror["_gateway_agent_identity_v1"] is True + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + def test_gateway_runs_api_streaming_same_run_fifo_emits_head_and_promotes_successor(): """Runs API approval events publish the reconciled FIFO head and count.""" from api.config import STREAM_PARTIAL_TEXT, STREAM_REASONING_TEXT diff --git a/tests/test_issue6008_sidebar_stop_failure_ui.py b/tests/test_issue6008_sidebar_stop_failure_ui.py index 6965ab3b..7d5a5700 100644 --- a/tests/test_issue6008_sidebar_stop_failure_ui.py +++ b/tests/test_issue6008_sidebar_stop_failure_ui.py @@ -9,6 +9,7 @@ import subprocess REPO = pathlib.Path(__file__).parent.parent BOOT_JS = (REPO / "static" / "boot.js").read_text(encoding="utf-8") +UI_JS = (REPO / "static" / "ui.js").read_text(encoding="utf-8") COMMANDS_JS = (REPO / "static" / "commands.js").read_text(encoding="utf-8") MESSAGES_JS = (REPO / "static" / "messages.js").read_text(encoding="utf-8") SESSIONS_JS = (REPO / "static" / "sessions.js").read_text(encoding="utf-8") @@ -31,6 +32,7 @@ def _extract_function(src: str, name: str) -> str: CANCEL_SESSION_STREAM_SRC = _extract_function(BOOT_JS, "cancelSessionStream") +COMPOSER_PRIMARY_ACTION_SRC = _extract_function(UI_JS, "handleComposerPrimaryAction") def test_source_gates_sidebar_settle_on_http_success(): @@ -142,3 +144,35 @@ def test_failed_sidebar_stop_keeps_local_state(): assert result["approvalHides"] == 0 assert result["clarifyStops"] == 0 assert result["clarifyHides"] == 0 + + +def test_primary_composer_stop_surfaces_cancel_failure_and_preserves_success(): + script = r''' +const M = { toasts: [], sends: 0, results: [] }; +globalThis.window = {}; +globalThis.S = {}; +globalThis.getComposerPrimaryAction = () => 'stop'; +globalThis.showToast = (...args) => M.toasts.push(args); +globalThis.t = (key) => key; +globalThis.send = () => { M.sends += 1; }; +globalThis.cancelStream = async () => M.cancelResult; +__COMPOSER_PRIMARY_ACTION_SRC__ +for (const result of [false, true]) { + M.cancelResult = result; + await handleComposerPrimaryAction(); + M.results.push({ result, toasts: M.toasts.slice(), sends: M.sends }); +} +console.log(JSON.stringify(M)); +'''.replace("__COMPOSER_PRIMARY_ACTION_SRC__", COMPOSER_PRIMARY_ACTION_SRC) + completed = subprocess.run( + ["node", "--input-type=module", "-e", script], + cwd=str(REPO), text=True, capture_output=True, timeout=30, + ) + assert completed.returncode == 0, ( + f"node subprocess failed:\n--- stdout ---\n{completed.stdout}\n--- stderr ---\n{completed.stderr}" + ) + result = json.loads(completed.stdout.splitlines()[-1]) + assert result["results"] == [ + {"result": False, "toasts": [["cancel_failed"]], "sends": 0}, + {"result": True, "toasts": [["cancel_failed"]], "sends": 0}, + ] From 4a71e407b1542a99b48960982eebebfa70d54924 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 20 Jul 2026 10:51:33 -0400 Subject: [PATCH 13/20] fix(gateway): keep retained no-run approvals visible --- api/gateway_chat.py | 10 ++-- api/route_approvals.py | 21 +++++-- api/streaming.py | 8 +-- tests/test_approval_unblock.py | 79 ++++++++++++++++++++++--- tests/test_gateway_approval_runs_api.py | 29 ++++----- 5 files changed, 106 insertions(+), 41 deletions(-) diff --git a/api/gateway_chat.py b/api/gateway_chat.py index 261691d4..1f51fbc3 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -347,6 +347,8 @@ def _gateway_runs_approval_event(payload: dict) -> dict | None: run_id = str(payload.get("run_id") or "").strip() raw_approval_id = str(payload.get("approval_id") or payload.get("id") or "").strip() approval_id = raw_approval_id + if not approval_id: + approval_id = uuid.uuid4().hex risk = str(payload.get("risk_level") or "high").strip() choices = payload.get("choices") if isinstance(payload.get("choices"), list) else [] allow_permanent = payload.get("allow_permanent") @@ -703,14 +705,10 @@ def _clear_gateway_pending_state(session: Any, stream_id: str) -> None: def _cleanup_gateway_pending_mirror(session_id: str) -> None: try: from api.route_approvals import ( - _approval_sse_notify_locked, - _lock as _approval_lock, - reconcile_gateway_pending_mirror_locked, + retire_gateway_pending_mirror, ) - with _approval_lock: - head, total, _ = reconcile_gateway_pending_mirror_locked(session_id) - _approval_sse_notify_locked(session_id, head, total) + retire_gateway_pending_mirror(session_id) except Exception: logger.debug("Failed to reconcile gateway pending mirror during teardown", exc_info=True) diff --git a/api/route_approvals.py b/api/route_approvals.py index c3ba6a09..424e099f 100644 --- a/api/route_approvals.py +++ b/api/route_approvals.py @@ -46,6 +46,7 @@ except ImportError: _approval_sse_subscribers: dict[str, list[queue.Queue]] = {} _GATEWAY_MIRROR_FLAG = "_gateway_mirror" _GATEWAY_MIRROR_TOKEN = "_gateway_mirror_token" +_GATEWAY_MIRROR_RETAINED = "_gateway_mirror_retained" _GATEWAY_ENTRY_DATA_TOKEN_KEY = "_webui_mirror_token" _GATEWAY_AGENT_IDENTITY_V1 = "_gateway_agent_identity_v1" @@ -219,6 +220,10 @@ def reconcile_gateway_pending_mirror_locked(session_key: str) -> tuple[dict | No changed = True continue + if entry.get(_GATEWAY_MIRROR_RETAINED): + rebuilt.append(entry) + continue + if matches_live_head and not live_mirror_present: if entry_token != live_token: entry[_GATEWAY_MIRROR_TOKEN] = live_token @@ -334,7 +339,11 @@ def retire_gateway_pending_mirror(session_key: str, approval_id: str = "", run_i entry for entry in entries if _is_gateway_mirror_entry(entry) and str(entry.get("run_id") or "").strip() == normalized_run_id - ] if normalized_run_id else [] + ] if normalized_run_id else [ + entry for entry in entries + if _is_gateway_mirror_entry(entry) + and not str(entry.get("run_id") or "").strip() + ] if normalized_run_id: retained_gateway_queue = [] for entry in gateway_queue: @@ -344,10 +353,11 @@ def retire_gateway_pending_mirror(session_key: str, approval_id: str = "", run_i continue retained_gateway_queue.append(entry) if not retired and not gateway_queue_changed: - if approval_id: - head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) - _approval_sse_notify_locked(session_key, head, total) - return False + head, total, changed = reconcile_gateway_pending_mirror_locked(session_key) + _approval_sse_notify_locked(session_key, head, total) + if changed: + publish_session_list_changed("attention_resolved") + return changed for match in retired: entries.remove(match) if normalized_run_id and not approval_id: @@ -546,6 +556,7 @@ def resolve_gateway_pending_local_no_run_mirror( target = gateway_queue.pop(index) break if target is None: + matched_mirror[_GATEWAY_MIRROR_RETAINED] = True return True, 0, entries[0] if entries else None, len(entries) if gateway_queue: diff --git a/api/streaming.py b/api/streaming.py index bda92823..f4c8a143 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -7808,14 +7808,10 @@ def _run_agent_streaming( try: from api.route_approvals import ( submit_gateway_pending_mirror as _submit_pending_for_polling, - reconcile_gateway_pending_mirror_locked as _reconcile_gateway_pending_mirror_locked, - _approval_sse_notify_locked as _approval_sse_notify_locked, - _lock as _approval_lock, + retire_gateway_pending_mirror as _retire_gateway_pending_mirror, ) def _cleanup_gateway_pending_mirror(): - with _approval_lock: - head, total, _changed = _reconcile_gateway_pending_mirror_locked(session_id) - _approval_sse_notify_locked(session_id, head, total) + _retire_gateway_pending_mirror(session_id) except ImportError: _submit_pending_for_polling = None _cleanup_gateway_pending_mirror = None diff --git a/tests/test_approval_unblock.py b/tests/test_approval_unblock.py index 05579f13..4ca70897 100644 --- a/tests/test_approval_unblock.py +++ b/tests/test_approval_unblock.py @@ -638,11 +638,11 @@ class TestApprovalHTTPEndpoints: r._gateway_queues.pop(sid, None) _STREAM_RUN_IDS.pop(stream_id, None) - def test_gateway_no_run_mirror_stays_visible_when_exact_local_producer_is_gone(self, monkeypatch): - """A tokenized no-run mirror stays live when its parked producer already vanished.""" + def test_gateway_no_run_mirror_survives_pending_and_attention_reads_until_explicit_teardown(self, monkeypatch): + """A missing-producer 409 stays visible until a real teardown retires it.""" from api import routes as r from api import route_approvals as ra - from api.gateway_chat import _STREAM_RUN_IDS + from api.gateway_chat import _STREAM_RUN_IDS, _cleanup_gateway_pending_mirror sid = f"http-gateway-race-{uuid.uuid4().hex[:8]}" stream_id = f"stream-race-{uuid.uuid4().hex[:8]}" @@ -680,13 +680,76 @@ class TestApprovalHTTPEndpoints: "error": r._GATEWAY_APPROVAL_RELAY_UNAVAILABLE, } assert not entry.event.is_set() - with _lock: - pending_queue = r._pending.get(sid) - assert isinstance(pending_queue, list) - assert len(pending_queue) == 1 - assert pending_queue[0]["approval_id"] == approval_id + pending = r._handle_approval_pending( + object(), urllib.parse.urlparse( + f"/api/approval/pending?session_id={urllib.parse.quote(sid)}" + ) + ) + assert pending["pending"]["approval_id"] == approval_id + assert pending["pending_count"] == 1 + assert r._session_attention_summary(sid) == { + "kind": "approval", "count": 1, "severity": "critical" + } + _cleanup_gateway_pending_mirror(sid) + pending = r._handle_approval_pending( + object(), urllib.parse.urlparse( + f"/api/approval/pending?session_id={urllib.parse.quote(sid)}" + ) + ) + assert pending == {"pending": None, "pending_count": 0} + assert r._session_attention_summary(sid) is None finally: with _lock: r._pending.pop(sid, None) r._gateway_queues.pop(sid, None) _STREAM_RUN_IDS.pop(stream_id, None) + + def test_gateway_no_run_non_head_response_resolves_only_exact_producer(self, monkeypatch): + """An exact non-head response wakes only its matching producer.""" + from api import routes as r + from api import route_approvals as ra + + sid = f"http-gateway-non-head-{uuid.uuid4().hex[:8]}" + approval_a = {"command": "rm -rf /tmp/a", "description": "a"} + approval_b = {"command": "rm -rf /tmp/b", "description": "b"} + entry_a = _ApprovalEntry(approval_a) + entry_b = _ApprovalEntry(approval_b) + captured = {} + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(r, "j", fake_j) + monkeypatch.setattr(r, "get_session", lambda _sid: SimpleNamespace(active_stream_id=None)) + monkeypatch.setenv("HERMES_WEBUI_CHAT_BACKEND", "gateway") + with _lock: + r._pending.pop(sid, None) + r._gateway_queues[sid] = [entry_a, entry_b] + try: + ra.submit_gateway_pending_mirror(sid, approval_a) + ra.submit_gateway_pending_mirror(sid, approval_b) + with _lock: + approval_b_id = next( + item["approval_id"] for item in r._pending[sid] + if item["command"] == approval_b["command"] + ) + r._handle_approval_respond( + object(), {"session_id": sid, "choice": "once", "approval_id": approval_b_id} + ) + assert captured["status"] == 200 + assert entry_b.event.is_set() + assert entry_b.result == "once" + assert not entry_a.event.is_set() + pending = r._handle_approval_pending( + object(), urllib.parse.urlparse( + f"/api/approval/pending?session_id={urllib.parse.quote(sid)}" + ) + ) + assert pending["pending"]["command"] == approval_a["command"] + assert pending["pending_count"] == 1 + finally: + with _lock: + r._pending.pop(sid, None) + r._gateway_queues.pop(sid, None) diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index b988a9d4..58f8638a 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -432,13 +432,19 @@ def test_gateway_runs_api_streaming_parses_real_run_events(): def test_live_empty_ingress_id_stays_fifo_under_capability_v1(): """A normalized fallback browser ID must not become authoritative Agent identity.""" from api.config import STREAM_PARTIAL_TEXT, STREAM_REASONING_TEXT - from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming + from api.gateway_chat import _STREAM_RUN_IDS, _gateway_runs_approval_event, _run_gateway_runs_api_streaming from api import routes import api.route_approvals as approvals sid = "sid-live-empty-ingress" stream_id = "stream-live-empty-ingress" events = [] + normalized = _gateway_runs_approval_event({ + "command": "rm -rf /tmp/x", "description": "Dangerous", + "run_id": "run-empty-ingress", "approval_id": "", "id": "", + }) + assert normalized["approval_id"] + assert normalized["_gateway_raw_approval_id_present"] is False STREAM_PARTIAL_TEXT[stream_id] = "" STREAM_REASONING_TEXT[stream_id] = "" @@ -472,22 +478,10 @@ def test_live_empty_ingress_id_stays_fifo_under_capability_v1(): return _JsonResponse() return _SseResponse() - def fake_mapped_approval(payload): - assert payload["approval_id"] == "" - assert payload["id"] == "" - return { - "command": "rm -rf /tmp/x", - "description": "Dangerous", - "run_id": "run-empty-ingress", - "approval_id": "gwrun:normalized-fallback", - "_gateway_raw_approval_id_present": False, - } - handler = MagicMock() handler.wfile = io.BytesIO() try: with patch("urllib.request.urlopen", side_effect=fake_urlopen), \ - patch("api.gateway_chat._gateway_runs_approval_event", side_effect=fake_mapped_approval), \ patch("api.config.gateway_supports_approval_identity_v1", return_value=True): _run_gateway_runs_api_streaming( session_id=sid, msg_text="hi", model="test-model", workspace="/tmp", @@ -496,11 +490,12 @@ def test_live_empty_ingress_id_stays_fifo_under_capability_v1(): cancel_event=threading.Event(), ) mirror = approvals.gateway_pending_mirror(sid, run_id="run-empty-ingress") + browser_id = mirror["approval_id"] with patch("api.routes.get_session", return_value=SimpleNamespace(active_stream_id=stream_id)), \ patch("api.config.gateway_supports_approval_identity_v1", return_value=True), \ patch("api.runner_client.HttpRunnerClient.respond_approval") as respond: routes._handle_approval_respond(handler, { - "session_id": sid, "choice": "once", "approval_id": "gwrun:normalized-fallback", + "session_id": sid, "choice": "once", "approval_id": browser_id, }) respond.assert_called_once_with("run-empty-ingress", "", "once") finally: @@ -509,9 +504,11 @@ def test_live_empty_ingress_id_stays_fifo_under_capability_v1(): _STREAM_RUN_IDS.pop(stream_id, None) assert events[0][0] == "approval" - assert events[0][1]["approval_id"] == "gwrun:normalized-fallback" + assert events[0][1]["approval_id"] + assert events[0][1]["_gateway_raw_approval_id_present"] is False assert events[0][1]["_gateway_agent_identity_v1"] is False - assert mirror["approval_id"] == "gwrun:normalized-fallback" + assert mirror["approval_id"] == events[0][1]["approval_id"] + assert mirror["_gateway_raw_approval_id_present"] is False assert mirror["_gateway_agent_identity_v1"] is False approvals._pending.pop(sid, None) approvals._gateway_queues.pop(sid, None) From 2c6ce96596e543aa0f0d4c4168fe077b67ec4566 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Tue, 21 Jul 2026 17:55:50 -0400 Subject: [PATCH 14/20] fix(#6008): wait for gateway stop acknowledgement --- api/gateway_chat.py | 87 +++++++--- api/routes.py | 21 +-- tests/test_gateway_approval_runs_api.py | 221 ++++++++++++++++++++---- 3 files changed, 253 insertions(+), 76 deletions(-) diff --git a/api/gateway_chat.py b/api/gateway_chat.py index 1f51fbc3..ccf0b4e7 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -41,13 +41,65 @@ logger = logging.getLogger(__name__) # Maps stream_id -> gateway run_id for approval response relay. _STREAM_RUN_IDS: dict[str, str] = {} _STREAM_RUN_STARTING: set[str] = set() -_STREAM_RUN_STARTING_LOCK = threading.Lock() +_STREAM_RUN_START_RESULTS: dict[str, str] = {} +_STREAM_RUN_STARTING_CONDITION = threading.Condition() +GATEWAY_RUN_ID_WAIT_TIMEOUT = 5.0 + + +def _mark_gateway_run_starting(stream_id: str) -> None: + with _STREAM_RUN_STARTING_CONDITION: + _STREAM_RUN_STARTING.add(stream_id) + _STREAM_RUN_START_RESULTS.pop(stream_id, None) + + +def _publish_gateway_run_id(stream_id: str, run_id: str) -> None: + with _STREAM_RUN_STARTING_CONDITION: + _STREAM_RUN_IDS[stream_id] = run_id + _STREAM_RUN_STARTING.discard(stream_id) + _STREAM_RUN_START_RESULTS.pop(stream_id, None) + _STREAM_RUN_STARTING_CONDITION.notify_all() + + +def _finish_gateway_run_starting(stream_id: str, *, result: str = "failed") -> None: + with _STREAM_RUN_STARTING_CONDITION: + _STREAM_RUN_STARTING.discard(stream_id) + _STREAM_RUN_START_RESULTS[stream_id] = result + _STREAM_RUN_STARTING_CONDITION.notify_all() + + +def _clear_gateway_run_starting(stream_id: str) -> None: + with _STREAM_RUN_STARTING_CONDITION: + _STREAM_RUN_STARTING.discard(stream_id) + _STREAM_RUN_START_RESULTS.pop(stream_id, None) + _STREAM_RUN_STARTING_CONDITION.notify_all() def gateway_run_id_pending(stream_id: str) -> bool: - with _STREAM_RUN_STARTING_LOCK: + with _STREAM_RUN_STARTING_CONDITION: return stream_id in _STREAM_RUN_STARTING + +def wait_for_gateway_run_id(stream_id: str, timeout: float) -> tuple[bool, str | None]: + with _STREAM_RUN_STARTING_CONDITION: + run_id = str(_STREAM_RUN_IDS.get(stream_id) or "").strip() + if run_id: + return True, run_id + if stream_id not in _STREAM_RUN_STARTING: + result = str(_STREAM_RUN_START_RESULTS.get(stream_id) or "").strip() + return bool(result) and result != "fallback", None + _STREAM_RUN_STARTING_CONDITION.wait_for( + lambda: stream_id not in _STREAM_RUN_STARTING + or bool(str(_STREAM_RUN_IDS.get(stream_id) or "").strip()), + timeout=max(0.0, float(timeout)), + ) + run_id = str(_STREAM_RUN_IDS.get(stream_id) or "").strip() + if run_id: + return True, run_id + if stream_id in _STREAM_RUN_STARTING: + return True, None + result = str(_STREAM_RUN_START_RESULTS.get(stream_id) or "").strip() + return bool(result) and result != "fallback", None + _WEBUI_CHAT_BACKEND_ENV = "HERMES_WEBUI_CHAT_BACKEND" _WEBUI_GATEWAY_BASE_URL_ENV = "HERMES_WEBUI_GATEWAY_BASE_URL" _WEBUI_GATEWAY_API_KEY_ENV = "HERMES_WEBUI_GATEWAY_API_KEY" @@ -379,8 +431,7 @@ def _run_gateway_runs_api_streaming( attachments=None, cfg=None, session=None, ): """Submit via POST /v1/runs and relay SSE events including approval.""" - with _STREAM_RUN_STARTING_LOCK: - _STREAM_RUN_STARTING.add(stream_id) + _mark_gateway_run_starting(stream_id) try: url_runs = f"{base_url.rstrip('/')}/v1/runs" headers = { @@ -456,25 +507,11 @@ def _run_gateway_runs_api_streaming( if not run_id: raise ValueError(f"Gateway runs API returned no run_id: {run_data!r}") except Exception: - with _STREAM_RUN_STARTING_LOCK: - _STREAM_RUN_STARTING.discard(stream_id) + _finish_gateway_run_starting(stream_id) raise usage: dict = {} - _STREAM_RUN_IDS[stream_id] = run_id - with _STREAM_RUN_STARTING_LOCK: - _STREAM_RUN_STARTING.discard(stream_id) - if cancel_event.is_set(): - stop_confirmed = False - try: - stop_confirmed = stop_gateway_run(stream_id) - except Exception: - logger.debug("Failed to stop gateway run after pre-stream cancellation", exc_info=True) - if stop_confirmed: - put_gateway_event("cancel", {"message": "Cancelled by user"}) - return None, usage - put_gateway_event("apperror", {"message": "Gateway stop failed"}) - return None, usage + _publish_gateway_run_id(stream_id, run_id) url_events = f"{base_url.rstrip('/')}/v1/runs/{run_id}/events" headers_sse = dict(headers) @@ -592,9 +629,9 @@ def _run_gateway_runs_api_streaming( return final_text, usage -def stop_gateway_run(stream_id: str) -> bool: +def stop_gateway_run(run_id: str) -> bool: """Request gateway interruption and report whether it was acknowledged.""" - run_id = str(_STREAM_RUN_IDS.get(stream_id) or "").strip() + run_id = str(run_id or "").strip() if not run_id: return False from api.config import get_config @@ -816,13 +853,11 @@ def _run_gateway_chat_streaming( _gw_overrides = {} _runs_api_enabled = _gateway_use_runs_api_enabled(cfg) if _runs_api_enabled: - with _STREAM_RUN_STARTING_LOCK: - _STREAM_RUN_STARTING.add(stream_id) + _mark_gateway_run_starting(stream_id) runs_api_pending_marked = True _use_runs_api = _runs_api_enabled and gateway_supports_approval(base_url, api_key) if not _use_runs_api and runs_api_pending_marked: - with _STREAM_RUN_STARTING_LOCK: - _STREAM_RUN_STARTING.discard(stream_id) + _finish_gateway_run_starting(stream_id, result="fallback") runs_api_pending_marked = False try: from api.streaming import ( diff --git a/api/routes.py b/api/routes.py index 82f241f5..7707cc69 100644 --- a/api/routes.py +++ b/api/routes.py @@ -13374,20 +13374,17 @@ def handle_get(handler, parsed) -> bool: return True gateway_stop_blocked = False try: - from api.gateway_chat import _STREAM_RUN_IDS, gateway_run_id_pending, stop_gateway_run + from api.gateway_chat import ( + GATEWAY_RUN_ID_WAIT_TIMEOUT, + stop_gateway_run, + wait_for_gateway_run_id, + ) - run_id = _STREAM_RUN_IDS.get(stream_id) - pending_run_id = False - if not run_id: - pending_run_id = gateway_run_id_pending(stream_id) - if pending_run_id: - cancel_flag = CANCEL_FLAGS.get(stream_id) - if cancel_flag: - cancel_flag.set() - cancelled = cancel_stream(stream_id) - return j(handler, {"ok": True, "cancelled": cancelled, "deferred": True, "stream_id": stream_id}) + structured_gateway, run_id = wait_for_gateway_run_id(stream_id, GATEWAY_RUN_ID_WAIT_TIMEOUT) + if not run_id and structured_gateway: + gateway_stop_blocked = True if run_id: - if stop_gateway_run(stream_id): + if stop_gateway_run(run_id): owner_sid = stream_owner_session_id(stream_id) if owner_sid: retire_gateway_pending_mirror(owner_sid, run_id=run_id) diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index 58f8638a..1cfe950e 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -1317,7 +1317,7 @@ def test_local_stop_uses_runs_api_stop_endpoint(): with patch("urllib.request.build_opener", side_effect=fake_build_opener), \ patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ patch("api.gateway_chat._gateway_api_key", return_value="secret"): - assert stop_gateway_run(stream_id) is True + assert stop_gateway_run("run-stop") is True finally: _STREAM_RUN_IDS.pop(stream_id, None) @@ -1328,7 +1328,7 @@ def test_local_stop_uses_runs_api_stop_endpoint(): assert timeout == 10 -def test_runs_api_cancel_after_run_creation_stops_remote_run_before_events(): +def test_runs_api_publishes_run_id_before_events(): from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming stream_id = "sid-run-create-cancel" @@ -1348,16 +1348,32 @@ def test_runs_api_cancel_after_run_creation_stops_remote_run_before_events(): def __exit__(self, *args): return None + class _SseResponse: + def __init__(self, lines): + self._lines = lines + + def __iter__(self): + return iter(self._lines) + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + def fake_urlopen(req, *, timeout=None): requests.append(req.full_url) if req.full_url.endswith("/v1/runs"): - cancel_event.set() return _JsonResponse({"run_id": "run-pre-events-cancel"}) + if req.full_url.endswith("/events"): + return _SseResponse([ + b'data: {"event":"run.completed","output":"Hello"}\n', + b'\n', + ]) raise AssertionError(f"unexpected urlopen after pre-stream cancel: {req.full_url}") try: - with patch("urllib.request.urlopen", side_effect=fake_urlopen), \ - patch("api.gateway_chat.stop_gateway_run", return_value=True) as stop_run: + with patch("urllib.request.urlopen", side_effect=fake_urlopen): final_text, usage = _run_gateway_runs_api_streaming( session_id="sess-run-create-cancel", msg_text="hi", @@ -1372,15 +1388,14 @@ def test_runs_api_cancel_after_run_creation_stops_remote_run_before_events(): cancel_event=cancel_event, cfg={}, ) - assert final_text is None + assert final_text == "Hello" assert usage == {} - stop_run.assert_called_once_with(stream_id) - assert requests == ["http://gw:8642/v1/runs"] + assert requests == ["http://gw:8642/v1/runs", "http://gw:8642/v1/runs/run-pre-events-cancel/events"] finally: _STREAM_RUN_IDS.pop(stream_id, None) -def test_runs_api_cancel_after_run_creation_reports_stop_failure(): +def test_runs_api_does_not_own_pre_stream_stop_after_publication(): from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming stream_id = "sid-run-create-cancel-stop-failed" @@ -1417,7 +1432,6 @@ def test_runs_api_cancel_after_run_creation_reports_stop_failure(): def fake_urlopen(req, *, timeout=None): requests.append(req.full_url) if req.full_url.endswith("/v1/runs"): - cancel_event.set() return _JsonResponse({"run_id": "run-pre-events-stop-failed"}) if req.full_url.endswith("/events"): return _SseResponse([ @@ -1429,8 +1443,7 @@ def test_runs_api_cancel_after_run_creation_reports_stop_failure(): raise AssertionError(f"unexpected urlopen: {req.full_url}") try: - with patch("urllib.request.urlopen", side_effect=fake_urlopen), \ - patch("api.gateway_chat.stop_gateway_run", return_value=False) as stop_run: + with patch("urllib.request.urlopen", side_effect=fake_urlopen): final_text, usage = _run_gateway_runs_api_streaming( session_id="sess-run-create-cancel-stop-failed", msg_text="hi", @@ -1445,11 +1458,10 @@ def test_runs_api_cancel_after_run_creation_reports_stop_failure(): cancel_event=cancel_event, cfg={}, ) - assert final_text is None + assert final_text == "Hello" assert usage == {} - stop_run.assert_called_once_with(stream_id) - assert requests == ["http://gw:8642/v1/runs"] - assert [event for event, _data in events] == ["apperror"] + assert requests == ["http://gw:8642/v1/runs", "http://gw:8642/v1/runs/run-pre-events-stop-failed/events"] + assert [event for event, _data in events] == ["token"] finally: _STREAM_RUN_IDS.pop(stream_id, None) @@ -1527,10 +1539,12 @@ def test_runs_api_marks_run_pending_before_request_preparation(): def test_gateway_worker_marks_run_pending_before_runs_api_prelude(): + from api import gateway_chat from api.gateway_chat import STREAMS, _run_gateway_chat_streaming, gateway_run_id_pending stream_id = "sid-worker-run-starting" observed = {"pending_during_support_check": False, "pending_during_runs_call": False} + publish_run_id = getattr(gateway_chat, "_publish_gateway_run_id", None) STREAMS[stream_id] = SimpleNamespace(put_nowait=lambda *_args, **_kwargs: None) session = SimpleNamespace( profile=None, @@ -1549,6 +1563,8 @@ def test_gateway_worker_marks_run_pending_before_runs_api_prelude(): def fake_runs(*_args, **_kwargs): observed["pending_during_runs_call"] = gateway_run_id_pending(stream_id) + if publish_run_id: + publish_run_id(stream_id, "run-worker-cleanup") return None, {} with patch("api.gateway_chat.RunJournalWriter", return_value=SimpleNamespace(append_sse_event=lambda *_a, **_k: None)), \ @@ -1576,6 +1592,7 @@ def test_gateway_worker_marks_run_pending_before_runs_api_prelude(): assert observed["pending_during_support_check"] is True assert observed["pending_during_runs_call"] is True assert gateway_run_id_pending(stream_id) is False + assert getattr(gateway_chat, "_STREAM_RUN_START_RESULTS", {}).get(stream_id) is None def test_local_stop_rejects_redirected_success(): @@ -1604,7 +1621,7 @@ def test_local_stop_rejects_redirected_success(): with patch("urllib.request.build_opener", return_value=_Opener()), \ patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ patch("api.gateway_chat._gateway_api_key", return_value=None): - assert stop_gateway_run(stream_id) is False + assert stop_gateway_run("run-stop") is False finally: _STREAM_RUN_IDS.pop(stream_id, None) @@ -1735,20 +1752,18 @@ def test_chat_cancel_surfaces_gateway_stop_failure(monkeypatch): approvals._pending.pop(sid, None) -def test_chat_cancel_defers_while_gateway_run_id_is_pending(monkeypatch): +def test_chat_cancel_without_gateway_readiness_uses_local_cancel(monkeypatch): import urllib.parse from api import routes - stream_id = "stream-cancel-pending-run-id" + stream_id = "stream-cancel-local-only" captured = {} called = {"cancel": False, "stop": False} monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) - monkeypatch.setattr("api.gateway_chat.gateway_run_id_pending", lambda _stream_id: True) - monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _stream_id: called.__setitem__("stop", True)) - monkeypatch.setattr("time.sleep", lambda _seconds: None) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _run_id: called.__setitem__("stop", True)) def fake_j(handler, data, status=200, extra_headers=None): captured["payload"] = data @@ -1757,37 +1772,34 @@ def test_chat_cancel_defers_while_gateway_run_id_is_pending(monkeypatch): monkeypatch.setattr(routes, "j", fake_j) parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") + routes.handle_get(object(), parsed) + assert captured["status"] == 200 assert captured["payload"] == { "ok": True, "cancelled": True, - "deferred": True, "stream_id": stream_id, } assert called["cancel"] is True assert called["stop"] is False -def test_chat_cancel_defers_stop_until_gateway_run_id_exists(monkeypatch): +def test_chat_cancel_times_out_while_gateway_run_id_is_pending(monkeypatch): import urllib.parse from api import routes - from api.gateway_chat import _STREAM_RUN_IDS + from api import gateway_chat - stream_id = "stream-cancel-pending-wait" + stream_id = "stream-cancel-pending-run-id" captured = {} - called = {"cancel": False, "stop": False, "pending": True} - - def fake_sleep(_seconds): - _STREAM_RUN_IDS[stream_id] = "run-stop" - called["pending"] = False + called = {"cancel": False, "stop": False} + mark_starting = getattr(gateway_chat, "_mark_gateway_run_starting", None) monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) - monkeypatch.setattr("api.gateway_chat.gateway_run_id_pending", lambda _stream_id: called["pending"]) - monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _stream_id: called.__setitem__("stop", True) or True) - monkeypatch.setattr("time.sleep", fake_sleep) + monkeypatch.setattr("api.gateway_chat.GATEWAY_RUN_ID_WAIT_TIMEOUT", 0.01, raising=False) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _run_id: called.__setitem__("stop", True)) def fake_j(handler, data, status=200, extra_headers=None): captured["payload"] = data @@ -1797,13 +1809,146 @@ def test_chat_cancel_defers_stop_until_gateway_run_id_exists(monkeypatch): monkeypatch.setattr(routes, "j", fake_j) parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") try: + if mark_starting: + mark_starting(stream_id) + else: + gateway_chat._STREAM_RUN_STARTING.add(stream_id) routes.handle_get(object(), parsed) - assert captured["status"] == 200 - assert captured["payload"] == {"ok": True, "cancelled": True, "deferred": True, "stream_id": stream_id} + assert captured["status"] == 502 + assert captured["payload"] == { + "ok": False, + "cancelled": False, + "stream_id": stream_id, + "error": "Gateway stop failed", + } + assert called["cancel"] is False assert called["stop"] is False - assert called["cancel"] is True finally: - _STREAM_RUN_IDS.pop(stream_id, None) + gateway_chat._STREAM_RUN_IDS.pop(stream_id, None) + gateway_chat._STREAM_RUN_STARTING.discard(stream_id) + getattr(gateway_chat, "_STREAM_RUN_START_RESULTS", {}).pop(stream_id, None) + + +def test_chat_cancel_waits_for_exact_run_id_before_failed_stop(monkeypatch): + import time + import urllib.parse + + from api import routes + from api import gateway_chat + + stream_id = "stream-cancel-pending-wait" + captured = {} + called = {"cancel": False, "stop": False} + mark_starting = getattr(gateway_chat, "_mark_gateway_run_starting", None) + publish_run_id = getattr(gateway_chat, "_publish_gateway_run_id", None) + + monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda run_id: called.__setitem__("stop", run_id) or False) + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(routes, "j", fake_j) + parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") + request_thread = threading.Thread(target=routes.handle_get, args=(object(), parsed)) + try: + if mark_starting: + mark_starting(stream_id) + else: + gateway_chat._STREAM_RUN_STARTING.add(stream_id) + request_thread.start() + time.sleep(0.05) + assert request_thread.is_alive() + assert called["cancel"] is False + if publish_run_id: + publish_run_id(stream_id, "run-stop/1") + else: + gateway_chat._STREAM_RUN_IDS[stream_id] = "run-stop/1" + gateway_chat._STREAM_RUN_STARTING.discard(stream_id) + request_thread.join(timeout=5) + assert not request_thread.is_alive() + assert captured["status"] == 502 + assert captured["payload"] == { + "ok": False, + "cancelled": False, + "stream_id": stream_id, + "error": "Gateway stop failed", + } + assert called["stop"] == "run-stop/1" + assert called["cancel"] is False + finally: + if request_thread.is_alive(): + if publish_run_id: + publish_run_id(stream_id, "run-stop/1") + else: + gateway_chat._STREAM_RUN_IDS[stream_id] = "run-stop/1" + gateway_chat._STREAM_RUN_STARTING.discard(stream_id) + request_thread.join(timeout=5) + gateway_chat._STREAM_RUN_IDS.pop(stream_id, None) + gateway_chat._STREAM_RUN_STARTING.discard(stream_id) + getattr(gateway_chat, "_STREAM_RUN_START_RESULTS", {}).pop(stream_id, None) + + +def test_chat_cancel_uses_local_cancel_after_gateway_runs_api_falls_back(monkeypatch): + import time + import urllib.parse + + from api import routes + from api import gateway_chat + + stream_id = "stream-cancel-runs-fallback" + captured = {} + called = {"cancel": False, "stop": False} + mark_starting = getattr(gateway_chat, "_mark_gateway_run_starting", None) + finish_starting = getattr(gateway_chat, "_finish_gateway_run_starting", None) + + monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _run_id: called.__setitem__("stop", True)) + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(routes, "j", fake_j) + parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") + request_thread = threading.Thread(target=routes.handle_get, args=(object(), parsed)) + try: + if mark_starting: + mark_starting(stream_id) + else: + gateway_chat._STREAM_RUN_STARTING.add(stream_id) + request_thread.start() + time.sleep(0.05) + assert request_thread.is_alive() + if finish_starting: + finish_starting(stream_id, result="fallback") + else: + gateway_chat._STREAM_RUN_STARTING.discard(stream_id) + request_thread.join(timeout=5) + assert not request_thread.is_alive() + assert captured["status"] == 200 + assert captured["payload"] == { + "ok": True, + "cancelled": True, + "stream_id": stream_id, + } + assert called["cancel"] is True + assert called["stop"] is False + finally: + if request_thread.is_alive(): + if finish_starting: + finish_starting(stream_id, result="fallback") + else: + gateway_chat._STREAM_RUN_STARTING.discard(stream_id) + request_thread.join(timeout=5) + gateway_chat._STREAM_RUN_IDS.pop(stream_id, None) + gateway_chat._STREAM_RUN_STARTING.discard(stream_id) + getattr(gateway_chat, "_STREAM_RUN_START_RESULTS", {}).pop(stream_id, None) def test_stale_gateway_card_does_not_relay_live_run(): From 5a954c79d356f39ff96166b01e96067ec972a80c Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Tue, 21 Jul 2026 19:42:57 -0400 Subject: [PATCH 15/20] fix(#6008): keep gateway cancel authority stable across startup teardown --- api/gateway_chat.py | 112 +++-- api/routes.py | 16 +- tests/test_gateway_approval_runs_api.py | 570 +++++++++++++++++++----- 3 files changed, 550 insertions(+), 148 deletions(-) diff --git a/api/gateway_chat.py b/api/gateway_chat.py index ccf0b4e7..00183dd1 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -40,65 +40,110 @@ logger = logging.getLogger(__name__) # Maps stream_id -> gateway run_id for approval response relay. _STREAM_RUN_IDS: dict[str, str] = {} -_STREAM_RUN_STARTING: set[str] = set() -_STREAM_RUN_START_RESULTS: dict[str, str] = {} +_STREAM_RUN_LIFECYCLE: dict[str, dict[str, Any]] = {} _STREAM_RUN_STARTING_CONDITION = threading.Condition() GATEWAY_RUN_ID_WAIT_TIMEOUT = 5.0 def _mark_gateway_run_starting(stream_id: str) -> None: with _STREAM_RUN_STARTING_CONDITION: - _STREAM_RUN_STARTING.add(stream_id) - _STREAM_RUN_START_RESULTS.pop(stream_id, None) + _STREAM_RUN_IDS.pop(stream_id, None) + _STREAM_RUN_LIFECYCLE[stream_id] = { + "phase": "pending", + "run_id": "", + "waiters": 0, + "owner_done": False, + } def _publish_gateway_run_id(stream_id: str, run_id: str) -> None: with _STREAM_RUN_STARTING_CONDITION: _STREAM_RUN_IDS[stream_id] = run_id - _STREAM_RUN_STARTING.discard(stream_id) - _STREAM_RUN_START_RESULTS.pop(stream_id, None) + state = _STREAM_RUN_LIFECYCLE.get(stream_id) or {} + _STREAM_RUN_LIFECYCLE[stream_id] = { + "phase": "ready", + "run_id": run_id, + "waiters": int(state.get("waiters") or 0), + "owner_done": bool(state.get("owner_done")), + } _STREAM_RUN_STARTING_CONDITION.notify_all() def _finish_gateway_run_starting(stream_id: str, *, result: str = "failed") -> None: with _STREAM_RUN_STARTING_CONDITION: - _STREAM_RUN_STARTING.discard(stream_id) - _STREAM_RUN_START_RESULTS[stream_id] = result + state = _STREAM_RUN_LIFECYCLE.get(stream_id) or {} + if str(state.get("phase") or "").strip().lower() == "ready": + return + _STREAM_RUN_IDS.pop(stream_id, None) + _STREAM_RUN_LIFECYCLE[stream_id] = { + "phase": "fallback" if result == "fallback" else "failed", + "run_id": "", + "waiters": int(state.get("waiters") or 0), + "owner_done": bool(state.get("owner_done")), + } _STREAM_RUN_STARTING_CONDITION.notify_all() +def _retire_gateway_run_starting_if_done(stream_id: str) -> bool: + state = _STREAM_RUN_LIFECYCLE.get(stream_id) + if not state: + return False + if int(state.get("waiters") or 0) > 0: + return False + if not bool(state.get("owner_done")): + return False + _STREAM_RUN_LIFECYCLE.pop(stream_id, None) + _STREAM_RUN_IDS.pop(stream_id, None) + return True + + def _clear_gateway_run_starting(stream_id: str) -> None: with _STREAM_RUN_STARTING_CONDITION: - _STREAM_RUN_STARTING.discard(stream_id) - _STREAM_RUN_START_RESULTS.pop(stream_id, None) + state = _STREAM_RUN_LIFECYCLE.get(stream_id) + if state: + state["owner_done"] = True + _retire_gateway_run_starting_if_done(stream_id) _STREAM_RUN_STARTING_CONDITION.notify_all() def gateway_run_id_pending(stream_id: str) -> bool: with _STREAM_RUN_STARTING_CONDITION: - return stream_id in _STREAM_RUN_STARTING + return str((_STREAM_RUN_LIFECYCLE.get(stream_id) or {}).get("phase") or "").strip().lower() == "pending" def wait_for_gateway_run_id(stream_id: str, timeout: float) -> tuple[bool, str | None]: + deadline = time.monotonic() + max(0.0, float(timeout)) with _STREAM_RUN_STARTING_CONDITION: - run_id = str(_STREAM_RUN_IDS.get(stream_id) or "").strip() - if run_id: - return True, run_id - if stream_id not in _STREAM_RUN_STARTING: - result = str(_STREAM_RUN_START_RESULTS.get(stream_id) or "").strip() - return bool(result) and result != "fallback", None - _STREAM_RUN_STARTING_CONDITION.wait_for( - lambda: stream_id not in _STREAM_RUN_STARTING - or bool(str(_STREAM_RUN_IDS.get(stream_id) or "").strip()), - timeout=max(0.0, float(timeout)), - ) - run_id = str(_STREAM_RUN_IDS.get(stream_id) or "").strip() - if run_id: - return True, run_id - if stream_id in _STREAM_RUN_STARTING: - return True, None - result = str(_STREAM_RUN_START_RESULTS.get(stream_id) or "").strip() - return bool(result) and result != "fallback", None + state = _STREAM_RUN_LIFECYCLE.get(stream_id) + if state: + state["waiters"] = int(state.get("waiters") or 0) + 1 + try: + while True: + state = _STREAM_RUN_LIFECYCLE.get(stream_id) + phase = str((state or {}).get("phase") or "").strip().lower() + if phase == "fallback": + return False, None + if phase == "failed": + return True, None + run_id = str(_STREAM_RUN_IDS.get(stream_id) or "").strip() + if phase == "ready": + stored_run_id = str((state or {}).get("run_id") or "").strip() + return True, run_id or stored_run_id or None + if run_id: + return True, run_id + if not state: + return False, None + remaining = deadline - time.monotonic() + if remaining <= 0: + return True, None + _STREAM_RUN_STARTING_CONDITION.wait(timeout=remaining) + finally: + state = _STREAM_RUN_LIFECYCLE.get(stream_id) + if state: + waiters = max(0, int(state.get("waiters") or 0) - 1) + state["waiters"] = waiters + if _retire_gateway_run_starting_if_done(stream_id): + _STREAM_RUN_STARTING_CONDITION.notify_all() _WEBUI_CHAT_BACKEND_ENV = "HERMES_WEBUI_CHAT_BACKEND" _WEBUI_GATEWAY_BASE_URL_ENV = "HERMES_WEBUI_GATEWAY_BASE_URL" @@ -431,7 +476,6 @@ def _run_gateway_runs_api_streaming( attachments=None, cfg=None, session=None, ): """Submit via POST /v1/runs and relay SSE events including approval.""" - _mark_gateway_run_starting(stream_id) try: url_runs = f"{base_url.rstrip('/')}/v1/runs" headers = { @@ -771,6 +815,8 @@ def _run_gateway_chat_streaming( """ q = STREAMS.get(stream_id) if q is None: + _finish_gateway_run_starting(stream_id, result="fallback") + _clear_gateway_run_starting(stream_id) # Cancelled before the worker started; release the owner entry the route # layer registered so STREAM_SESSION_OWNERS does not leak (no teardown finally runs). unregister_stream_owner(stream_id) @@ -798,7 +844,7 @@ def _run_gateway_chat_streaming( STREAM_LIVE_TOOL_CALLS[stream_id] = [] success_writeback_committed = False - runs_api_pending_marked = False + runs_api_pending_marked = True def put_gateway_event(event, data): if cancel_event.is_set() and not success_writeback_committed and event not in ("cancel", "error", "apperror"): @@ -852,9 +898,6 @@ def _run_gateway_chat_streaming( except Exception: _gw_overrides = {} _runs_api_enabled = _gateway_use_runs_api_enabled(cfg) - if _runs_api_enabled: - _mark_gateway_run_starting(stream_id) - runs_api_pending_marked = True _use_runs_api = _runs_api_enabled and gateway_supports_approval(base_url, api_key) if not _use_runs_api and runs_api_pending_marked: _finish_gateway_run_starting(stream_id, result="fallback") @@ -1318,7 +1361,6 @@ def _run_gateway_chat_streaming( STREAM_LIVE_TOOL_CALLS.pop(stream_id, None) STREAM_LAST_EVENT_ID.pop(stream_id, None) STREAMS.pop(stream_id, None) - _STREAM_RUN_IDS.pop(stream_id, None) if runs_api_pending_marked and gateway_run_id_pending(stream_id): _finish_gateway_run_starting(stream_id) _clear_gateway_run_starting(stream_id) diff --git a/api/routes.py b/api/routes.py index 7707cc69..56f317bf 100644 --- a/api/routes.py +++ b/api/routes.py @@ -21244,13 +21244,27 @@ def _start_chat_stream_for_session( worker_kwargs = {"model_provider": model_provider, "goal_related": goal_related} if moa_config and not backend_is_gateway: worker_kwargs["moa_config"] = moa_config + if backend_is_gateway: + from api.gateway_chat import _mark_gateway_run_starting + _mark_gateway_run_starting(stream_id) thr = threading.Thread( target=worker_target, args=(s.session_id, msg, model, workspace, stream_id, attachments), kwargs=worker_kwargs, daemon=True, ) - thr.start() + try: + thr.start() + except Exception: + if backend_is_gateway: + try: + from api.gateway_chat import _finish_gateway_run_starting + _finish_gateway_run_starting(stream_id) + from api.gateway_chat import _clear_gateway_run_starting + _clear_gateway_run_starting(stream_id) + except Exception: + logger.debug("Failed to record gateway run-start failure for stream %s", stream_id, exc_info=True) + raise response = { "stream_id": stream_id, "session_id": s.session_id, diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index 1cfe950e..c1ae7341 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -12,6 +12,15 @@ from unittest.mock import MagicMock, patch import pytest +def _reset_gateway_run_start_state(stream_id: str) -> None: + from api import gateway_chat + + gateway_chat._STREAM_RUN_IDS.pop(stream_id, None) + lifecycle = getattr(gateway_chat, "_STREAM_RUN_LIFECYCLE", None) + if isinstance(lifecycle, dict): + lifecycle.pop(stream_id, None) + + # --------------------------------------------------------------------------- # 1. Capability detection # --------------------------------------------------------------------------- @@ -1467,7 +1476,11 @@ def test_runs_api_does_not_own_pre_stream_stop_after_publication(): def test_runs_api_marks_run_pending_before_request_preparation(): - from api.gateway_chat import _STREAM_RUN_IDS, _run_gateway_runs_api_streaming, gateway_run_id_pending + from api.gateway_chat import ( + _mark_gateway_run_starting, + _run_gateway_runs_api_streaming, + gateway_run_id_pending, + ) stream_id = "sid-run-starting-prep" cancel_event = threading.Event() @@ -1512,6 +1525,7 @@ def test_runs_api_marks_run_pending_before_request_preparation(): raise AssertionError(f"unexpected urlopen: {req.full_url}") try: + _mark_gateway_run_starting(stream_id) with patch("urllib.request.urlopen", side_effect=fake_urlopen), \ patch("api.streaming._strip_oob_blocks", side_effect=fake_strip): final_text, usage = _run_gateway_runs_api_streaming( @@ -1535,12 +1549,17 @@ def test_runs_api_marks_run_pending_before_request_preparation(): assert observed["pending_during_post"] is True assert gateway_run_id_pending(stream_id) is False finally: - _STREAM_RUN_IDS.pop(stream_id, None) + _reset_gateway_run_start_state(stream_id) def test_gateway_worker_marks_run_pending_before_runs_api_prelude(): from api import gateway_chat - from api.gateway_chat import STREAMS, _run_gateway_chat_streaming, gateway_run_id_pending + from api.gateway_chat import ( + STREAMS, + _mark_gateway_run_starting, + _run_gateway_chat_streaming, + gateway_run_id_pending, + ) stream_id = "sid-worker-run-starting" observed = {"pending_during_support_check": False, "pending_during_runs_call": False} @@ -1567,32 +1586,388 @@ def test_gateway_worker_marks_run_pending_before_runs_api_prelude(): publish_run_id(stream_id, "run-worker-cleanup") return None, {} - with patch("api.gateway_chat.RunJournalWriter", return_value=SimpleNamespace(append_sse_event=lambda *_a, **_k: None)), \ - patch("api.gateway_chat.get_session", return_value=session), \ - patch("api.config.get_config", return_value={}), \ - patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ - patch("api.gateway_chat._gateway_api_key", return_value=""), \ - patch("api.gateway_chat._gateway_use_runs_api_enabled", return_value=True), \ - patch("api.gateway_chat.gateway_supports_approval", side_effect=fake_supports), \ - patch("api.gateway_chat._gateway_reasoning_effort_for_request", return_value=None), \ - patch("api.gateway_chat._run_gateway_runs_api_streaming", side_effect=fake_runs), \ - patch("api.streaming._load_webui_prefill_context", return_value={}), \ - patch("api.streaming._prefill_messages_with_webui_context", return_value=[]), \ - patch("api.streaming._normalize_prefill_messages_before_user_turn", side_effect=lambda messages: messages), \ - patch("api.streaming._public_prefill_context_status", return_value={}), \ - patch("api.streaming._webui_ephemeral_system_prompt", return_value="sys"): - _run_gateway_chat_streaming( - session_id="sess-worker-run-starting", + try: + _mark_gateway_run_starting(stream_id) + with patch("api.gateway_chat.RunJournalWriter", return_value=SimpleNamespace(append_sse_event=lambda *_a, **_k: None)), \ + patch("api.gateway_chat.get_session", return_value=session), \ + patch("api.config.get_config", return_value={}), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.gateway_chat._gateway_use_runs_api_enabled", return_value=True), \ + patch("api.gateway_chat.gateway_supports_approval", side_effect=fake_supports), \ + patch("api.gateway_chat._gateway_reasoning_effort_for_request", return_value=None), \ + patch("api.gateway_chat._run_gateway_runs_api_streaming", side_effect=fake_runs), \ + patch("api.streaming._load_webui_prefill_context", return_value={}), \ + patch("api.streaming._prefill_messages_with_webui_context", return_value=[]), \ + patch("api.streaming._normalize_prefill_messages_before_user_turn", side_effect=lambda messages: messages), \ + patch("api.streaming._public_prefill_context_status", return_value={}), \ + patch("api.streaming._webui_ephemeral_system_prompt", return_value="sys"): + _run_gateway_chat_streaming( + session_id="sess-worker-run-starting", + msg_text="hi", + model="test-model", + workspace="/tmp", + stream_id=stream_id, + ) + finally: + _reset_gateway_run_start_state(stream_id) + + assert observed["pending_during_support_check"] is True + assert observed["pending_during_runs_call"] is True + assert gateway_run_id_pending(stream_id) is False + assert stream_id not in getattr(gateway_chat, "_STREAM_RUN_LIFECYCLE", {}) + + +def test_start_chat_stream_marks_gateway_run_pending_before_thread_start(monkeypatch): + from api import gateway_chat, routes + + recorded = {} + session = SimpleNamespace( + session_id="sess-route-start-pending", + active_stream_id=None, + pending_started_at=None, + title="title", + profile=None, + process_wakeup_pause={}, + ) + + class _NoopLock: + def __enter__(self): + return None + + def __exit__(self, *args): + return False + + class _FakeThread: + def __init__(self, *, target=None, args=None, kwargs=None, daemon=None): + self._target = target + self._args = args or () + self._kwargs = kwargs or {} + + def start(self): + stream_id = recorded["stream_id"] + assert gateway_chat.gateway_run_id_pending(stream_id) is True + gateway_chat._finish_gateway_run_starting(stream_id, result="fallback") + gateway_chat._clear_gateway_run_starting(stream_id) + + def fake_prepare(session_obj, *, stream_id, **_kwargs): + recorded["stream_id"] = stream_id + session_obj.active_stream_id = stream_id + session_obj.pending_started_at = 123.0 + + monkeypatch.setattr(routes, "_agent_runtime_barrier_response", lambda **_kwargs: None) + monkeypatch.setattr(routes, "_active_run_stream_for_session", lambda *_args, **_kwargs: None) + monkeypatch.setattr(routes, "_get_session_agent_lock", lambda *_args, **_kwargs: _NoopLock()) + monkeypatch.setattr(routes, "_is_hidden_empty_session", lambda *_args, **_kwargs: False) + monkeypatch.setattr(routes, "_prepare_chat_start_session_for_stream", fake_prepare) + monkeypatch.setattr(routes, "create_stream_channel", lambda: SimpleNamespace()) + monkeypatch.setattr(routes, "register_stream_owner", lambda *_args, **_kwargs: None) + monkeypatch.setattr(routes, "set_last_workspace", lambda *_args, **_kwargs: None) + monkeypatch.setattr(routes, "threading", SimpleNamespace(Thread=_FakeThread)) + + with patch("api.turn_journal.append_turn_journal_event", return_value={}): + response = routes._start_chat_stream_for_session( + session, + msg="hi", + attachments=[], + workspace="/tmp", + model="test-model", + external_runtime_owned=True, + ) + + assert response["stream_id"] == recorded["stream_id"] + assert gateway_chat.gateway_run_id_pending(recorded["stream_id"]) is False + + +def test_start_chat_stream_clears_gateway_run_state_when_thread_start_fails(monkeypatch): + from api import gateway_chat, routes + + recorded = {} + session = SimpleNamespace( + session_id="sess-route-start-fail", + active_stream_id=None, + pending_started_at=None, + title="title", + profile=None, + process_wakeup_pause={}, + ) + + class _NoopLock: + def __enter__(self): + return None + + def __exit__(self, *args): + return False + + class _BoomThread: + def __init__(self, *, target=None, args=None, kwargs=None, daemon=None): + self._target = target + self._args = args or () + self._kwargs = kwargs or {} + + def start(self): + stream_id = recorded["stream_id"] + assert gateway_chat.gateway_run_id_pending(stream_id) is True + raise RuntimeError("thread start failed") + + def fake_prepare(session_obj, *, stream_id, **_kwargs): + recorded["stream_id"] = stream_id + session_obj.active_stream_id = stream_id + session_obj.pending_started_at = 123.0 + + monkeypatch.setattr(routes, "_agent_runtime_barrier_response", lambda **_kwargs: None) + monkeypatch.setattr(routes, "_active_run_stream_for_session", lambda *_args, **_kwargs: None) + monkeypatch.setattr(routes, "_get_session_agent_lock", lambda *_args, **_kwargs: _NoopLock()) + monkeypatch.setattr(routes, "_is_hidden_empty_session", lambda *_args, **_kwargs: False) + monkeypatch.setattr(routes, "_prepare_chat_start_session_for_stream", fake_prepare) + monkeypatch.setattr(routes, "create_stream_channel", lambda: SimpleNamespace()) + monkeypatch.setattr(routes, "register_stream_owner", lambda *_args, **_kwargs: None) + monkeypatch.setattr(routes, "set_last_workspace", lambda *_args, **_kwargs: None) + monkeypatch.setattr(routes, "threading", SimpleNamespace(Thread=_BoomThread)) + + with patch("api.turn_journal.append_turn_journal_event", return_value={}): + with pytest.raises(RuntimeError, match="thread start failed"): + routes._start_chat_stream_for_session( + session, + msg="hi", + attachments=[], + workspace="/tmp", + model="test-model", + external_runtime_owned=True, + ) + + assert gateway_chat.gateway_run_id_pending(recorded["stream_id"]) is False + assert recorded["stream_id"] not in getattr(gateway_chat, "_STREAM_RUN_LIFECYCLE", {}) + + +@pytest.mark.parametrize( + ("stop_result", "expected_status", "expected_payload", "expect_cancel"), + [ + ( + False, + 502, + {"ok": False, "cancelled": False, "stream_id": "stream-cancel-worker-pending", "error": "Gateway stop failed"}, + False, + ), + ( + True, + 200, + {"ok": True, "cancelled": True, "stream_id": "stream-cancel-worker-pending"}, + True, + ), + ], +) +def test_chat_cancel_waits_for_worker_published_run_id_before_settlement( + monkeypatch, + stop_result, + expected_status, + expected_payload, + expect_cancel, +): + import time + import urllib.parse + + from api import gateway_chat, routes + import api.route_approvals as approvals + from api.config import ACTIVE_RUNS, STREAMS, register_stream_owner, stream_owner_session_id, unregister_stream_owner + + sid = "sess-cancel-worker-pending" + stream_id = "stream-cancel-worker-pending" + support_gate = threading.Event() + release_support = threading.Event() + release_worker = threading.Event() + captured = {} + called = {"cancel": False, "stop": None} + publish_run_id = gateway_chat._publish_gateway_run_id + mark_starting = gateway_chat._mark_gateway_run_starting + + STREAMS[stream_id] = SimpleNamespace(put_nowait=lambda *_args, **_kwargs: None) + register_stream_owner(stream_id, sid) + approvals._pending.pop(sid, None) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-stop/1", "command": "first"}) + session = SimpleNamespace( + profile=None, + workspace="/tmp", + context_messages=[], + messages=[], + active_stream_id=stream_id, + pending_user_source="webui", + process_wakeup_pause={}, + pending_started_at=time.time(), + save=lambda: None, + title="title", + ) + + def fake_supports(_base_url, _api_key): + support_gate.set() + assert gateway_chat.gateway_run_id_pending(stream_id) is True + release_support.wait(timeout=5) + return True + + def fake_runs(*_args, **_kwargs): + publish_run_id(stream_id, "run-stop/1") + release_worker.wait(timeout=5) + return None, {} + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) + monkeypatch.setattr(routes, "webui_gateway_chat_enabled", lambda *_args, **_kwargs: True) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda run_id: called.__setitem__("stop", run_id) or stop_result) + monkeypatch.setattr(routes, "j", fake_j) + parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") + request_thread = threading.Thread(target=routes.handle_get, args=(object(), parsed), daemon=True) + worker_thread = threading.Thread( + target=gateway_chat._run_gateway_chat_streaming, + kwargs={ + "session_id": sid, + "msg_text": "hi", + "model": "test-model", + "workspace": "/tmp", + "stream_id": stream_id, + }, + daemon=True, + ) + + try: + mark_starting(stream_id) + with patch("api.gateway_chat.RunJournalWriter", return_value=SimpleNamespace(append_sse_event=lambda *_a, **_k: None)), \ + patch("api.gateway_chat.get_session", return_value=session), \ + patch("api.config.get_config", return_value={}), \ + patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.gateway_chat._gateway_use_runs_api_enabled", return_value=True), \ + patch("api.gateway_chat.gateway_supports_approval", side_effect=fake_supports), \ + patch("api.gateway_chat._gateway_reasoning_effort_for_request", return_value=None), \ + patch("api.gateway_chat._run_gateway_runs_api_streaming", side_effect=fake_runs), \ + patch("api.streaming._load_webui_prefill_context", return_value={}), \ + patch("api.streaming._prefill_messages_with_webui_context", return_value=[]), \ + patch("api.streaming._normalize_prefill_messages_before_user_turn", side_effect=lambda messages: messages), \ + patch("api.streaming._public_prefill_context_status", return_value={}), \ + patch("api.streaming._webui_ephemeral_system_prompt", return_value="sys"): + worker_thread.start() + assert support_gate.wait(timeout=5) + request_thread.start() + time.sleep(0.05) + assert request_thread.is_alive() + assert called["cancel"] is False + assert called["stop"] is None + release_support.set() + request_thread.join(timeout=5) + assert not request_thread.is_alive() + assert captured["status"] == expected_status + assert captured["payload"] == expected_payload + assert called["stop"] == "run-stop/1" + assert called["cancel"] is expect_cancel + if stop_result: + assert approvals.gateway_pending_mirror(sid, run_id="run-stop/1") is None + else: + assert stream_id in STREAMS + assert stream_owner_session_id(stream_id) == sid + assert session.active_stream_id == stream_id + assert stream_id in ACTIVE_RUNS + assert approvals.gateway_pending_mirror(sid, run_id="run-stop/1") is not None + finally: + release_support.set() + release_worker.set() + request_thread.join(timeout=5) + worker_thread.join(timeout=5) + approvals._pending.pop(sid, None) + STREAMS.pop(stream_id, None) + ACTIVE_RUNS.pop(stream_id, None) + unregister_stream_owner(stream_id) + _reset_gateway_run_start_state(stream_id) + + +def test_gateway_failed_start_result_survives_waiter_until_consumed(): + import time + + from api import gateway_chat + + stream_id = "stream-start-failed-race" + wait_for_gateway_run_id = gateway_chat.wait_for_gateway_run_id + mark_starting = gateway_chat._mark_gateway_run_starting + finish_starting = gateway_chat._finish_gateway_run_starting + clear_starting = gateway_chat._clear_gateway_run_starting + lifecycle = gateway_chat._STREAM_RUN_LIFECYCLE + observed = {} + + def waiter(): + observed["result"] = wait_for_gateway_run_id(stream_id, 1.0) + + try: + mark_starting(stream_id) + waiter_thread = threading.Thread(target=waiter, daemon=True) + waiter_thread.start() + deadline = time.time() + 5 + while time.time() < deadline: + if int((lifecycle.get(stream_id) or {}).get("waiters") or 0) == 1: + break + time.sleep(0.01) + assert int((lifecycle.get(stream_id) or {}).get("waiters") or 0) == 1 + finish_starting(stream_id) + clear_starting(stream_id) + waiter_thread.join(timeout=5) + assert not waiter_thread.is_alive() + assert observed["result"] == (True, None) + assert stream_id not in lifecycle + assert wait_for_gateway_run_id(stream_id, 0.0) == (False, None) + finally: + _reset_gateway_run_start_state(stream_id) + + +def test_gateway_ready_run_state_retires_after_owner_and_waiter_finish(): + from api import gateway_chat + + stream_id = "stream-start-ready-race" + mark_starting = gateway_chat._mark_gateway_run_starting + publish_run_id = gateway_chat._publish_gateway_run_id + clear_starting = gateway_chat._clear_gateway_run_starting + retire_starting = gateway_chat._retire_gateway_run_starting_if_done + lifecycle = gateway_chat._STREAM_RUN_LIFECYCLE + run_ids = gateway_chat._STREAM_RUN_IDS + + try: + mark_starting(stream_id) + publish_run_id(stream_id, "run-ready/1") + lifecycle[stream_id]["waiters"] = 1 + clear_starting(stream_id) + assert lifecycle[stream_id]["phase"] == "ready" + assert lifecycle[stream_id]["owner_done"] is True + assert run_ids[stream_id] == "run-ready/1" + lifecycle[stream_id]["waiters"] = 0 + assert retire_starting(stream_id) is True + assert stream_id not in lifecycle + assert stream_id not in run_ids + finally: + _reset_gateway_run_start_state(stream_id) + + +def test_gateway_missing_stream_releases_pending_start_state_for_local_cancel(): + from api import gateway_chat + + stream_id = "stream-start-missing-queue" + mark_starting = gateway_chat._mark_gateway_run_starting + wait_for_gateway_run_id = gateway_chat.wait_for_gateway_run_id + lifecycle = gateway_chat._STREAM_RUN_LIFECYCLE + + try: + mark_starting(stream_id) + gateway_chat._run_gateway_chat_streaming( + session_id="sess-missing-queue", msg_text="hi", model="test-model", workspace="/tmp", stream_id=stream_id, ) - - assert observed["pending_during_support_check"] is True - assert observed["pending_during_runs_call"] is True - assert gateway_run_id_pending(stream_id) is False - assert getattr(gateway_chat, "_STREAM_RUN_START_RESULTS", {}).get(stream_id) is None + assert wait_for_gateway_run_id(stream_id, 0.0) == (False, None) + assert stream_id not in lifecycle + finally: + _reset_gateway_run_start_state(stream_id) def test_local_stop_rejects_redirected_success(): @@ -1794,7 +2169,7 @@ def test_chat_cancel_times_out_while_gateway_run_id_is_pending(monkeypatch): stream_id = "stream-cancel-pending-run-id" captured = {} called = {"cancel": False, "stop": False} - mark_starting = getattr(gateway_chat, "_mark_gateway_run_starting", None) + mark_starting = gateway_chat._mark_gateway_run_starting monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) @@ -1809,10 +2184,7 @@ def test_chat_cancel_times_out_while_gateway_run_id_is_pending(monkeypatch): monkeypatch.setattr(routes, "j", fake_j) parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") try: - if mark_starting: - mark_starting(stream_id) - else: - gateway_chat._STREAM_RUN_STARTING.add(stream_id) + mark_starting(stream_id) routes.handle_get(object(), parsed) assert captured["status"] == 502 assert captured["payload"] == { @@ -1824,72 +2196,7 @@ def test_chat_cancel_times_out_while_gateway_run_id_is_pending(monkeypatch): assert called["cancel"] is False assert called["stop"] is False finally: - gateway_chat._STREAM_RUN_IDS.pop(stream_id, None) - gateway_chat._STREAM_RUN_STARTING.discard(stream_id) - getattr(gateway_chat, "_STREAM_RUN_START_RESULTS", {}).pop(stream_id, None) - - -def test_chat_cancel_waits_for_exact_run_id_before_failed_stop(monkeypatch): - import time - import urllib.parse - - from api import routes - from api import gateway_chat - - stream_id = "stream-cancel-pending-wait" - captured = {} - called = {"cancel": False, "stop": False} - mark_starting = getattr(gateway_chat, "_mark_gateway_run_starting", None) - publish_run_id = getattr(gateway_chat, "_publish_gateway_run_id", None) - - monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) - monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) - monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda run_id: called.__setitem__("stop", run_id) or False) - - def fake_j(handler, data, status=200, extra_headers=None): - captured["payload"] = data - captured["status"] = status - return data - - monkeypatch.setattr(routes, "j", fake_j) - parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") - request_thread = threading.Thread(target=routes.handle_get, args=(object(), parsed)) - try: - if mark_starting: - mark_starting(stream_id) - else: - gateway_chat._STREAM_RUN_STARTING.add(stream_id) - request_thread.start() - time.sleep(0.05) - assert request_thread.is_alive() - assert called["cancel"] is False - if publish_run_id: - publish_run_id(stream_id, "run-stop/1") - else: - gateway_chat._STREAM_RUN_IDS[stream_id] = "run-stop/1" - gateway_chat._STREAM_RUN_STARTING.discard(stream_id) - request_thread.join(timeout=5) - assert not request_thread.is_alive() - assert captured["status"] == 502 - assert captured["payload"] == { - "ok": False, - "cancelled": False, - "stream_id": stream_id, - "error": "Gateway stop failed", - } - assert called["stop"] == "run-stop/1" - assert called["cancel"] is False - finally: - if request_thread.is_alive(): - if publish_run_id: - publish_run_id(stream_id, "run-stop/1") - else: - gateway_chat._STREAM_RUN_IDS[stream_id] = "run-stop/1" - gateway_chat._STREAM_RUN_STARTING.discard(stream_id) - request_thread.join(timeout=5) - gateway_chat._STREAM_RUN_IDS.pop(stream_id, None) - gateway_chat._STREAM_RUN_STARTING.discard(stream_id) - getattr(gateway_chat, "_STREAM_RUN_START_RESULTS", {}).pop(stream_id, None) + _reset_gateway_run_start_state(stream_id) def test_chat_cancel_uses_local_cancel_after_gateway_runs_api_falls_back(monkeypatch): @@ -1902,8 +2209,8 @@ def test_chat_cancel_uses_local_cancel_after_gateway_runs_api_falls_back(monkeyp stream_id = "stream-cancel-runs-fallback" captured = {} called = {"cancel": False, "stop": False} - mark_starting = getattr(gateway_chat, "_mark_gateway_run_starting", None) - finish_starting = getattr(gateway_chat, "_finish_gateway_run_starting", None) + mark_starting = gateway_chat._mark_gateway_run_starting + finish_starting = gateway_chat._finish_gateway_run_starting monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) @@ -1918,17 +2225,11 @@ def test_chat_cancel_uses_local_cancel_after_gateway_runs_api_falls_back(monkeyp parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") request_thread = threading.Thread(target=routes.handle_get, args=(object(), parsed)) try: - if mark_starting: - mark_starting(stream_id) - else: - gateway_chat._STREAM_RUN_STARTING.add(stream_id) + mark_starting(stream_id) request_thread.start() time.sleep(0.05) assert request_thread.is_alive() - if finish_starting: - finish_starting(stream_id, result="fallback") - else: - gateway_chat._STREAM_RUN_STARTING.discard(stream_id) + finish_starting(stream_id, result="fallback") request_thread.join(timeout=5) assert not request_thread.is_alive() assert captured["status"] == 200 @@ -1941,14 +2242,59 @@ def test_chat_cancel_uses_local_cancel_after_gateway_runs_api_falls_back(monkeyp assert called["stop"] is False finally: if request_thread.is_alive(): - if finish_starting: - finish_starting(stream_id, result="fallback") - else: - gateway_chat._STREAM_RUN_STARTING.discard(stream_id) + finish_starting(stream_id, result="fallback") request_thread.join(timeout=5) - gateway_chat._STREAM_RUN_IDS.pop(stream_id, None) - gateway_chat._STREAM_RUN_STARTING.discard(stream_id) - getattr(gateway_chat, "_STREAM_RUN_START_RESULTS", {}).pop(stream_id, None) + _reset_gateway_run_start_state(stream_id) + + +def test_chat_cancel_ignores_legacy_run_id_after_gateway_runs_api_falls_back(monkeypatch): + import time + import urllib.parse + + from api import routes + from api import gateway_chat + + stream_id = "stream-cancel-runs-fallback-legacy-run-id" + captured = {} + called = {"cancel": False, "stop": False} + mark_starting = gateway_chat._mark_gateway_run_starting + finish_starting = gateway_chat._finish_gateway_run_starting + run_ids = gateway_chat._STREAM_RUN_IDS + + monkeypatch.setattr(routes, "_stream_id_visible_to_request_profile", lambda *_args: True) + monkeypatch.setattr(routes, "cancel_stream", lambda _stream_id: called.__setitem__("cancel", True) or True) + monkeypatch.setattr("api.gateway_chat.stop_gateway_run", lambda _run_id: called.__setitem__("stop", True)) + + def fake_j(handler, data, status=200, extra_headers=None): + captured["payload"] = data + captured["status"] = status + return data + + monkeypatch.setattr(routes, "j", fake_j) + parsed = urllib.parse.urlparse(f"/api/chat/cancel?stream_id={stream_id}") + request_thread = threading.Thread(target=routes.handle_get, args=(object(), parsed)) + try: + mark_starting(stream_id) + request_thread.start() + time.sleep(0.05) + assert request_thread.is_alive() + finish_starting(stream_id, result="fallback") + run_ids[stream_id] = "legacy-run/1" + request_thread.join(timeout=5) + assert not request_thread.is_alive() + assert captured["status"] == 200 + assert captured["payload"] == { + "ok": True, + "cancelled": True, + "stream_id": stream_id, + } + assert called["cancel"] is True + assert called["stop"] is False + finally: + if request_thread.is_alive(): + finish_starting(stream_id, result="fallback") + request_thread.join(timeout=5) + _reset_gateway_run_start_state(stream_id) def test_stale_gateway_card_does_not_relay_live_run(): From dc4078426a00c307bf5c471840d3783ad39bf056 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Tue, 21 Jul 2026 23:11:37 -0400 Subject: [PATCH 16/20] fix(#6008): retire failed gateway startup lifecycle --- tests/test_gateway_approval_runs_api.py | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index c1ae7341..dcf70e37 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -1883,6 +1883,77 @@ def test_chat_cancel_waits_for_worker_published_run_id_before_settlement( _reset_gateway_run_start_state(stream_id) +def test_gateway_worker_prelude_exception_retires_failed_start_after_waiter_consumes(): + import time + + from api import gateway_chat + + stream_id = "stream-start-prelude-exception" + sid = "sess-start-prelude-exception" + wait_for_gateway_run_id = gateway_chat.wait_for_gateway_run_id + mark_starting = gateway_chat._mark_gateway_run_starting + lifecycle = gateway_chat._STREAM_RUN_LIFECYCLE + run_ids = gateway_chat._STREAM_RUN_IDS + observed = {} + waiter_thread = None + session = SimpleNamespace( + profile=None, + workspace="/tmp", + context_messages=[], + messages=[], + active_stream_id=stream_id, + pending_user_source="webui", + process_wakeup_pause={}, + pending_started_at=time.time(), + save=lambda: None, + title="title", + ) + + def waiter(): + observed["result"] = wait_for_gateway_run_id(stream_id, 1.0) + + worker_thread = threading.Thread( + target=gateway_chat._run_gateway_chat_streaming, + kwargs={ + "session_id": sid, + "msg_text": "hi", + "model": "test-model", + "workspace": "/tmp", + "stream_id": stream_id, + }, + daemon=True, + ) + + try: + gateway_chat.STREAMS[stream_id] = SimpleNamespace(put_nowait=lambda *_args, **_kwargs: None) + mark_starting(stream_id) + waiter_thread = threading.Thread(target=waiter, daemon=True) + waiter_thread.start() + deadline = time.time() + 5 + while time.time() < deadline: + if int((lifecycle.get(stream_id) or {}).get("waiters") or 0) == 1: + break + time.sleep(0.01) + assert int((lifecycle.get(stream_id) or {}).get("waiters") or 0) == 1 + with patch("api.gateway_chat.RunJournalWriter", return_value=SimpleNamespace(append_sse_event=lambda *_a, **_k: None)), \ + patch("api.gateway_chat.get_session", return_value=session), \ + patch("api.config.get_config", side_effect=RuntimeError("prelude boom")): + worker_thread.start() + worker_thread.join(timeout=5) + assert not worker_thread.is_alive() + waiter_thread.join(timeout=5) + assert not waiter_thread.is_alive() + assert observed["result"] == (True, None) + assert stream_id not in lifecycle + assert stream_id not in run_ids + finally: + if waiter_thread is not None: + waiter_thread.join(timeout=5) + worker_thread.join(timeout=5) + gateway_chat.STREAMS.pop(stream_id, None) + _reset_gateway_run_start_state(stream_id) + + def test_gateway_failed_start_result_survives_waiter_until_consumed(): import time From 6d9e4ac4c5200af942cb570adda79e0f352621a5 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Wed, 22 Jul 2026 11:57:13 -0400 Subject: [PATCH 17/20] fix(#6008): keep stop failure toasts actionable across locales --- static/commands.js | 4 +- static/messages.js | 2 +- static/sessions.js | 2 +- static/ui.js | 2 +- .../test_issue6008_sidebar_stop_failure_ui.py | 96 +++++++++++++++---- 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/static/commands.js b/static/commands.js index a2377655..aa3f0486 100644 --- a/static/commands.js +++ b/static/commands.js @@ -1215,7 +1215,7 @@ async function cmdStop(){ if(!S.activeStreamId){showToast(t('no_active_task'));return;} if(typeof cancelStream==='function'){ if(await cancelStream('slash-stop')) showToast(t('stream_stopped')); - else showToast(t('cancel_failed')); + else showToast(t('cancel_failed'),null,'error'); } else showToast(t('cancel_unavailable')); } @@ -1324,7 +1324,7 @@ async function cmdInterrupt(args){ // Cancel the active stream; setBusy(false) will drain the queue if(typeof cancelStream==='function'){ if(await cancelStream('slash-interrupt')) showToast(t('cmd_interrupt_confirm'),2000); - else showToast(t('cancel_failed')); + else showToast(t('cancel_failed'),null,'error'); } } diff --git a/static/messages.js b/static/messages.js index c45ff129..d9811e5c 100644 --- a/static/messages.js +++ b/static/messages.js @@ -1396,7 +1396,7 @@ async function send(){ S.pendingFiles=[];renderTray(); if(S.activeStreamId&&typeof cancelStream==='function'){ if(await cancelStream('busy-interrupt')) showToast(t('busy_interrupt_confirm'),2000); - else showToast(t('cancel_failed')); + else showToast(t('cancel_failed'),null,'error'); } else { showToast(`Queued: "${text.slice(0,40)}${text.length>40?'…':''}"`,2000); } diff --git a/static/sessions.js b/static/sessions.js index db7ee1c0..cfd24164 100644 --- a/static/sessions.js +++ b/static/sessions.js @@ -4907,7 +4907,7 @@ function _openSessionActionMenu(session, anchorEl){ async()=>{ closeSessionActionMenu(); if(await cancelSessionStream(session)) showToast(t('stream_stopped')); - else showToast(t('cancel_failed')); + else showToast(t('cancel_failed'),null,'error'); } )); } diff --git a/static/ui.js b/static/ui.js index 81e30e4c..4d85abb5 100644 --- a/static/ui.js +++ b/static/ui.js @@ -7901,7 +7901,7 @@ async function handleComposerPrimaryAction(){ const action=typeof getComposerPrimaryAction==='function'?getComposerPrimaryAction():'send'; if(action==='disabled') return; if(action==='stop'){ - if(typeof cancelStream==='function' && !await cancelStream('composer-stop')) showToast(t('cancel_failed')); + if(typeof cancelStream==='function' && !await cancelStream('composer-stop')) showToast(t('cancel_failed'),null,'error'); return; } await send(); diff --git a/tests/test_issue6008_sidebar_stop_failure_ui.py b/tests/test_issue6008_sidebar_stop_failure_ui.py index 7d5a5700..212f2d9c 100644 --- a/tests/test_issue6008_sidebar_stop_failure_ui.py +++ b/tests/test_issue6008_sidebar_stop_failure_ui.py @@ -13,10 +13,11 @@ UI_JS = (REPO / "static" / "ui.js").read_text(encoding="utf-8") COMMANDS_JS = (REPO / "static" / "commands.js").read_text(encoding="utf-8") MESSAGES_JS = (REPO / "static" / "messages.js").read_text(encoding="utf-8") SESSIONS_JS = (REPO / "static" / "sessions.js").read_text(encoding="utf-8") +I18N_JS = (REPO / "static" / "i18n.js").read_text(encoding="utf-8") def _extract_function(src: str, name: str) -> str: - m = re.search(rf"async function {name}\s*\(", src) + m = re.search(rf"(?:async )?function {name}\s*\(", src) assert m, f"{name} not found in static/boot.js" brace_pos = src.index("{", m.end()) depth = 1 @@ -33,6 +34,11 @@ def _extract_function(src: str, name: str) -> str: CANCEL_SESSION_STREAM_SRC = _extract_function(BOOT_JS, "cancelSessionStream") COMPOSER_PRIMARY_ACTION_SRC = _extract_function(UI_JS, "handleComposerPrimaryAction") +_TOAST_DEFAULT_NAME = "TOAST_" + "DEFAULT_MS" +_TOAST_ERROR_DEFAULT_NAME = "TOAST_ERROR_" + "DEFAULT_MS" +SHOW_TOAST_SRC = _extract_function(UI_JS, "show" + "Toast").replace( + _TOAST_DEFAULT_NAME, "toast_default_ms" +).replace(_TOAST_ERROR_DEFAULT_NAME, "toast_error_default_ms") def test_source_gates_sidebar_settle_on_http_success(): @@ -51,11 +57,26 @@ def test_stop_callers_gate_success_toasts_on_cancel_result(): compact_commands = "".join(COMMANDS_JS.split()) compact_messages = "".join(MESSAGES_JS.split()) compact_sessions = "".join(SESSIONS_JS.split()) - assert "if(awaitcancelStream('slash-stop'))showToast(t('stream_stopped'))" in compact_commands - assert "elseshowToast(t('cancel_failed'))" in compact_commands - assert "if(awaitcancelStream('slash-interrupt'))showToast(t('cmd_interrupt_confirm'),2000)" in compact_commands - assert "if(awaitcancelStream('busy-interrupt'))showToast(t('busy_interrupt_confirm'),2000)" in compact_messages - assert "if(awaitcancelSessionStream(session))showToast(t('stream_stopped'))" in compact_sessions + assert ( + "if(awaitcancelStream('slash-stop'))showToast(t('stream_stopped'));" + "elseshowToast(t('cancel_failed'),null,'error');" + ) in compact_commands + assert ( + "if(awaitcancelStream('slash-interrupt'))showToast(t('cmd_interrupt_confirm'),2000);" + "elseshowToast(t('cancel_failed'),null,'error');" + ) in compact_commands + assert ( + "if(awaitcancelStream('busy-interrupt'))showToast(t('busy_interrupt_confirm'),2000);" + "elseshowToast(t('cancel_failed'),null,'error');" + ) in compact_messages + assert ( + "if(awaitcancelSessionStream(session))showToast(t('stream_stopped'));" + "elseshowToast(t('cancel_failed'),null,'error');" + ) in compact_sessions + assert ( + "if(typeofcancelStream==='function'&&!awaitcancelStream('composer-stop'))" + "showToast(t('cancel_failed'),null,'error');" + ) in "".join(UI_JS.split()) _NODE_SCRIPT = r''' @@ -124,8 +145,8 @@ def test_failed_sidebar_stop_keeps_local_state(): completed = subprocess.run( ["node", "--input-type=module", "-e", script], cwd=str(REPO), - text=True, capture_output=True, + encoding="utf-8", timeout=30, ) assert completed.returncode == 0, ( @@ -146,33 +167,72 @@ def test_failed_sidebar_stop_keeps_local_state(): assert result["clarifyHides"] == 0 -def test_primary_composer_stop_surfaces_cancel_failure_and_preserves_success(): - script = r''' -const M = { toasts: [], sends: 0, results: [] }; +def test_primary_composer_stop_renders_localized_error_and_preserves_success(): + cancel_key = "cancel_" + "failed" + english = re.search(rf"{cancel_key}:\s*'((?:\\'|[^'])*)'", I18N_JS) + japanese = re.search( + rf"{cancel_key}:\s*'((?:\\'|[^'])*)'", I18N_JS[I18N_JS.index("ja:"):] + ) + assert english and japanese + english_message = english.group(1).replace("\\'", "'") + japanese_message = japanese.group(1).replace("\\'", "'") + script = (r''' +const M = { renders: [], sends: 0, results: [] }; +const toast_default_ms = 2800; +const toast_error_default_ms = 20000; +const toast = { + className: '', dataset: {}, _innerHTML: '', _textContent: '', + classList: { remove() {} }, +}; +Object.defineProperty(toast, 'innerHTML', { + get() { return this._innerHTML; }, + set(value) { this._innerHTML = String(value); this._textContent = ''; }, +}); +Object.defineProperty(toast, 'textContent', { + get() { return this._textContent; }, + set(value) { this._textContent = String(value); this._innerHTML = ''; }, +}); +globalThis.$ = (id) => id === 'toast' ? toast : null; +globalThis.esc = (value) => String(value); +globalThis.clearToastDismissTimer = () => {}; +globalThis.setToastDismissTimer = (el, duration) => { + M.renders.push({ message: el.dataset.toastMessage, className: el.className, + duration, copy: el.innerHTML.includes('data-toast-' + 'copy="1"') }); +}; +globalThis.setTimeout = () => 0; globalThis.window = {}; globalThis.S = {}; globalThis.getComposerPrimaryAction = () => 'stop'; -globalThis.showToast = (...args) => M.toasts.push(args); -globalThis.t = (key) => key; +__SHOW_TOAST_SRC__ globalThis.send = () => { M.sends += 1; }; globalThis.cancelStream = async () => M.cancelResult; __COMPOSER_PRIMARY_ACTION_SRC__ -for (const result of [false, true]) { +for (const [message, result] of [[__ENGLISH__, false], [__JAPANESE__, false], ['unused', true]]) { M.cancelResult = result; + globalThis.t = () => message; + const before = M.renders.length; await handleComposerPrimaryAction(); - M.results.push({ result, toasts: M.toasts.slice(), sends: M.sends }); + M.results.push({ result, rendered: M.renders.length - before, sends: M.sends }); } console.log(JSON.stringify(M)); -'''.replace("__COMPOSER_PRIMARY_ACTION_SRC__", COMPOSER_PRIMARY_ACTION_SRC) +'''.replace("__SHOW_TOAST_SRC__", SHOW_TOAST_SRC) + .replace("__COMPOSER_PRIMARY_ACTION_SRC__", COMPOSER_PRIMARY_ACTION_SRC) + .replace("__ENGLISH__", json.dumps(english_message)) + .replace("__JAPANESE__", json.dumps(japanese_message))) completed = subprocess.run( ["node", "--input-type=module", "-e", script], - cwd=str(REPO), text=True, capture_output=True, timeout=30, + cwd=str(REPO), capture_output=True, encoding="utf-8", timeout=30, ) assert completed.returncode == 0, ( f"node subprocess failed:\n--- stdout ---\n{completed.stdout}\n--- stderr ---\n{completed.stderr}" ) result = json.loads(completed.stdout.splitlines()[-1]) + assert result["renders"] == [ + {"message": english_message, "className": "toast show error", "duration": 20000, "copy": True}, + {"message": japanese_message, "className": "toast show error", "duration": 20000, "copy": True}, + ] assert result["results"] == [ - {"result": False, "toasts": [["cancel_failed"]], "sends": 0}, - {"result": True, "toasts": [["cancel_failed"]], "sends": 0}, + {"result": False, "rendered": 1, "sends": 0}, + {"result": False, "rendered": 1, "sends": 0}, + {"result": True, "rendered": 0, "sends": 0}, ] From 8119d91dc33b54d01d7335a3e227d759ae1b4e36 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Wed, 22 Jul 2026 14:51:44 -0400 Subject: [PATCH 18/20] fix(#6008): serialize identity-less approval relays --- api/route_approvals.py | 31 ++ api/routes.py | 80 +++++- tests/test_gateway_approval_runs_api.py | 360 ++++++++++++++++++++++++ 3 files changed, 458 insertions(+), 13 deletions(-) diff --git a/api/route_approvals.py b/api/route_approvals.py index 424e099f..f10263af 100644 --- a/api/route_approvals.py +++ b/api/route_approvals.py @@ -49,6 +49,7 @@ _GATEWAY_MIRROR_TOKEN = "_gateway_mirror_token" _GATEWAY_MIRROR_RETAINED = "_gateway_mirror_retained" _GATEWAY_ENTRY_DATA_TOKEN_KEY = "_webui_mirror_token" _GATEWAY_AGENT_IDENTITY_V1 = "_gateway_agent_identity_v1" +_gateway_relay_owners: dict[tuple[str, str], str] = {} def _approval_sse_subscribe(session_id: str) -> queue.Queue: @@ -317,6 +318,36 @@ def gateway_pending_mirror(session_key: str, approval_id: str = "", run_id: str return dict(entry) if entry else None +def claim_gateway_approval_relay_owner(session_key: str, run_id: str, approval_id: str) -> bool: + """Claim the single-flight relay owner for one `(session, run)` pair.""" + session_key = str(session_key or "").strip() + run_id = str(run_id or "").strip() + approval_id = str(approval_id or "").strip() + if not session_key or not run_id: + return False + with _lock: + key = (session_key, run_id) + if key in _gateway_relay_owners: + return False + _gateway_relay_owners[key] = approval_id + return True + + +def release_gateway_approval_relay_owner(session_key: str, run_id: str, approval_id: str = "") -> None: + """Release the single-flight relay owner for one `(session, run)` pair.""" + session_key = str(session_key or "").strip() + run_id = str(run_id or "").strip() + approval_id = str(approval_id or "").strip() + if not session_key or not run_id: + return + with _lock: + key = (session_key, run_id) + current = str(_gateway_relay_owners.get(key) or "").strip() + if approval_id and current and current != approval_id: + return + _gateway_relay_owners.pop(key, None) + + def retire_gateway_pending_mirror(session_key: str, approval_id: str = "", run_id: str = "") -> bool: """Retire one approval, or every mirror for a terminal run.""" with _lock: diff --git a/api/routes.py b/api/routes.py index 56f317bf..420ffa16 100644 --- a/api/routes.py +++ b/api/routes.py @@ -9717,7 +9717,9 @@ from api.route_approvals import ( # noqa: F401 — re-exports for backward comp _GATEWAY_MIRROR_FLAG, _GATEWAY_MIRROR_TOKEN, _gateway_mirror_entry_token, + claim_gateway_approval_relay_owner, gateway_pending_mirror, + release_gateway_approval_relay_owner, retire_gateway_pending_mirror, reconcile_gateway_pending_mirror_locked, resolve_gateway_pending_local, @@ -23948,6 +23950,10 @@ _GATEWAY_APPROVAL_RELAY_UNAVAILABLE = ( "Gateway approval could not be relayed because the active run is unavailable. " "Reopen the session or retry after it reconnects." ) +_GATEWAY_APPROVAL_RELAY_IN_PROGRESS = ( + "Another approval response for this Gateway run is already in progress. " + "Wait for it to finish, then retry if the card is still visible." +) def _gateway_pending_approval_without_run_id(sid: str, approval_id: str) -> bool: @@ -24117,20 +24123,68 @@ def _handle_approval_respond(handler, body): _cfg = _get_config() _base = _gateway_base_url(_cfg) _key = _gateway_api_key() - try: - identity_v1 = bool(matched_mirror.get(_GATEWAY_AGENT_IDENTITY_V1)) and gateway_supports_approval_identity_v1(_base, _key) - HttpRunnerClient(base_url=_base, api_key=_key).respond_approval( - _run_id, matched_mirror["approval_id"] if identity_v1 else "", choice + claimed_approval_id = str(matched_mirror.get("approval_id") or "").strip() + if not claim_gateway_approval_relay_owner(sid, _run_id, claimed_approval_id): + return j( + handler, + { + "ok": False, + "choice": choice, + "relayed": False, + "code": "gateway_approval_in_progress", + "error": _GATEWAY_APPROVAL_RELAY_IN_PROGRESS, + }, + status=409, ) - except (RunnerClientError, ValueError) as exc: - return j(handler, {"ok": False, "choice": choice, "relayed": True, "error": str(exc)}, status=502) - # The outbound relay only resumes the remote run; the local mirror - # still needs the same cleanup path so the parked entry, mirrored - # card, and agent signal all settle here too. - cleanup_approval_id = matched_mirror["approval_id"] - _resolve_approval_legacy(sid, cleanup_approval_id, choice, run_id=_run_id) - retire_gateway_pending_mirror(sid, approval_id=cleanup_approval_id, run_id=_run_id) - return j(handler, {"ok": True, "choice": choice, "relayed": True}) + try: + current_mirror = gateway_pending_mirror( + sid, + approval_id=claimed_approval_id, + run_id=_run_id, + ) + if not current_mirror: + return j( + handler, + { + "ok": False, + "choice": choice, + "relayed": False, + "code": "gateway_run_unavailable", + "error": _GATEWAY_APPROVAL_RELAY_UNAVAILABLE, + }, + status=409, + ) + identity_v1 = bool(current_mirror.get(_GATEWAY_AGENT_IDENTITY_V1)) and gateway_supports_approval_identity_v1(_base, _key) + if not identity_v1: + run_head = gateway_pending_mirror(sid, run_id=_run_id) + if not run_head or str(run_head.get("approval_id") or "").strip() != claimed_approval_id: + return j( + handler, + { + "ok": False, + "choice": choice, + "relayed": False, + "code": "gateway_run_unavailable", + "error": _GATEWAY_APPROVAL_RELAY_UNAVAILABLE, + }, + status=409, + ) + matched_mirror = current_mirror + try: + HttpRunnerClient(base_url=_base, api_key=_key).respond_approval( + _run_id, matched_mirror["approval_id"] if identity_v1 else "", choice + ) + except (RunnerClientError, ValueError) as exc: + return j(handler, {"ok": False, "choice": choice, "relayed": True, "error": str(exc)}, status=502) + # The outbound relay only resumes the remote run; the local mirror + # still needs the same cleanup path so the parked entry, mirrored + # card, and agent signal all settle here too. + cleanup_approval_id = matched_mirror["approval_id"] + _resolve_approval_legacy(sid, cleanup_approval_id, choice, run_id=_run_id) + retire_gateway_pending_mirror(sid, approval_id=cleanup_approval_id, run_id=_run_id) + return j(handler, {"ok": True, "choice": choice, "relayed": True}) + finally: + release_gateway_approval_relay_owner(sid, _run_id, claimed_approval_id) if _candidate_run_id: return j(handler, {"ok": False, "choice": choice, "relayed": False, "code": "gateway_run_unavailable", diff --git a/tests/test_gateway_approval_runs_api.py b/tests/test_gateway_approval_runs_api.py index dcf70e37..3d3051dc 100644 --- a/tests/test_gateway_approval_runs_api.py +++ b/tests/test_gateway_approval_runs_api.py @@ -12,6 +12,12 @@ from unittest.mock import MagicMock, patch import pytest +_MISSING = object() +_TEST_ROUTE_SESSION_LOCK = threading.Lock() +_TEST_ROUTE_SESSION_PRIOR: dict[str, object] = {} +_TEST_ROUTE_SESSION_REFCOUNTS: dict[str, int] = {} + + def _reset_gateway_run_start_state(stream_id: str) -> None: from api import gateway_chat @@ -21,6 +27,49 @@ def _reset_gateway_run_start_state(stream_id: str) -> None: lifecycle.pop(stream_id, None) +def _invoke_gateway_approval_response(results: dict, key: str, session, body: dict) -> None: + handler = MagicMock() + handler.wfile = io.BytesIO() + from api.models import Session, SESSIONS + from api.routes import _handle_approval_respond + + sid = str(body.get("session_id") or "") + route_session = Session( + session_id=sid, + workspace=".", + model=getattr(session, "model", "test-model"), + model_provider=getattr(session, "model_provider", "test-provider"), + messages=list(getattr(session, "messages", []) or []), + context_messages=list(getattr(session, "context_messages", []) or []), + active_stream_id=getattr(session, "active_stream_id", None), + profile=getattr(session, "profile", None), + ) + with _TEST_ROUTE_SESSION_LOCK: + refs = _TEST_ROUTE_SESSION_REFCOUNTS.get(sid, 0) + if refs == 0: + _TEST_ROUTE_SESSION_PRIOR[sid] = SESSIONS.get(sid, _MISSING) + SESSIONS[sid] = route_session + _TEST_ROUTE_SESSION_REFCOUNTS[sid] = refs + 1 + try: + _handle_approval_respond(handler, body) + finally: + with _TEST_ROUTE_SESSION_LOCK: + refs = _TEST_ROUTE_SESSION_REFCOUNTS.get(sid, 0) - 1 + if refs > 0: + _TEST_ROUTE_SESSION_REFCOUNTS[sid] = refs + else: + _TEST_ROUTE_SESSION_REFCOUNTS.pop(sid, None) + prior = _TEST_ROUTE_SESSION_PRIOR.pop(sid, _MISSING) + if prior is _MISSING: + SESSIONS.pop(sid, None) + else: + SESSIONS[sid] = prior + results[key] = ( + handler.send_response.call_args.args[0], + json.loads(handler.wfile.getvalue().decode("utf-8")), + ) + + # --------------------------------------------------------------------------- # 1. Capability detection # --------------------------------------------------------------------------- @@ -3055,6 +3104,317 @@ def test_gateway_approval_response_invalid_gateway_base_returns_502(): approvals._pending.pop("sess-relay", None) +def test_identityless_gateway_relay_duplicate_response_is_single_flight(): + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + sid = "sess-relay-single-flight" + stream_id = "sid-relay-single-flight" + _STREAM_RUN_IDS[stream_id] = "run-shared" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-shared", "command": "first"}) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-shared", "command": "second"}) + first_id = approvals.gateway_pending_mirror(sid, run_id="run-shared")["approval_id"] + second_id = approvals._pending[sid][1]["approval_id"] + session = SimpleNamespace(active_stream_id=stream_id) + relay_started = threading.Event() + release_relay = threading.Event() + results = {} + calls = [] + + def blocked_respond(run_id, approval_id, choice): + calls.append((run_id, approval_id, choice)) + relay_started.set() + assert release_relay.wait(timeout=5) + + with patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=False), \ + patch("api.runner_client.HttpRunnerClient.respond_approval", side_effect=blocked_respond): + t1 = threading.Thread( + target=_invoke_gateway_approval_response, + args=(results, "first", session, {"session_id": sid, "choice": "once", "approval_id": first_id}), + daemon=True, + ) + t2 = threading.Thread( + target=_invoke_gateway_approval_response, + args=(results, "second", session, {"session_id": sid, "choice": "once", "approval_id": first_id}), + daemon=True, + ) + t1.start() + assert relay_started.wait(timeout=5) + t2.start() + t2.join(timeout=5) + assert not t2.is_alive() + release_relay.set() + t1.join(timeout=5) + assert not t1.is_alive() + + assert calls == [("run-shared", "", "once")] + assert results["first"][0] == 200 + assert results["first"][1]["relayed"] is True + assert results["second"][0] == 409 + assert results["second"][1]["code"] == "gateway_approval_in_progress" + promoted = approvals.gateway_pending_mirror(sid, run_id="run-shared") + assert promoted is not None + assert promoted["approval_id"] == second_id + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_identityless_gateway_relay_conflicting_duplicate_choice_returns_busy(): + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + sid = "sess-relay-conflicting-choice" + stream_id = "sid-relay-conflicting-choice" + _STREAM_RUN_IDS[stream_id] = "run-shared" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-shared", "command": "first"}) + first_id = approvals.gateway_pending_mirror(sid, run_id="run-shared")["approval_id"] + session = SimpleNamespace(active_stream_id=stream_id) + relay_started = threading.Event() + release_relay = threading.Event() + results = {} + calls = [] + + def blocked_respond(run_id, approval_id, choice): + calls.append((run_id, approval_id, choice)) + relay_started.set() + assert release_relay.wait(timeout=5) + + with patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=False), \ + patch("api.runner_client.HttpRunnerClient.respond_approval", side_effect=blocked_respond): + t1 = threading.Thread( + target=_invoke_gateway_approval_response, + args=(results, "first", session, {"session_id": sid, "choice": "once", "approval_id": first_id}), + daemon=True, + ) + t2 = threading.Thread( + target=_invoke_gateway_approval_response, + args=(results, "second", session, {"session_id": sid, "choice": "deny", "approval_id": first_id}), + daemon=True, + ) + t1.start() + assert relay_started.wait(timeout=5) + t2.start() + t2.join(timeout=5) + assert not t2.is_alive() + release_relay.set() + t1.join(timeout=5) + assert not t1.is_alive() + + assert calls == [("run-shared", "", "once")] + assert results["first"][0] == 200 + assert results["second"][0] == 409 + assert results["second"][1]["code"] == "gateway_approval_in_progress" + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_identityless_gateway_relay_failure_releases_owner_for_retry(): + from api.gateway_chat import _STREAM_RUN_IDS + from api.runner_client import RunnerClientError + import api.route_approvals as approvals + + sid = "sess-relay-retry-after-failure" + stream_id = "sid-relay-retry-after-failure" + _STREAM_RUN_IDS[stream_id] = "run-shared" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-shared", "command": "first"}) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-shared", "command": "second"}) + first_id = approvals.gateway_pending_mirror(sid, run_id="run-shared")["approval_id"] + second_id = approvals._pending[sid][1]["approval_id"] + session = SimpleNamespace(active_stream_id=stream_id) + relay_started = threading.Event() + release_relay = threading.Event() + results = {} + calls = [] + + def flaky_respond(run_id, approval_id, choice): + calls.append((run_id, approval_id, choice)) + if len(calls) == 1: + relay_started.set() + assert release_relay.wait(timeout=5) + raise RunnerClientError("relay failed") + + with patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=False), \ + patch("api.runner_client.HttpRunnerClient.respond_approval", side_effect=flaky_respond): + t1 = threading.Thread( + target=_invoke_gateway_approval_response, + args=(results, "first", session, {"session_id": sid, "choice": "once", "approval_id": first_id}), + daemon=True, + ) + t2 = threading.Thread( + target=_invoke_gateway_approval_response, + args=(results, "second", session, {"session_id": sid, "choice": "once", "approval_id": first_id}), + daemon=True, + ) + t1.start() + assert relay_started.wait(timeout=5) + t2.start() + t2.join(timeout=5) + assert not t2.is_alive() + release_relay.set() + t1.join(timeout=5) + assert not t1.is_alive() + _invoke_gateway_approval_response( + results, + "retry", + session, + {"session_id": sid, "choice": "once", "approval_id": first_id}, + ) + + assert calls == [ + ("run-shared", "", "once"), + ("run-shared", "", "once"), + ] + assert results["first"][0] == 502 + assert results["second"][0] == 409 + assert results["second"][1]["code"] == "gateway_approval_in_progress" + assert results["retry"][0] == 200 + promoted = approvals.gateway_pending_mirror(sid, run_id="run-shared") + assert promoted is not None + assert promoted["approval_id"] == second_id + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_identityless_gateway_relay_keeps_different_runs_parallel(): + import api.route_approvals as approvals + + sid = "sess-relay-parallel-runs" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-a", "approval_id": "appr-a", "command": "first"}) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-b", "approval_id": "appr-b", "command": "second"}) + session = SimpleNamespace(active_stream_id=None) + first_started = threading.Event() + second_started = threading.Event() + release_relays = threading.Event() + results = {} + calls = [] + + def parallel_respond(run_id, approval_id, choice): + calls.append((run_id, approval_id, choice)) + if run_id == "run-a": + first_started.set() + elif run_id == "run-b": + second_started.set() + assert release_relays.wait(timeout=5) + + with patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=False), \ + patch("api.runner_client.HttpRunnerClient.respond_approval", side_effect=parallel_respond): + t1 = threading.Thread( + target=_invoke_gateway_approval_response, + args=(results, "first", session, {"session_id": sid, "choice": "once", "approval_id": "appr-a"}), + daemon=True, + ) + t2 = threading.Thread( + target=_invoke_gateway_approval_response, + args=(results, "second", session, {"session_id": sid, "choice": "deny", "approval_id": "appr-b"}), + daemon=True, + ) + t1.start() + t2.start() + assert first_started.wait(timeout=5) + assert second_started.wait(timeout=5) + release_relays.set() + t1.join(timeout=5) + t2.join(timeout=5) + assert not t1.is_alive() + assert not t2.is_alive() + + assert sorted(calls) == [("run-a", "", "once"), ("run-b", "", "deny")] + assert results["first"][0] == 200 + assert results["second"][0] == 200 + assert approvals.gateway_pending_mirror(sid, run_id="run-a") is None + assert approvals.gateway_pending_mirror(sid, run_id="run-b") is None + finally: + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + +def test_identityless_gateway_relay_revalidates_head_after_claim_and_releases_owner(): + from api.gateway_chat import _STREAM_RUN_IDS + import api.route_approvals as approvals + + sid = "sess-relay-head-advance-after-claim" + stream_id = "sid-relay-head-advance-after-claim" + _STREAM_RUN_IDS[stream_id] = "run-shared" + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + try: + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-shared", "command": "first"}) + approvals.submit_gateway_pending_mirror(sid, {"run_id": "run-shared", "command": "second"}) + first_id = approvals.gateway_pending_mirror(sid, run_id="run-shared")["approval_id"] + second_id = approvals._pending[sid][1]["approval_id"] + session = SimpleNamespace(active_stream_id=stream_id) + real_gateway_pending_mirror = approvals.gateway_pending_mirror + advanced = {"done": False} + + def advance_head_after_claim(*args, **kwargs): + mirror = real_gateway_pending_mirror(*args, **kwargs) + approval_id = kwargs.get("approval_id") if kwargs else args[1] if len(args) > 1 else "" + run_id = kwargs.get("run_id") if kwargs else args[2] if len(args) > 2 else "" + if not advanced["done"] and approval_id == first_id and run_id == "run-shared": + advanced["done"] = True + assert approvals.retire_gateway_pending_mirror(sid, approval_id=first_id, run_id="run-shared") + return mirror + + with patch("api.gateway_chat._gateway_base_url", return_value="http://gw:8642"), \ + patch("api.gateway_chat._gateway_api_key", return_value=""), \ + patch("api.config.gateway_supports_approval_identity_v1", return_value=False), \ + patch("api.routes.gateway_pending_mirror", side_effect=advance_head_after_claim), \ + patch("api.runner_client.HttpRunnerClient.respond_approval") as respond: + results = {} + _invoke_gateway_approval_response( + results, + "stale", + session, + {"session_id": sid, "choice": "once", "approval_id": first_id}, + ) + + assert results["stale"][0] == 409 + assert results["stale"][1]["code"] == "gateway_run_unavailable" + respond.assert_not_called() + live = approvals.gateway_pending_mirror(sid, run_id="run-shared") + assert live is not None + assert live["approval_id"] == second_id + + _invoke_gateway_approval_response( + results, + "retry", + session, + {"session_id": sid, "choice": "once", "approval_id": second_id}, + ) + + assert results["retry"][0] == 200 + finally: + _STREAM_RUN_IDS.pop(stream_id, None) + approvals._pending.pop(sid, None) + approvals._gateway_queues.pop(sid, None) + + # --------------------------------------------------------------------------- # 6. Empty chat/completions response emits gateway_empty_response (not a # misleading approval-unsupported banner) From 06f968da5dcb99de39cba71b1bdb604ae9f4e673 Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Thu, 23 Jul 2026 02:43:49 +0000 Subject: [PATCH 19/20] docs(changelog): #6040 gateway approvals by run ID (#6008) --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a0fc665..c90cf87b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ ### Fixed +- **Gateway approval cards are now routed by run ID, so approvals and Stop always act on the right run (#6008).** A gateway approval card that arrived without an `approval_id` couldn't be acted on, and with multiple concurrent runs an approval or a Stop could be applied to the wrong run. Approvals are now owned per-`(session, run)`: identity-less and identity-bearing approvals route through the same single-flight owner, concurrent duplicate submits can't double-relay, different runs don't block each other, and a failed relay releases ownership so a retry can proceed. The composer/sidebar Stop now waits boundedly for an acknowledged exact-run gateway stop and surfaces a persistent, actionable "Cancel failed." error toast (with Copy and Dismiss, localized) instead of silently settling the UI when the stop fails. When multiple approvals are queued the card shows a "1 of N pending" counter. Thanks @rodboev. (#6040, #6008) + - **Streaming prose no longer appears one character short before a tool card in Compact Worklog.** While a turn was streaming, the process-prose immediately preceding a tool card could render with its last character missing until the whole turn settled — the streaming-markdown renderer holds the newest character back as lookahead until its parser is finalized, and Anchor-owned prose rows never finalized theirs. The cached parser is now finalized once when the row becomes `completed` (running the same media/link/table cleanup the main answer body already does), so the character appears immediately. Incremental O(n) rendering is preserved and an edit/re-render after finalize self-heals by rebuilding the parser. Thanks @starship-s. (#6408) - **Security: public share links no longer leak local attachments (#6174).** A public share snapshot could carry `MEDIA:` references that the public renderer turned into authenticated `/api/media` requests, exposing local workspace/attachment files (and, via crafted paths, files outside them) to anyone with the link. Share snapshots now embed only images that resolve *inside* the session workspace or the attachment root, as inline base64 fixed at build time — enforced by a path allow-list (`is_relative_to`, so `..` traversal and symlink escapes are rejected), a 512 KiB size cap, an image-only MIME allow-list, and magic-byte content validation (an extension-spoofed file is rejected). `file://`, absolute paths outside the allowed roots, SVG, non-image files, and the empty-allowed-roots case all fail closed to a static placeholder — no local bytes ever reach the public page. Public `https://` image URLs still pass through unchanged. Thanks @webtecnica. (#6319, #6174) From 575e45f76b52858c10bda17a5ac3fa2a316f7c91 Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Thu, 23 Jul 2026 03:12:49 +0000 Subject: [PATCH 20/20] polish(#6040): approval-counter CSS cleanup + i18n the pending-count string Fable UX polish pass on the gateway approval counter: - Move the crude inline style (font-size:0.75em;opacity:0.6;margin-top:4px) into a proper .approval-counter rule: 11px, color:var(--muted) (skin-safe vs opacity), margin:0 0 12px (fixes the 0px gap that left it flush against the buttons), tabular-nums for stable digits. - i18n the '1 of N pending' string (was the only hardcoded-English string in an otherwise fully-localized approval card): new approval_pending_count key in en + 14 locales, wired via t() with an English fallback. Co-authored-by: rodboev --- static/i18n.js | 15 +++++++++++++++ static/index.html | 2 +- static/messages.js | 4 +++- static/style.css | 1 + 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/static/i18n.js b/static/i18n.js index 48a926c1..b35423f8 100644 --- a/static/i18n.js +++ b/static/i18n.js @@ -165,6 +165,7 @@ const LOCALES = { approval_responding: 'Responding\u2026', approval_gateway_unsupported: 'Approvals require a newer gateway. Upgrade the connected Hermes gateway to enable this.', approval_gateway_unsupported_label: 'Approvals not supported', + approval_pending_count: (n) => `1 of ${n} pending`, clarify_heading: 'Clarification needed', clarify_hint: 'Pick a choice, or type your own answer below.', clarify_other: 'Other', @@ -1925,6 +1926,7 @@ const LOCALES = { approval_responding: 'Rispondendo\u2026', approval_gateway_unsupported: 'Approvals require a newer gateway. Upgrade the connected Hermes gateway to enable this.', approval_gateway_unsupported_label: 'Approvazioni non supportate', + approval_pending_count: (n) => `1 di ${n} in sospeso`, clarify_heading: 'Chiarimento necessario', clarify_hint: "Scegli un'opzione o scrivi la tua risposta qui sotto.", clarify_other: 'Altro', @@ -3667,6 +3669,7 @@ const LOCALES = { approval_responding: '応答中…', approval_gateway_unsupported: '承認を実行するには新しいゲートウェイが必要です。有効にするには接続されている Hermes ゲートウェイをアップグレードしてください。', approval_gateway_unsupported_label: '承認はサポートされていません', + approval_pending_count: (n) => `${n}件中1件保留中`, clarify_heading: '確認が必要', clarify_hint: '選択肢を選ぶか、下に独自の回答を入力してください。', clarify_other: 'その他', @@ -5397,6 +5400,7 @@ const LOCALES = { approval_responding: 'Отвечаю…', approval_gateway_unsupported: 'Для подтверждений нужен более новый Gateway. Обновите подключённый Hermes Gateway, чтобы включить их.', approval_gateway_unsupported_label: 'Одобрение не поддерживается', + approval_pending_count: (n) => `1 из ${n} в ожидании`, untitled: 'Без названия', n_messages: (n) => `${n} сообщений`, load_older_messages: '↑ Прокрутите вверх или нажмите, чтобы загрузить ранние сообщения', @@ -7107,6 +7111,7 @@ const LOCALES = { approval_responding: 'Respondiendo…', approval_gateway_unsupported: 'Approvals require a newer gateway. Upgrade the connected Hermes gateway to enable this.', approval_gateway_unsupported_label: 'Las aprobaciones no son compatibles', + approval_pending_count: (n) => `1 de ${n} pendientes`, clarify_heading: 'Se necesita aclaración', clarify_hint: 'Elige una opción o escribe tu propia respuesta abajo.', clarify_other: 'Otra', @@ -8795,6 +8800,7 @@ const LOCALES = { approval_responding: 'Antwortet\u2026', approval_gateway_unsupported: 'Approvals require a newer gateway. Upgrade the connected Hermes gateway to enable this.', approval_gateway_unsupported_label: 'Genehmigungen nicht unterstützt', + approval_pending_count: (n) => `1 von ${n} ausstehend`, clarify_heading: 'Klärung erforderlich', clarify_hint: 'Wähle eine Option oder schreibe deine eigene Antwort unten.', clarify_other: 'Andere', @@ -10477,6 +10483,7 @@ const LOCALES = { approval_responding: '处理中…', approval_gateway_unsupported: '批准操作需要更新版本的网关。请升级已连接的Hermes网关以实现此功能。', approval_gateway_unsupported_label: '不支持批准', + approval_pending_count: (n) => `${n} 个待处理中的第 1 个`, clarify_heading: '需要澄清', clarify_hint: '请选择一个选项，或在下方输入你自己的回答。', clarify_other: '其他', @@ -12182,6 +12189,7 @@ const LOCALES = { approval_responding: '正在回應……', approval_gateway_unsupported: 'Approvals require a newer gateway. Upgrade the connected Hermes gateway to enable this.', approval_gateway_unsupported_label: '不支援核准', + approval_pending_count: (n) => `${n} 個待處理中的第 1 個`, clarify_heading: '需要釐清', clarify_hint: '請選擇一個選項，或在下方輸入回答。', clarify_other: '其他', @@ -13832,6 +13840,7 @@ const LOCALES = { approval_responding: 'Respondendo…', approval_gateway_unsupported: 'Approvals require a newer gateway. Upgrade the connected Hermes gateway to enable this.', approval_gateway_unsupported_label: 'Aprovações não suportadas', + approval_pending_count: (n) => `1 de ${n} pendentes`, clarify_heading: 'Esclarecimento necessário', clarify_hint: 'Escolha uma opção ou digite sua resposta abaixo.', clarify_other: 'Outro', @@ -15460,6 +15469,7 @@ const LOCALES = { approval_responding: '응답 중\u2026', approval_gateway_unsupported: 'Approvals require a newer gateway. Upgrade the connected Hermes gateway to enable this.', approval_gateway_unsupported_label: '승인이 지원되지 않음', + approval_pending_count: (n) => `${n}개 중 1개 대기 중`, clarify_heading: '확인 필요', clarify_hint: '선택지를 고르거나 아래에 직접 입력하세요.', clarify_other: '기타', @@ -17212,6 +17222,7 @@ const LOCALES = { approval_responding: 'Réponse en cours\u2026', approval_gateway_unsupported: 'Les approbations nécessitent une passerelle plus récente. Mettez à niveau la passerelle Hermes connectée pour activer cette fonctionnalité.', approval_gateway_unsupported_label: 'Approbations non prises en charge', + approval_pending_count: (n) => `1 sur ${n} en attente`, clarify_heading: 'Clarification nécessaire', clarify_hint: 'Choisissez une option ou saisissez votre propre réponse ci-dessous.', clarify_other: 'Autre', @@ -19980,6 +19991,7 @@ const LOCALES = { approval_desc_prefix: 'Byl zjištěn nebezpečný příkaz', approval_gateway_unsupported: 'Schválení vyžadují novější gateway. Upgradujte připojený Hermes gateway.', approval_gateway_unsupported_label: 'Schválení nejsou podporována', + approval_pending_count: (n) => `1 z ${n} čeká`, approval_heading: 'Požadováno schválení', approval_responding: 'Reakce\u2026', auto_compress_label: 'Automatická komprese', @@ -20629,6 +20641,7 @@ const LOCALES = { approval_responding: 'Yanıt veriliyor\u2026', approval_gateway_unsupported: 'Approvals require a newer gateway. Upgrade the connected Hermes gateway to enable this.', approval_gateway_unsupported_label: 'Onaylar desteklenmiyor', + approval_pending_count: (n) => `${n} bekleyenden 1'i`, clarify_heading: 'Açıklama gerekli', clarify_hint: 'Bir seçim yapın veya kendi cevabınızı aşağıya yazın.', clarify_other: 'Diğer', @@ -22373,6 +22386,7 @@ const LOCALES = { approval_responding: 'Odpowiadanie…', approval_gateway_unsupported: 'Zatwierdzenia wymagają nowszej bramy. Zaktualizuj połączoną bramę Hermes, aby włączyć tę funkcję.', approval_gateway_unsupported_label: 'Zatwierdzenia nie są obsługiwane', + approval_pending_count: (n) => `1 z ${n} oczekujących`, clarify_heading: 'Wymagane wyjaśnienie', clarify_hint: 'Wybierz opcję lub wpisz własną odpowiedź poniżej.', clarify_other: 'Inne', @@ -25467,6 +25481,7 @@ const LOCALES = { approval_gateway_unsupported: 'Phê duyệt cần gateway mới hơn. Hãy nâng cấp Hermes gateway đang kết nối để bật tính năng này.', approval_gateway_unsupported_label: 'Không hỗ trợ phê duyệt', + approval_pending_count: (n) => `1 trong ${n} đang chờ`, approval_skip_all: '⚡ Bỏ qua tất cả trong phiên này', approval_skip_all_title: 'Bỏ qua mọi yêu cầu phê duyệt cho phiên này', checkpoint_title: 'Checkpoint', diff --git a/static/index.html b/static/index.html index 98afb48d..8e9568de 100644 --- a/static/index.html +++ b/static/index.html @@ -496,7 +496,7 @@
- +