"""Regression coverage for #1897 — same-session profile switch identity bleed.""" from __future__ import annotations import os import queue import sys import types from pathlib import Path REPO = Path(__file__).resolve().parent.parent STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8") def _compute_agent_cache_signature_source() -> str: """Return the source of the `_compute_agent_cache_signature()` helper. The signature blob used to be inlined in the streaming send path; it now lives in this helper so the initial send and both self-heal retry paths derive the signature from the same final runtime bundle. """ start = STREAMING_PY.index("def _compute_agent_cache_signature(") end = STREAMING_PY.index("\ndef ", start) return STREAMING_PY[start:end] def _signature_block() -> str: """Return the `_json.dumps([...])` field list the signature hashes.""" helper = _compute_agent_cache_signature_source() sig_start = helper.index("_sig_blob = _json.dumps") sig_end = helper.index("], sort_keys=True)", sig_start) return helper[sig_start:sig_end] def _production_signature_calls() -> list[tuple[int, str]]: """Return `(offset, source)` for every production signature call site. Paren-balanced so the whole multi-line keyword-argument list is captured, and every live call site is returned so a retry path cannot silently drop a field that the initial send still passes. Commented-out occurrences are skipped -- a disabled retry call must read as missing, not as present. """ marker = "_agent_sig = _compute_agent_cache_signature(" calls: list[tuple[int, str]] = [] pos = STREAMING_PY.find(marker) while pos != -1: line_start = STREAMING_PY.rfind("\n", 0, pos) + 1 if STREAMING_PY[line_start:pos].strip(): # Commented-out (or otherwise non-statement) occurrence: it no # longer runs, so it must not count as a live call site. pos = STREAMING_PY.find(marker, pos + 1) continue depth = 0 for idx in range(pos + len(marker) - 1, len(STREAMING_PY)): char = STREAMING_PY[idx] if char == "(": depth += 1 elif char == ")": depth -= 1 if depth == 0: calls.append((pos, STREAMING_PY[pos:idx + 1])) break else: raise AssertionError("unterminated _compute_agent_cache_signature( call") pos = STREAMING_PY.find(marker, pos + 1) return calls # Lifecycle anchors for the streaming send path. Pinning one signature call to # each region keeps the oracle honest: a retry call that silently disappears # fails the region check instead of hiding behind the calls that remain. INITIAL_SEND_MARKER = "# ── Agent cache: reuse across messages in the same session ──" RETURNED_ERROR_SELF_HEAL_MARKER = ( "logger.info('[webui] self-heal: retrying stream after credential refresh')" ) RAISED_EXCEPTION_SELF_HEAL_MARKER = ( "logger.info('[webui] self-heal (except path): " "retrying stream after credential refresh')" ) # Each region ends at the cache write that stores the signature it just # computed. Bounding the region at the write -- rather than at the next # lifecycle anchor or at EOF -- is what proves the signature is recomputed # *before* it is cached: a call moved below its own write lands outside the # region and fails, instead of passing while the write consumes a stale # `_agent_sig`. INITIAL_SEND_CACHE_WRITE_MARKER = ( "SESSION_AGENT_CACHE[session_id] = (agent, _agent_sig)" ) RETURNED_ERROR_SELF_HEAL_CACHE_WRITE_MARKER = ( "_SAC[session_id] = (agent, _agent_sig)" ) RAISED_EXCEPTION_SELF_HEAL_CACHE_WRITE_MARKER = ( "_SAC2[session_id] = (_heal_agent, _agent_sig)" ) def _marker_offset(marker: str) -> int: """Return the single offset of a lifecycle marker in streaming.py.""" offset = STREAMING_PY.index(marker) assert STREAMING_PY.find(marker, offset + 1) == -1, ( "lifecycle marker is no longer unique in streaming.py:\n" + marker ) return offset def _signature_calls_by_region( calls: list[tuple[int, str]], ) -> dict[str, tuple[int, str]]: """Map each lifecycle region to the single signature call inside it. The regions are the initial send, the returned-error self-heal retry and the raised-exception self-heal retry; each runs from its lifecycle anchor to the cache write that consumes `_agent_sig`. Requiring exactly one call inside those bounds means neither a dropped retry call (masked by its surviving siblings) nor a call recomputed after its own cache write (which would leave the write storing a stale signature) can read as correct. """ initial = _marker_offset(INITIAL_SEND_MARKER) initial_cache_write = _marker_offset(INITIAL_SEND_CACHE_WRITE_MARKER) returned_error = _marker_offset(RETURNED_ERROR_SELF_HEAL_MARKER) returned_error_cache_write = _marker_offset( RETURNED_ERROR_SELF_HEAL_CACHE_WRITE_MARKER ) raised_exception = _marker_offset(RAISED_EXCEPTION_SELF_HEAL_MARKER) raised_exception_cache_write = _marker_offset( RAISED_EXCEPTION_SELF_HEAL_CACHE_WRITE_MARKER ) assert ( initial < initial_cache_write < returned_error < returned_error_cache_write < raised_exception < raised_exception_cache_write ), ( "streaming.py lifecycle regions are out of order; these anchors no " "longer describe the send path" ) bounds = { "initial send": (initial, initial_cache_write), "returned-error self-heal": (returned_error, returned_error_cache_write), "raised-exception self-heal": ( raised_exception, raised_exception_cache_write, ), } by_region: dict[str, tuple[int, str]] = {} for region, (start, end) in bounds.items(): found = [(offset, call) for offset, call in calls if start < offset < end] assert len(found) == 1, ( "expected exactly one _compute_agent_cache_signature() call in the " f"{region} region of streaming.py -- between its lifecycle anchor " f"and the cache write that stores `_agent_sig` -- found " f"{len(found)}" ) by_region[region] = found[0] return by_region def test_same_session_profile_switch_rebuilds_agent_under_new_soul_home(tmp_path, monkeypatch): """Switching profiles in one WebUI session must not reuse old SOUL.md. The fake AIAgent mirrors the real failure mode: it reads SOUL.md from HERMES_HOME at construction time and keeps that value in a cached system prompt. Two consecutive turns on the same profile should reuse the agent; changing only ``session.profile`` should create a fresh agent whose cached prompt comes from the new synthetic profile home. """ sys.path.insert(0, str(REPO)) from api import config as cfg from api import oauth from api import profiles from api import streaming default_home = tmp_path / "hermes-home" profile_a_home = default_home / "profiles" / "alpha" profile_b_home = default_home / "profiles" / "beta" profile_a_home.mkdir(parents=True) profile_b_home.mkdir(parents=True) (profile_a_home / "SOUL.md").write_text( "PROFILE_ALPHA_SYNTHETIC_SOUL", encoding="utf-8", ) (profile_b_home / "SOUL.md").write_text( "PROFILE_BETA_SYNTHETIC_SOUL", encoding="utf-8", ) class FakeSession: def __init__(self): self.session_id = "issue1897-same-session" self.title = "Pinned test title" self.workspace = str(tmp_path) self.model = "test-model" self.model_provider = None self.profile = "alpha" self.personality = None self.messages = [] self.context_messages = [] self.tool_calls = [] self.input_tokens = 0 self.output_tokens = 0 self.estimated_cost = None self.context_length = 0 self.threshold_tokens = 0 self.last_prompt_tokens = 0 self.active_stream_id = None self.pending_user_message = None self.pending_attachments = [] self.pending_started_at = None self.llm_title_generated = True def save(self, *args, **kwargs): return None def compact(self): return { "session_id": self.session_id, "title": self.title, "workspace": self.workspace, "model": self.model, "created_at": 0, "updated_at": 0, "pinned": False, "archived": False, "project_id": None, "profile": self.profile, "input_tokens": self.input_tokens, "output_tokens": self.output_tokens, "estimated_cost": self.estimated_cost, "personality": self.personality, } constructed_agents = [] prompts_used_for_runs = [] homes_seen_during_runs = [] class SoulCachingAgent: def __init__(self, **kwargs): self.session_id = kwargs.get("session_id") self.model = kwargs.get("model") self.provider = kwargs.get("provider") self.base_url = kwargs.get("base_url") self.context_compressor = None self.session_prompt_tokens = 0 self.session_completion_tokens = 0 self.session_estimated_cost_usd = None self.ephemeral_system_prompt = None self._last_error = None self.stream_delta_callback = kwargs.get("stream_delta_callback") self.tool_progress_callback = kwargs.get("tool_progress_callback") self.reasoning_callback = kwargs.get("reasoning_callback") self.clarify_callback = kwargs.get("clarify_callback") home = Path(os.environ["HERMES_HOME"]) self.constructed_home = str(home) self._cached_system_prompt = (home / "SOUL.md").read_text(encoding="utf-8") constructed_agents.append(self) def run_conversation(self, **kwargs): prompts_used_for_runs.append(self._cached_system_prompt) homes_seen_during_runs.append(os.environ.get("HERMES_HOME")) history = list(kwargs.get("conversation_history") or []) return { "messages": history + [ {"role": "user", "content": kwargs.get("persist_user_message", "")}, { "role": "assistant", "content": f"reply from {self._cached_system_prompt}", }, ] } def interrupt(self, _message): return None fake_session = FakeSession() fake_runtime_module = types.ModuleType("hermes_cli.runtime_provider") fake_runtime_module.resolve_runtime_provider = lambda requested=None: { "provider": requested or "test-provider", "api_key": "synthetic-key", "base_url": None, } fake_hermes_cli = types.ModuleType("hermes_cli") fake_hermes_cli.runtime_provider = fake_runtime_module fake_hermes_state = types.ModuleType("hermes_state") fake_hermes_state.SessionDB = lambda: None def home_for_profile(profile_name): return {"alpha": profile_a_home, "beta": profile_b_home}[profile_name] monkeypatch.setattr(streaming, "get_session", lambda _sid: fake_session) monkeypatch.setattr(streaming, "_get_ai_agent", lambda: SoulCachingAgent) monkeypatch.setattr( streaming, "resolve_model_provider", lambda _model, **_kw: ("test-model", "test-provider", None), ) monkeypatch.setattr(streaming, "_maybe_schedule_title_refresh", lambda *args, **kwargs: None) monkeypatch.setattr(profiles, "get_hermes_home_for_profile", home_for_profile) monkeypatch.setattr(profiles, "get_profile_runtime_env", lambda _home: {}) monkeypatch.setattr( oauth, "resolve_runtime_provider_with_anthropic_env_lock", lambda _resolver, requested=None: { "provider": requested or "test-provider", "api_key": "synthetic-key", "base_url": None, }, ) monkeypatch.setattr("api.config.get_config", lambda: {}) monkeypatch.setattr("api.config._resolve_cli_toolsets", lambda _cfg: []) monkeypatch.setattr("api.config.load_settings", lambda: {}) monkeypatch.setitem(sys.modules, "hermes_cli", fake_hermes_cli) monkeypatch.setitem(sys.modules, "hermes_cli.runtime_provider", fake_runtime_module) monkeypatch.setitem(sys.modules, "hermes_state", fake_hermes_state) with cfg.SESSION_AGENT_CACHE_LOCK: cfg.SESSION_AGENT_CACHE.clear() streaming.STREAMS.clear() streaming.CANCEL_FLAGS.clear() streaming.AGENT_INSTANCES.clear() streaming.STREAM_PARTIAL_TEXT.clear() streaming.STREAM_REASONING_TEXT.clear() streaming.STREAM_LIVE_TOOL_CALLS.clear() def run_turn(profile_name: str, stream_id: str, text: str): fake_session.profile = profile_name fake_session.active_stream_id = stream_id streaming.STREAMS[stream_id] = queue.Queue() streaming._run_agent_streaming( session_id=fake_session.session_id, msg_text=text, model="test-model", model_provider="test-provider", workspace=str(tmp_path), stream_id=stream_id, ) run_turn("alpha", "issue1897-stream-1", "first turn") run_turn("alpha", "issue1897-stream-2", "same profile second turn") assert len(constructed_agents) == 1, "same-profile turns should reuse the cached agent" run_turn("beta", "issue1897-stream-3", "profile switched turn") assert prompts_used_for_runs == [ "PROFILE_ALPHA_SYNTHETIC_SOUL", "PROFILE_ALPHA_SYNTHETIC_SOUL", "PROFILE_BETA_SYNTHETIC_SOUL", ] assert [agent.constructed_home for agent in constructed_agents] == [ str(profile_a_home), str(profile_b_home), ] assert homes_seen_during_runs == [ str(profile_a_home), str(profile_a_home), str(profile_b_home), ] with cfg.SESSION_AGENT_CACHE_LOCK: assert cfg.SESSION_AGENT_CACHE[fake_session.session_id][0] is constructed_agents[-1] def test_cache_signature_includes_profile_home(): block = _signature_block() assert "profile_home" in block, ( "SESSION_AGENT_CACHE signature is missing `profile_home`. Without this, " "same-session profile switches reuse the cached agent built under the " "previous profile's HERMES_HOME, leaking the old SOUL.md into new turns." ) calls = _production_signature_calls() assert len(calls) == 3, ( "expected exactly three _compute_agent_cache_signature() call sites " "(initial send plus both self-heal retries), found " f"{len(calls)}" ) for region, (_offset, call) in _signature_calls_by_region(calls).items(): assert "profile_home=_profile_home" in call, ( f"the {region} signature call site must pass the resolved profile home, " "or a retry re-caches the agent under a signature that ignores the active profile:\n" + call ) def test_profile_home_resolved_before_cache_signature(): profile_home_assignment = STREAMING_PY.index("_profile_home = str(_profile_home_path)") calls = _production_signature_calls() assert len(calls) == 3, ( "expected exactly three _compute_agent_cache_signature() call sites " "(initial send plus both self-heal retries), found " f"{len(calls)}" ) for region, (offset, call) in _signature_calls_by_region(calls).items(): assert profile_home_assignment < offset, ( "`_profile_home` must be resolved before the cache signature is " f"computed, otherwise the {region} signature hashes a stale/unbound " "home." ) assert "profile_home=_profile_home" in call, call assert "max_iterations_cfg=_max_iterations_cfg" in call, call assert "max_tokens_cfg=_max_tokens_cfg" in call, call def test_signature_uses_profile_home_with_fallback(): block = _signature_block() assert "profile_home or ''" in block, ( "Signature should use `profile_home or ''` so empty-home deployments get " "a stable cache key rather than unnecessary cache churn." ) for _offset, call in _production_signature_calls(): assert "profile_home=_profile_home" in call, call