416 lines
17 KiB
Python
416 lines
17 KiB
Python
"""Regression coverage for #1897 — same-session profile switch identity bleed."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import queue
|
|
import sys
|
|
import types
|
|
from pathlib import Path
|
|
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8")
|
|
|
|
|
|
def _compute_agent_cache_signature_source() -> str:
|
|
"""Return the source of the `_compute_agent_cache_signature()` helper.
|
|
|
|
The signature blob used to be inlined in the streaming send path; it now
|
|
lives in this helper so the initial send and both self-heal retry paths
|
|
derive the signature from the same final runtime bundle.
|
|
"""
|
|
start = STREAMING_PY.index("def _compute_agent_cache_signature(")
|
|
end = STREAMING_PY.index("\ndef ", start)
|
|
return STREAMING_PY[start:end]
|
|
|
|
|
|
def _signature_block() -> str:
|
|
"""Return the `_json.dumps([...])` field list the signature hashes."""
|
|
helper = _compute_agent_cache_signature_source()
|
|
sig_start = helper.index("_sig_blob = _json.dumps")
|
|
sig_end = helper.index("], sort_keys=True)", sig_start)
|
|
return helper[sig_start:sig_end]
|
|
|
|
|
|
def _production_signature_calls() -> list[tuple[int, str]]:
|
|
"""Return `(offset, source)` for every production signature call site.
|
|
|
|
Paren-balanced so the whole multi-line keyword-argument list is captured,
|
|
and every live call site is returned so a retry path cannot silently drop a
|
|
field that the initial send still passes. Commented-out occurrences are
|
|
skipped -- a disabled retry call must read as missing, not as present.
|
|
"""
|
|
marker = "_agent_sig = _compute_agent_cache_signature("
|
|
calls: list[tuple[int, str]] = []
|
|
pos = STREAMING_PY.find(marker)
|
|
while pos != -1:
|
|
line_start = STREAMING_PY.rfind("\n", 0, pos) + 1
|
|
if STREAMING_PY[line_start:pos].strip():
|
|
# Commented-out (or otherwise non-statement) occurrence: it no
|
|
# longer runs, so it must not count as a live call site.
|
|
pos = STREAMING_PY.find(marker, pos + 1)
|
|
continue
|
|
depth = 0
|
|
for idx in range(pos + len(marker) - 1, len(STREAMING_PY)):
|
|
char = STREAMING_PY[idx]
|
|
if char == "(":
|
|
depth += 1
|
|
elif char == ")":
|
|
depth -= 1
|
|
if depth == 0:
|
|
calls.append((pos, STREAMING_PY[pos:idx + 1]))
|
|
break
|
|
else:
|
|
raise AssertionError("unterminated _compute_agent_cache_signature( call")
|
|
pos = STREAMING_PY.find(marker, pos + 1)
|
|
return calls
|
|
|
|
|
|
# Lifecycle anchors for the streaming send path. Pinning one signature call to
|
|
# each region keeps the oracle honest: a retry call that silently disappears
|
|
# fails the region check instead of hiding behind the calls that remain.
|
|
INITIAL_SEND_MARKER = "# ── Agent cache: reuse across messages in the same session ──"
|
|
RETURNED_ERROR_SELF_HEAL_MARKER = (
|
|
"logger.info('[webui] self-heal: retrying stream after credential refresh')"
|
|
)
|
|
RAISED_EXCEPTION_SELF_HEAL_MARKER = (
|
|
"logger.info('[webui] self-heal (except path): "
|
|
"retrying stream after credential refresh')"
|
|
)
|
|
|
|
# Each region ends at the atomic registration/publication call that consumes
|
|
# the signature it just computed. The shared helper now writes the cache under
|
|
# the Stop lock; the call boundary must still follow fresh signature computation.
|
|
# A signature call moved below its publication lands outside the region and
|
|
# fails, rather than silently reusing a stale `_agent_sig` on a retry.
|
|
INITIAL_SEND_CACHE_WRITE_MARKER = (
|
|
"if not _register_agent_if_current(agent, _agent_sig if _cache_new_agent else None):"
|
|
)
|
|
RETURNED_ERROR_SELF_HEAL_CACHE_WRITE_MARKER = (
|
|
"if not _register_agent_if_current(agent, _agent_sig):"
|
|
)
|
|
RAISED_EXCEPTION_SELF_HEAL_CACHE_WRITE_MARKER = (
|
|
"if not _register_agent_if_current(_heal_agent, _agent_sig):"
|
|
)
|
|
|
|
|
|
def _marker_offset(marker: str) -> int:
|
|
"""Return the single offset of a lifecycle marker in streaming.py."""
|
|
offset = STREAMING_PY.index(marker)
|
|
assert STREAMING_PY.find(marker, offset + 1) == -1, (
|
|
"lifecycle marker is no longer unique in streaming.py:\n" + marker
|
|
)
|
|
return offset
|
|
|
|
|
|
def _signature_calls_by_region(
|
|
calls: list[tuple[int, str]],
|
|
) -> dict[str, tuple[int, str]]:
|
|
"""Map each lifecycle region to the single signature call inside it.
|
|
|
|
The regions are the initial send, the returned-error self-heal retry and
|
|
the raised-exception self-heal retry; each runs from its lifecycle anchor
|
|
to the atomic publication call that consumes `_agent_sig`. Requiring one call
|
|
inside those bounds means neither a dropped retry call (masked by its
|
|
surviving siblings) nor a call recomputed after its own cache write (which
|
|
would leave the write storing a stale signature) can read as correct.
|
|
"""
|
|
initial = _marker_offset(INITIAL_SEND_MARKER)
|
|
initial_cache_write = _marker_offset(INITIAL_SEND_CACHE_WRITE_MARKER)
|
|
returned_error = _marker_offset(RETURNED_ERROR_SELF_HEAL_MARKER)
|
|
returned_error_cache_write = _marker_offset(
|
|
RETURNED_ERROR_SELF_HEAL_CACHE_WRITE_MARKER
|
|
)
|
|
raised_exception = _marker_offset(RAISED_EXCEPTION_SELF_HEAL_MARKER)
|
|
raised_exception_cache_write = _marker_offset(
|
|
RAISED_EXCEPTION_SELF_HEAL_CACHE_WRITE_MARKER
|
|
)
|
|
assert (
|
|
initial
|
|
< initial_cache_write
|
|
< returned_error
|
|
< returned_error_cache_write
|
|
< raised_exception
|
|
< raised_exception_cache_write
|
|
), (
|
|
"streaming.py lifecycle regions are out of order; these anchors no "
|
|
"longer describe the send path"
|
|
)
|
|
|
|
bounds = {
|
|
"initial send": (initial, initial_cache_write),
|
|
"returned-error self-heal": (returned_error, returned_error_cache_write),
|
|
"raised-exception self-heal": (
|
|
raised_exception,
|
|
raised_exception_cache_write,
|
|
),
|
|
}
|
|
by_region: dict[str, tuple[int, str]] = {}
|
|
for region, (start, end) in bounds.items():
|
|
found = [(offset, call) for offset, call in calls if start < offset < end]
|
|
assert len(found) == 1, (
|
|
"expected exactly one _compute_agent_cache_signature() call in the "
|
|
f"{region} region of streaming.py -- between its lifecycle anchor "
|
|
f"and the cache write that stores `_agent_sig` -- found "
|
|
f"{len(found)}"
|
|
)
|
|
by_region[region] = found[0]
|
|
return by_region
|
|
|
|
|
|
def test_same_session_profile_switch_rebuilds_agent_under_new_soul_home(tmp_path, monkeypatch):
|
|
"""Switching profiles in one WebUI session must not reuse old SOUL.md.
|
|
|
|
The fake AIAgent mirrors the real failure mode: it reads SOUL.md from
|
|
HERMES_HOME at construction time and keeps that value in a cached system
|
|
prompt. Two consecutive turns on the same profile should reuse the agent;
|
|
changing only ``session.profile`` should create a fresh agent whose cached
|
|
prompt comes from the new synthetic profile home.
|
|
"""
|
|
sys.path.insert(0, str(REPO))
|
|
from api import config as cfg
|
|
from api import oauth
|
|
from api import profiles
|
|
from api import streaming
|
|
|
|
default_home = tmp_path / "hermes-home"
|
|
profile_a_home = default_home / "profiles" / "alpha"
|
|
profile_b_home = default_home / "profiles" / "beta"
|
|
profile_a_home.mkdir(parents=True)
|
|
profile_b_home.mkdir(parents=True)
|
|
(profile_a_home / "SOUL.md").write_text(
|
|
"PROFILE_ALPHA_SYNTHETIC_SOUL",
|
|
encoding="utf-8",
|
|
)
|
|
(profile_b_home / "SOUL.md").write_text(
|
|
"PROFILE_BETA_SYNTHETIC_SOUL",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
class FakeSession:
|
|
def __init__(self):
|
|
self.session_id = "issue1897-same-session"
|
|
self.title = "Pinned test title"
|
|
self.workspace = str(tmp_path)
|
|
self.model = "test-model"
|
|
self.model_provider = None
|
|
self.profile = "alpha"
|
|
self.personality = None
|
|
self.messages = []
|
|
self.context_messages = []
|
|
self.tool_calls = []
|
|
self.input_tokens = 0
|
|
self.output_tokens = 0
|
|
self.estimated_cost = None
|
|
self.context_length = 0
|
|
self.threshold_tokens = 0
|
|
self.last_prompt_tokens = 0
|
|
self.active_stream_id = None
|
|
self.pending_user_message = None
|
|
self.pending_attachments = []
|
|
self.pending_started_at = None
|
|
self.llm_title_generated = True
|
|
|
|
def save(self, *args, **kwargs):
|
|
return None
|
|
|
|
def compact(self):
|
|
return {
|
|
"session_id": self.session_id,
|
|
"title": self.title,
|
|
"workspace": self.workspace,
|
|
"model": self.model,
|
|
"created_at": 0,
|
|
"updated_at": 0,
|
|
"pinned": False,
|
|
"archived": False,
|
|
"project_id": None,
|
|
"profile": self.profile,
|
|
"input_tokens": self.input_tokens,
|
|
"output_tokens": self.output_tokens,
|
|
"estimated_cost": self.estimated_cost,
|
|
"personality": self.personality,
|
|
}
|
|
|
|
constructed_agents = []
|
|
prompts_used_for_runs = []
|
|
homes_seen_during_runs = []
|
|
|
|
class SoulCachingAgent:
|
|
def __init__(self, **kwargs):
|
|
self.session_id = kwargs.get("session_id")
|
|
self.model = kwargs.get("model")
|
|
self.provider = kwargs.get("provider")
|
|
self.base_url = kwargs.get("base_url")
|
|
self.context_compressor = None
|
|
self.session_prompt_tokens = 0
|
|
self.session_completion_tokens = 0
|
|
self.session_estimated_cost_usd = None
|
|
self.ephemeral_system_prompt = None
|
|
self._last_error = None
|
|
self.stream_delta_callback = kwargs.get("stream_delta_callback")
|
|
self.tool_progress_callback = kwargs.get("tool_progress_callback")
|
|
self.reasoning_callback = kwargs.get("reasoning_callback")
|
|
self.clarify_callback = kwargs.get("clarify_callback")
|
|
home = Path(os.environ["HERMES_HOME"])
|
|
self.constructed_home = str(home)
|
|
self._cached_system_prompt = (home / "SOUL.md").read_text(encoding="utf-8")
|
|
constructed_agents.append(self)
|
|
|
|
def run_conversation(self, **kwargs):
|
|
prompts_used_for_runs.append(self._cached_system_prompt)
|
|
homes_seen_during_runs.append(os.environ.get("HERMES_HOME"))
|
|
history = list(kwargs.get("conversation_history") or [])
|
|
return {
|
|
"messages": history
|
|
+ [
|
|
{"role": "user", "content": kwargs.get("persist_user_message", "")},
|
|
{
|
|
"role": "assistant",
|
|
"content": f"reply from {self._cached_system_prompt}",
|
|
},
|
|
]
|
|
}
|
|
|
|
def interrupt(self, _message):
|
|
return None
|
|
|
|
fake_session = FakeSession()
|
|
fake_runtime_module = types.ModuleType("hermes_cli.runtime_provider")
|
|
fake_runtime_module.resolve_runtime_provider = lambda requested=None: {
|
|
"provider": requested or "test-provider",
|
|
"api_key": "synthetic-key",
|
|
"base_url": None,
|
|
}
|
|
fake_hermes_cli = types.ModuleType("hermes_cli")
|
|
fake_hermes_cli.runtime_provider = fake_runtime_module
|
|
fake_hermes_state = types.ModuleType("hermes_state")
|
|
fake_hermes_state.SessionDB = lambda: None
|
|
|
|
def home_for_profile(profile_name):
|
|
return {"alpha": profile_a_home, "beta": profile_b_home}[profile_name]
|
|
|
|
monkeypatch.setattr(streaming, "get_session", lambda _sid: fake_session)
|
|
monkeypatch.setattr(streaming, "_get_ai_agent", lambda: SoulCachingAgent)
|
|
monkeypatch.setattr(
|
|
streaming,
|
|
"resolve_model_provider",
|
|
lambda _model, **_kw: ("test-model", "test-provider", None),
|
|
)
|
|
monkeypatch.setattr(streaming, "_maybe_schedule_title_refresh", lambda *args, **kwargs: None)
|
|
monkeypatch.setattr(profiles, "get_hermes_home_for_profile", home_for_profile)
|
|
monkeypatch.setattr(profiles, "get_profile_runtime_env", lambda _home: {})
|
|
monkeypatch.setattr(
|
|
oauth,
|
|
"resolve_runtime_provider_with_anthropic_env_lock",
|
|
lambda _resolver, requested=None: {
|
|
"provider": requested or "test-provider",
|
|
"api_key": "synthetic-key",
|
|
"base_url": None,
|
|
},
|
|
)
|
|
monkeypatch.setattr("api.config.get_config", lambda: {})
|
|
monkeypatch.setattr("api.config._resolve_cli_toolsets", lambda _cfg: [])
|
|
monkeypatch.setattr("api.config.load_settings", lambda: {})
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", fake_hermes_cli)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli.runtime_provider", fake_runtime_module)
|
|
monkeypatch.setitem(sys.modules, "hermes_state", fake_hermes_state)
|
|
|
|
with cfg.SESSION_AGENT_CACHE_LOCK:
|
|
cfg.SESSION_AGENT_CACHE.clear()
|
|
streaming.STREAMS.clear()
|
|
streaming.CANCEL_FLAGS.clear()
|
|
streaming.AGENT_INSTANCES.clear()
|
|
streaming.STREAM_PARTIAL_TEXT.clear()
|
|
streaming.STREAM_REASONING_TEXT.clear()
|
|
streaming.STREAM_LIVE_TOOL_CALLS.clear()
|
|
|
|
def run_turn(profile_name: str, stream_id: str, text: str):
|
|
fake_session.profile = profile_name
|
|
fake_session.active_stream_id = stream_id
|
|
streaming.STREAMS[stream_id] = queue.Queue()
|
|
streaming._run_agent_streaming(
|
|
session_id=fake_session.session_id,
|
|
msg_text=text,
|
|
model="test-model",
|
|
model_provider="test-provider",
|
|
workspace=str(tmp_path),
|
|
stream_id=stream_id,
|
|
)
|
|
|
|
run_turn("alpha", "issue1897-stream-1", "first turn")
|
|
run_turn("alpha", "issue1897-stream-2", "same profile second turn")
|
|
assert len(constructed_agents) == 1, "same-profile turns should reuse the cached agent"
|
|
|
|
run_turn("beta", "issue1897-stream-3", "profile switched turn")
|
|
|
|
assert prompts_used_for_runs == [
|
|
"PROFILE_ALPHA_SYNTHETIC_SOUL",
|
|
"PROFILE_ALPHA_SYNTHETIC_SOUL",
|
|
"PROFILE_BETA_SYNTHETIC_SOUL",
|
|
]
|
|
assert [agent.constructed_home for agent in constructed_agents] == [
|
|
str(profile_a_home),
|
|
str(profile_b_home),
|
|
]
|
|
assert homes_seen_during_runs == [
|
|
str(profile_a_home),
|
|
str(profile_a_home),
|
|
str(profile_b_home),
|
|
]
|
|
with cfg.SESSION_AGENT_CACHE_LOCK:
|
|
assert cfg.SESSION_AGENT_CACHE[fake_session.session_id][0] is constructed_agents[-1]
|
|
|
|
|
|
def test_cache_signature_includes_profile_home():
|
|
block = _signature_block()
|
|
assert "profile_home" in block, (
|
|
"SESSION_AGENT_CACHE signature is missing `profile_home`. Without this, "
|
|
"same-session profile switches reuse the cached agent built under the "
|
|
"previous profile's HERMES_HOME, leaking the old SOUL.md into new turns."
|
|
)
|
|
|
|
calls = _production_signature_calls()
|
|
assert len(calls) == 3, (
|
|
"expected exactly three _compute_agent_cache_signature() call sites "
|
|
"(initial send plus both self-heal retries), found "
|
|
f"{len(calls)}"
|
|
)
|
|
for region, (_offset, call) in _signature_calls_by_region(calls).items():
|
|
assert "profile_home=_profile_home" in call, (
|
|
f"the {region} signature call site must pass the resolved profile home, "
|
|
"or a retry re-caches the agent under a signature that ignores the active profile:\n" + call
|
|
)
|
|
|
|
|
|
def test_profile_home_resolved_before_cache_signature():
|
|
profile_home_assignment = STREAMING_PY.index("_profile_home = str(_profile_home_path)")
|
|
|
|
calls = _production_signature_calls()
|
|
assert len(calls) == 3, (
|
|
"expected exactly three _compute_agent_cache_signature() call sites "
|
|
"(initial send plus both self-heal retries), found "
|
|
f"{len(calls)}"
|
|
)
|
|
for region, (offset, call) in _signature_calls_by_region(calls).items():
|
|
assert profile_home_assignment < offset, (
|
|
"`_profile_home` must be resolved before the cache signature is "
|
|
f"computed, otherwise the {region} signature hashes a stale/unbound "
|
|
"home."
|
|
)
|
|
assert "profile_home=_profile_home" in call, call
|
|
assert "max_iterations_cfg=_max_iterations_cfg" in call, call
|
|
assert "max_tokens_cfg=_max_tokens_cfg" in call, call
|
|
|
|
|
|
def test_signature_uses_profile_home_with_fallback():
|
|
block = _signature_block()
|
|
assert "profile_home or ''" in block, (
|
|
"Signature should use `profile_home or ''` so empty-home deployments get "
|
|
"a stable cache key rather than unnecessary cache churn."
|
|
)
|
|
|
|
for _offset, call in _production_signature_calls():
|
|
assert "profile_home=_profile_home" in call, call
|