feat(streaming): publish observed local runtime model to SSE and journal

tests/test_issue5121_provider_auth_terminal_error.py

@@ -423,6 +423,72 @@ def test_auth_retry_success_does_not_append_error_turn(tmp_path, monkeypatch):
     assert not any(msg.get("_error") for msg in saved.messages)
 
 
+def test_auth_retry_success_observes_replacement_agent_runtime(tmp_path, monkeypatch):
+    from api import run_journal
+
+    session = _prepare_session("auth_runtime", "stream_auth_runtime", pending_user_message="Please retry")
+    Base = _build_auth_failure_agent(token_text="")
+
+    class ReplacementAgent(Base):
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+            self.model = "original" if self.__class__.runs == 0 else "healed-model"
+            self.provider = "old-provider" if self.__class__.runs == 0 else "healed-provider"
+
+    monkeypatch.setattr(run_journal, "_default_session_dir", lambda: tmp_path / "sessions")
+    q = queue.Queue()
+    streaming.STREAMS["stream_auth_runtime"] = q
+    config.STREAM_PARTIAL_TEXT["stream_auth_runtime"] = ""
+    heal_rt = {"provider": "test-provider", "api_key": "fresh-key", "base_url": None}
+    with mock.patch.object(streaming, "get_session", return_value=session), \
+         mock.patch.object(streaming, "_get_ai_agent", return_value=ReplacementAgent), \
+         mock.patch.object(streaming, "resolve_model_provider", return_value=("test-model", "test-provider", None)), \
+         mock.patch("api.config.get_config", return_value={}), \
+         mock.patch("api.config._resolve_cli_toolsets", return_value=[]), \
+         mock.patch.object(streaming, "_attempt_credential_self_heal", return_value=heal_rt):
+        streaming._run_agent_streaming(session.session_id, "Please retry", "test-model", str(tmp_path), "stream_auth_runtime")
+
+    events = _queue_events(q)
+    assert any(name == "done" for name, _ in events)
+    assert [(item["model"], item["provider"]) for name, item in events if name == "runtime_model"] == [
+        ("healed-model", "healed-provider")]
+    summary = run_journal.latest_run_summary("auth_runtime", "stream_auth_runtime", session_dir=tmp_path / "sessions")
+    assert summary["runtime_model"]["model"] == "healed-model"
+
+
+def test_auth_exception_retry_observes_replacement_agent(tmp_path, monkeypatch):
+    session = _prepare_session("auth_exception_runtime", "stream_auth_exception_runtime", pending_user_message="Retry")
+
+    class RetryAgent(MockAgent):
+        runs = 0
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+            self.model = "original" if type(self).runs == 0 else "retry-model"
+            self.provider = "original-provider" if type(self).runs == 0 else "retry-provider"
+        def run_conversation(self, **kwargs):
+            type(self).runs += 1
+            if type(self).runs == 1:
+                raise RuntimeError("401 unauthorized")
+            return {"messages": list(kwargs.get("conversation_history") or []) + [
+                {"role": "assistant", "content": "Recovered answer"}]}
+
+    q = queue.Queue()
+    streaming.STREAMS["stream_auth_exception_runtime"] = q
+    config.STREAM_PARTIAL_TEXT["stream_auth_exception_runtime"] = ""
+    heal_rt = {"provider": "test-provider", "api_key": "fresh-key", "base_url": None}
+    with mock.patch.object(streaming, "get_session", return_value=session), \
+         mock.patch.object(streaming, "_get_ai_agent", return_value=RetryAgent), \
+         mock.patch.object(streaming, "resolve_model_provider", return_value=("test-model", "test-provider", None)), \
+         mock.patch("api.config.get_config", return_value={}), \
+         mock.patch("api.config._resolve_cli_toolsets", return_value=[]), \
+         mock.patch.object(streaming, "_attempt_credential_self_heal", return_value=heal_rt):
+        streaming._run_agent_streaming(session.session_id, "Retry", "test-model", str(tmp_path), "stream_auth_exception_runtime")
+    events = _queue_events(q)
+    assert any(name == "done" for name, _ in events)
+    assert [(item["model"], item["provider"]) for name, item in events if name == "runtime_model"] == [
+        ("retry-model", "retry-provider")]
+
+
 def test_auth_exception_retry_structured_failure_still_emits_error(
     tmp_path,
     monkeypatch,

tests/test_issue6068_used_model_footer.py

@@ -140,10 +140,11 @@ console.log(JSON.stringify({{
 
 def test_streaming_stamps_used_model_on_assistant_message_and_usage_payload():
     assert "_dm['_usedModel'] = _used_model" in STREAMING_PY
-    # The served model must be read from the agent AFTER the run — the agent
-    # mutates agent.model when a fallback fires, so the pre-run resolved_model
-    # would mis-attribute fallback turns.
-    assert "_used_model = getattr(agent, 'model', None) or resolved_model or model" in STREAMING_PY
+    # Read the served model from the agent after the run (including fallback),
+    # without treating a requested/configured selection as proof of service.
+    assert "_observed_model = getattr(agent, 'model', None)" in STREAMING_PY
+    assert "_used_model = _observed_model.strip() if isinstance(_observed_model, str) else None" in STREAMING_PY
+    assert "_used_model = getattr(agent, 'model', None) or resolved_model or model" not in STREAMING_PY
     assert "usage['used_model'] = _used_model" in STREAMING_PY
 
 

tests/test_runtime_model_observation.py

@@ -0,0 +1,183 @@
+"""Observed Agent runtime, durable replay and HTTP reconnect projection."""
+import queue
+import sys
+import types
+from unittest import mock
+
+import pytest
+
+from api.run_journal import RunJournalWriter, latest_run_summary, read_run_events
+
+
+def observation(model="backup", *, session_id="session-a", stream_id="run-a"):
+    return {"session_id": session_id, "stream_id": stream_id, "model": model,
+            "provider": "custom", "fallback_active": True, "phase": "observed_output"}
+
+
+def test_journal_runtime_is_owned_and_invalidated(tmp_path):
+    writer = RunJournalWriter("session-a", "run-a", session_dir=tmp_path)
+    writer.append_sse_event("token", {"text": "answer"})
+    first = writer.append_sse_event("runtime_model", observation())
+    assert read_run_events("session-a", "run-a", session_dir=tmp_path)["events"][-1] == first
+    summary = latest_run_summary("session-a", "run-a", session_dir=tmp_path)
+    assert summary["runtime_model"] == observation()
+    summary["runtime_model"]["model"] = "corrupted"
+    assert latest_run_summary("session-a", "run-a", session_dir=tmp_path)["runtime_model"] == observation()
+    writer.append_sse_event("warning", {"type": "fallback", "message": "Trying another route"})
+    assert latest_run_summary("session-a", "run-a", session_dir=tmp_path)["runtime_model"] is None
+    writer.append_sse_event("runtime_model", {**observation(), "stream_id": "foreign"})
+    assert latest_run_summary("session-a", "run-a", session_dir=tmp_path)["runtime_model"] is None
+    writer.append_sse_event("runtime_model", observation("recovered"))
+    assert latest_run_summary("session-a", "run-a", session_dir=tmp_path)["runtime_model"]["model"] == "recovered"
+    writer.append_sse_event("runtime_model", "malformed observation")
+    assert latest_run_summary("session-a", "run-a", session_dir=tmp_path)["runtime_model"] is None
+    writer.append_sse_event("runtime_model", observation("recovered-again"))
+    assert latest_run_summary("session-a", "run-a", session_dir=tmp_path)["runtime_model"]["model"] == "recovered-again"
+    assert latest_run_summary("session-a", "another-run", session_dir=tmp_path)["runtime_model"] is None
+
+
+@pytest.mark.parametrize("suffix,expected", [("", "backup"), ("warning", None), ("foreign", None)])
+def test_http_snapshot_projects_exact_journal_window(tmp_path, monkeypatch, suffix, expected):
+    import api.routes as routes
+    writer = RunJournalWriter("session-a", "run-a", session_dir=tmp_path)
+    writer.append_sse_event("runtime_model", observation())
+    writer.append_sse_event("token", {"text": "answer"})
+    if suffix == "warning":
+        writer.append_sse_event("warning", {"type": "fallback"})
+    elif suffix == "foreign":
+        writer.append_sse_event("runtime_model", {**observation(), "session_id": "foreign"})
+    monkeypatch.setattr(routes, "find_run_summary", lambda rid: latest_run_summary("session-a", rid, session_dir=tmp_path))
+    monkeypatch.setattr(routes, "read_run_events", lambda sid, rid: read_run_events(sid, rid, session_dir=tmp_path))
+    snapshot = routes._run_journal_live_snapshot("run-a")
+    assert snapshot["last_assistant_text"] == "answer"
+    value = snapshot["runtime_model"]
+    assert (value["model"] if value else None) == expected
+    assert routes._runtime_journal_snapshot_for_session_payload(snapshot)["runtime_model"] == value
+
+
+def run_worker(steps, *, fail=False, ephemeral=False, tmp_path=None, monkeypatch=None):
+    """Drive the production local worker; fake only external Agent/provider calls."""
+    from api import streaming, run_journal
+    from api.config import SESSION_AGENT_CACHE
+    SESSION_AGENT_CACHE.clear()
+    monkeypatch.setattr(run_journal, "_default_session_dir", lambda: tmp_path)
+    class Session:
+        session_id = "session-a"
+        title = "Session"
+        workspace = "/tmp"
+        model = "primary"
+        model_provider = "anthropic"
+        profile = None
+        personality = None
+        messages = [{"role": "user", "content": "old"}, {"role": "assistant", "content": "old answer"}]
+        context_messages = messages
+        input_tokens = output_tokens = cache_read_tokens = cache_write_tokens = 0
+        estimated_cost = 0.0
+        tool_calls = []
+        gateway_routing = None
+        gateway_routing_history = []
+        active_stream_id = "run-a"
+        pending_user_message = None
+        pending_attachments = []
+        pending_started_at = None
+        context_length = threshold_tokens = last_prompt_tokens = 0
+        llm_title_generated = True
+        def save(self, *args, **kwargs): pass
+        def compact(self): return {"session_id": self.session_id}
+
+    class Agent:
+        def __init__(self, model=None, provider=None, session_id=None,
+                     stream_delta_callback=None, reasoning_callback=None,
+                     status_callback=None, **kwargs):
+            self.model, self.provider, self.session_id = model, provider, session_id
+            self.stream_delta_callback = stream_delta_callback
+            self.reasoning_callback = reasoning_callback
+            self.status_callback = status_callback
+            self._provider_fallback_active = False
+            self.context_compressor = None
+            self.session_prompt_tokens = self.session_completion_tokens = 0
+            self.session_estimated_cost_usd = None
+            self.session_cache_read_tokens = self.session_cache_write_tokens = 0
+            self.reasoning_config = self.ephemeral_system_prompt = self._last_error = None
+        def run_conversation(self, **kwargs):
+            for model, provider, fallback, channel, text in steps:
+                self.model, self.provider, self._provider_fallback_active = model, provider, fallback
+                if channel == "token": self.stream_delta_callback(text)
+                elif channel == "reasoning": self.reasoning_callback(text)
+                elif channel == "status": self.status_callback("lifecycle", text)
+            if fail: return {"error": "provider unavailable", "messages": []}
+            return {"messages": kwargs.get("conversation_history", []) + [
+                {"role": "user", "content": kwargs["persist_user_message"]},
+                {"role": "assistant", "content": "served answer"}]}
+        def interrupt(self, message): pass
+
+    runtime_module = types.ModuleType("hermes_cli.runtime_provider")
+    runtime_module.resolve_runtime_provider = mock.Mock(return_value={
+        "provider": "anthropic", "base_url": None, "api_key": "sk-test",
+        "api_mode": "chat_completions", "command": None, "args": [], "credential_pool": None})
+    cli_module = types.ModuleType("hermes_cli")
+    cli_module.runtime_provider = runtime_module
+    state_module = types.ModuleType("hermes_state")
+    state_module.SessionDB = mock.Mock(return_value=None)
+    injected = {"hermes_cli": cli_module, "hermes_cli.runtime_provider": runtime_module,
+                "hermes_state": state_module}
+    sentinel = object()
+    saved = {name: sys.modules.get(name, sentinel) for name in injected}
+    sys.modules.update(injected)
+    q = queue.Queue()
+    s = Session()
+    try:
+        with mock.patch.object(streaming, "get_session", return_value=s), \
+             mock.patch.object(streaming, "_get_ai_agent", return_value=Agent), \
+             mock.patch.object(streaming, "resolve_model_provider", return_value=("primary", "anthropic", None)), \
+             mock.patch("api.config.get_config", return_value={}), \
+             mock.patch("api.config._resolve_cli_toolsets", return_value=[]):
+            streaming.STREAMS["run-a"] = q
+            streaming._run_agent_streaming("session-a", "new turn", "primary", "/tmp", "run-a", ephemeral=ephemeral)
+    finally:
+        for name, original in saved.items():
+            if original is sentinel: sys.modules.pop(name, None)
+            else: sys.modules[name] = original
+        streaming.STREAMS.pop("run-a", None)
+        SESSION_AGENT_CACHE.clear()
+    return s, list(q.queue)
+
+
+def test_worker_observes_serving_runtime_not_requested(tmp_path, monkeypatch):
+    s, events = run_worker([
+        ("primary", "anthropic", False, "reasoning", "thinking"),
+        ("backup", "custom", True, "token", "answer"),
+        ("backup", "custom", True, "status", "Switched to fallback: backup (custom)"),
+    ], tmp_path=tmp_path, monkeypatch=monkeypatch)
+    names = [name for name, _ in events]
+    assert names.count("runtime_model") == 3  # primary, backup, post-warning correction
+    observed = [value for name, value in events if name == "runtime_model"]
+    assert [value["model"] for value in observed] == ["primary", "backup", "backup"]
+    assert observed[-1] == observation()
+    assert names.index("runtime_model") < names.index("reasoning")
+    assert names.index("warning") < len(names) - 1 - names[::-1].index("runtime_model")
+    assert latest_run_summary("session-a", "run-a", session_dir=tmp_path)["runtime_model"] == observation()
+    assert s.model == "primary"  # selection is not overwritten
+
+
+def test_worker_does_not_observe_attempt_or_missing_model(tmp_path, monkeypatch):
+    _, events = run_worker([
+        ("backup", "custom", True, "status", "Trying fallback backup"),
+        (None, "custom", True, "token", "partial"),
+    ], fail=True, tmp_path=tmp_path, monkeypatch=monkeypatch)
+    assert not any(name == "runtime_model" for name, _ in events)
+
+
+def test_missing_agent_model_does_not_stamp_requested_as_served(tmp_path, monkeypatch):
+    s, events = run_worker([(None, "custom", True, "token", "partial")],
+                           tmp_path=tmp_path, monkeypatch=monkeypatch)
+    assert not any(name == "runtime_model" for name, _ in events)
+    assert "_usedModel" not in s.messages[-1]
+    usage = next(value["usage"] for name, value in events if name == "done")
+    assert not usage.get("used_model")
+
+
+def test_worker_nonstreaming_success_observes_fallback(tmp_path, monkeypatch):
+    _, events = run_worker([("backup", "custom", True, "route", "")],
+                           tmp_path=tmp_path, monkeypatch=monkeypatch, ephemeral=True)
+    assert [value["model"] for name, value in events if name == "runtime_model"] == ["backup"]