Fix final answer reasoning echo
This commit is contained in:
1 parent
6b52b06954
commit
2aaad7a282
3 files changed
+218
-8
No files matched your search
@@ -134,11 +134,11 @@ def test_visible_progress_token_reasoning_and_interim_are_deduped(cleanup_test_s
|
||||
"credential_pool": None,
|
||||
}
|
||||
runtime_payload["api_" + "key"] = "***"
|
||||
fake_runtime_module.resolve_runtime_provider = mock.Mock(return_value=runtime_payload)
|
||||
fake_runtime_module.__dict__["resolve_runtime_provider"] = mock.Mock(return_value=runtime_payload)
|
||||
fake_hermes_cli = types.ModuleType("hermes_cli")
|
||||
fake_hermes_cli.runtime_provider = fake_runtime_module
|
||||
fake_hermes_cli.__dict__["runtime_provider"] = fake_runtime_module
|
||||
fake_hermes_state = types.ModuleType("hermes_state")
|
||||
fake_hermes_state.SessionDB = mock.Mock(return_value=None)
|
||||
fake_hermes_state.__dict__["SessionDB"] = mock.Mock(return_value=None)
|
||||
injected = {
|
||||
"hermes_cli": fake_hermes_cli,
|
||||
"hermes_cli.runtime_provider": fake_runtime_module,
|
||||
@@ -298,11 +298,11 @@ def test_reasoning_then_interim_progress_marks_reasoning_echo(cleanup_test_sessi
|
||||
"credential_pool": None,
|
||||
}
|
||||
runtime_payload["api_" + "key"] = "***"
|
||||
fake_runtime_module.resolve_runtime_provider = mock.Mock(return_value=runtime_payload)
|
||||
fake_runtime_module.__dict__["resolve_runtime_provider"] = mock.Mock(return_value=runtime_payload)
|
||||
fake_hermes_cli = types.ModuleType("hermes_cli")
|
||||
fake_hermes_cli.runtime_provider = fake_runtime_module
|
||||
fake_hermes_cli.__dict__["runtime_provider"] = fake_runtime_module
|
||||
fake_hermes_state = types.ModuleType("hermes_state")
|
||||
fake_hermes_state.SessionDB = mock.Mock(return_value=None)
|
||||
fake_hermes_state.__dict__["SessionDB"] = mock.Mock(return_value=None)
|
||||
injected = {
|
||||
"hermes_cli": fake_hermes_cli,
|
||||
"hermes_cli.runtime_provider": fake_runtime_module,
|
||||
@@ -343,3 +343,197 @@ def test_reasoning_then_interim_progress_marks_reasoning_echo(cleanup_test_sessi
|
||||
assert done_payloads, "run should settle"
|
||||
final_messages = done_payloads[-1]["session"]["messages"]
|
||||
assert not any(message.get("reasoning") == progress for message in final_messages)
|
||||
|
||||
|
||||
def test_final_answer_prefix_reasoning_echo_is_not_journaled_or_merged(cleanup_test_sessions):
|
||||
"""A final-answer prefix mirrored through reasoning must not enter Worklog.
|
||||
|
||||
The observed production failure had the final answer stream normally through
|
||||
token events, then a later reasoning event carried the first 500 characters
|
||||
of that same final answer. Since `put()` journals before queue delivery, this
|
||||
regression covers the live stream and run-journal replay boundary together;
|
||||
the done payload covers final session merge/reload state.
|
||||
"""
|
||||
import api.streaming as streaming
|
||||
from api.run_journal import read_run_events
|
||||
|
||||
final_answer = (
|
||||
"已按 Hermes WebUI workflow 在独立 worktree 完成,本地 review-ready;"
|
||||
"没有 push、没有开 PR、没有改真实 cron/config,也没有在主 checkout 实现。\n\n"
|
||||
"## 位置\n\n"
|
||||
"| 项 | 值 |\n|---|---|\n"
|
||||
"| Worktree | `/Users/xuefusong/hermes-webui-worktrees/example` |\n"
|
||||
"| Branch | `fix/example` |\n\n"
|
||||
"## 根因\n\n"
|
||||
"Final Answer 正文已经作为 assistant token 流出,不应该再作为 reasoning/Worklog 事件出现。\n\n"
|
||||
"## 验证\n\n"
|
||||
"相关 targeted regression 覆盖 live stream、run journal replay 和 final merge。"
|
||||
)
|
||||
leaked_prefix = final_answer[:500]
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self):
|
||||
self.session_id = "issue_final_answer_reasoning_echo"
|
||||
self.title = "Final echo"
|
||||
self.workspace = "/tmp"
|
||||
self.model = "gpt-test"
|
||||
self.model_provider = None
|
||||
self.profile = None
|
||||
self.personality = None
|
||||
self.messages = []
|
||||
self.context_messages = []
|
||||
self.input_tokens = 0
|
||||
self.output_tokens = 0
|
||||
self.estimated_cost = 0
|
||||
self.cache_read_tokens = 0
|
||||
self.cache_write_tokens = 0
|
||||
self.tool_calls = []
|
||||
self.gateway_routing = None
|
||||
self.gateway_routing_history = []
|
||||
self.active_stream_id = ""
|
||||
self.pending_user_message = None
|
||||
self.pending_attachments = []
|
||||
self.pending_started_at = None
|
||||
self.context_length = 0
|
||||
self.threshold_tokens = 0
|
||||
self.last_prompt_tokens = 0
|
||||
self.llm_title_generated = True
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def compact(self):
|
||||
return {
|
||||
"session_id": self.session_id,
|
||||
"title": self.title,
|
||||
"workspace": self.workspace,
|
||||
"model": self.model,
|
||||
"created_at": 0,
|
||||
"updated_at": 0,
|
||||
"pinned": False,
|
||||
"archived": False,
|
||||
"project_id": None,
|
||||
"profile": self.profile,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"estimated_cost": self.estimated_cost,
|
||||
"cache_read_tokens": self.cache_read_tokens,
|
||||
"cache_write_tokens": self.cache_write_tokens,
|
||||
"personality": self.personality,
|
||||
}
|
||||
|
||||
class FinalEchoAgent:
|
||||
def __init__(
|
||||
self,
|
||||
model=None,
|
||||
provider=None,
|
||||
base_url=None,
|
||||
platform=None,
|
||||
quiet_mode=False,
|
||||
enabled_toolsets=None,
|
||||
fallback_model=None,
|
||||
session_id=None,
|
||||
session_db=None,
|
||||
prefill_messages=None,
|
||||
stream_delta_callback=None,
|
||||
reasoning_callback=None,
|
||||
tool_progress_callback=None,
|
||||
clarify_callback=None,
|
||||
interim_assistant_callback=None,
|
||||
**_kwargs,
|
||||
):
|
||||
self.stream_delta_callback = cast(Callable[[str], None], stream_delta_callback)
|
||||
self.reasoning_callback = cast(Callable[[str], None], reasoning_callback)
|
||||
self.context_compressor = None
|
||||
self.session_prompt_tokens = 0
|
||||
self.session_completion_tokens = 0
|
||||
self.session_estimated_cost_usd = 0
|
||||
self.session_cache_read_tokens = 0
|
||||
self.session_cache_write_tokens = 0
|
||||
self.reasoning_config = None
|
||||
self.ephemeral_system_prompt = None
|
||||
self._last_error = None
|
||||
|
||||
def run_conversation(self, **kwargs):
|
||||
self.stream_delta_callback(final_answer)
|
||||
# This mirrors the production journal shape: final answer content was
|
||||
# emitted as visible tokens first, then incorrectly reported as a
|
||||
# reasoning delta near stream end.
|
||||
self.reasoning_callback(leaked_prefix)
|
||||
history = kwargs.get("conversation_history", [])
|
||||
return {"messages": history + [
|
||||
{"role": "user", "content": kwargs["persist_user_message"]},
|
||||
{"role": "assistant", "content": final_answer},
|
||||
]}
|
||||
|
||||
def interrupt(self, _message):
|
||||
pass
|
||||
|
||||
fake_session = FakeSession()
|
||||
fake_stream_id = "stream_issue_final_answer_reasoning_echo"
|
||||
fake_session.active_stream_id = fake_stream_id
|
||||
fake_queue = queue.Queue()
|
||||
fake_runtime_module = types.ModuleType("hermes_cli.runtime_provider")
|
||||
runtime_payload = {
|
||||
"provider": "openai",
|
||||
"base_url": None,
|
||||
"api_mode": "chat_completions",
|
||||
"command": None,
|
||||
"args": [],
|
||||
"credential_pool": None,
|
||||
}
|
||||
runtime_payload["api_" + "key"] = "***"
|
||||
fake_runtime_module.__dict__["resolve_runtime_provider"] = mock.Mock(return_value=runtime_payload)
|
||||
fake_hermes_cli = types.ModuleType("hermes_cli")
|
||||
fake_hermes_cli.__dict__["runtime_provider"] = fake_runtime_module
|
||||
fake_hermes_state = types.ModuleType("hermes_state")
|
||||
fake_hermes_state.__dict__["SessionDB"] = mock.Mock(return_value=None)
|
||||
injected = {
|
||||
"hermes_cli": fake_hermes_cli,
|
||||
"hermes_cli.runtime_provider": fake_runtime_module,
|
||||
"hermes_state": fake_hermes_state,
|
||||
}
|
||||
saved = {k: sys.modules.get(k, _MISSING) for k in injected}
|
||||
sys.modules.update(injected)
|
||||
try:
|
||||
with mock.patch.object(streaming, "get_session", return_value=fake_session), \
|
||||
mock.patch.object(streaming, "_get_ai_agent", return_value=FinalEchoAgent), \
|
||||
mock.patch.object(streaming, "resolve_model_provider", return_value=("gpt-test", "openai", None)), \
|
||||
mock.patch("api.config.get_config", return_value={}), \
|
||||
mock.patch("api.config._resolve_cli_toolsets", return_value=[]):
|
||||
streaming.STREAMS[fake_stream_id] = fake_queue
|
||||
streaming._run_agent_streaming(
|
||||
session_id=fake_session.session_id,
|
||||
msg_text="ship it",
|
||||
model="gpt-test",
|
||||
workspace="/tmp",
|
||||
stream_id=fake_stream_id,
|
||||
)
|
||||
finally:
|
||||
streaming.STREAMS.pop(fake_stream_id, None)
|
||||
for k, prev in saved.items():
|
||||
if prev is _MISSING:
|
||||
sys.modules.pop(k, None)
|
||||
else:
|
||||
sys.modules[k] = cast(types.ModuleType, prev)
|
||||
|
||||
events = list(fake_queue.queue)
|
||||
assert [(event, payload) for event, payload in events if event == "token"] == [
|
||||
("token", {"text": final_answer})
|
||||
]
|
||||
assert not [payload for event, payload in events if event == "reasoning" and payload.get("text") == leaked_prefix]
|
||||
|
||||
journal_events = read_run_events(fake_session.session_id, fake_stream_id)["events"]
|
||||
assert any(event.get("type") == "token" and event.get("payload", {}).get("text") == final_answer for event in journal_events)
|
||||
assert not [
|
||||
event for event in journal_events
|
||||
if event.get("type") == "reasoning" and event.get("payload", {}).get("text") == leaked_prefix
|
||||
]
|
||||
|
||||
done_payloads = [payload for event, payload in events if event == "done"]
|
||||
assert done_payloads, "run should settle"
|
||||
final_messages = done_payloads[-1]["session"]["messages"]
|
||||
assistant_messages = [message for message in final_messages if message.get("role") == "assistant"]
|
||||
assert assistant_messages[-1]["content"] == final_answer
|
||||
assert leaked_prefix not in str(assistant_messages[-1].get("reasoning") or "")
|
||||
assert leaked_prefix not in str(assistant_messages[-1].get("reasoning_content") or "")
|
||||
Reference in new issue
Block a user