From ec23dac922848ba6a45a55ec7d563eebb51b7fe9 Mon Sep 17 00:00:00 2001 From: Frank Song Date: Mon, 8 Jun 2026 16:24:34 +0800 Subject: [PATCH 1/4] Handle tool limit terminal state --- CHANGELOG.md | 4 + api/run_journal.py | 6 + api/streaming.py | 122 +++++++++++- static/messages.js | 5 +- tests/test_run_journal.py | 4 + tests/test_tool_limit_terminal_state.py | 254 ++++++++++++++++++++++++ 6 files changed, 391 insertions(+), 4 deletions(-) create mode 100644 tests/test_tool_limit_terminal_state.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1422858d..e1c95992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## [Unreleased] +### Fixed + +- **Tool iteration limit stops are now surfaced explicitly instead of looking like normal user turns.** WebUI filters Hermes Agent's synthetic max-iteration summary prompt out of both the visible transcript and model-facing context, marks usable final answers with `tool_limit_reached`, and shows a no-final terminal error when the limit fires before an assistant answer is available. (#3821) + ## [v0.51.365] — 2026-06-11 — Release MD (lineage-segment open + reasoning chip fixes) ### Fixed diff --git a/api/run_journal.py b/api/run_journal.py index c52e6b77..3ac639eb 100644 --- a/api/run_journal.py +++ b/api/run_journal.py @@ -84,11 +84,17 @@ def _next_seq(path: Path) -> int: def _terminal_state_for_event(event_name: str, payload) -> str | None: name = str(event_name or "") if name == "done" or name == "stream_end": + if isinstance(payload, dict): + explicit_state = str(payload.get("terminal_state") or "").strip().lower() + if explicit_state: + return explicit_state return "completed" if name == "cancel": return "interrupted-by-user" if name in {"apperror", "error"}: err_type = str((payload or {}).get("type") or "").strip().lower() if isinstance(payload, dict) else "" + if err_type == "tool_limit_reached": + return "tool_limit_reached" if err_type in {"cancelled", "canceled"}: return "interrupted-by-user" if err_type == "interrupted": diff --git a/api/streaming.py b/api/streaming.py index 88c3cf49..2be1a0c0 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -952,6 +952,89 @@ def _provider_error_payload(message: str, err_type: str, hint: str = '') -> dict return payload +_MAX_ITERATION_SUMMARY_REQUEST = ( + "You've reached the maximum number of tool-calling iterations allowed. " + "Please provide a final response summarizing what you've found and accomplished " + "so far, without calling any more tools." +) + + +def _is_synthetic_max_iteration_summary_request(message) -> bool: + """Return True for Hermes Agent's internal max-iteration summary prompt.""" + if not isinstance(message, dict) or message.get('role') != 'user': + return False + text = " ".join(_message_text(message.get('content', '')).split()) + expected = " ".join(_MAX_ITERATION_SUMMARY_REQUEST.split()) + return text == expected + + +def _drop_synthetic_max_iteration_summary_requests(messages, *, enabled: bool = True): + """Remove Agent-internal max-iteration summary prompts from WebUI state.""" + if not enabled: + return list(messages or []) + return [ + msg + for msg in list(messages or []) + if not _is_synthetic_max_iteration_summary_request(msg) + ] + + +def _agent_result_tool_limit_reached(result) -> bool: + """Return True when Hermes Agent stopped because the tool iteration cap fired.""" + if not isinstance(result, dict): + return False + fields = [ + result.get('turn_exit_reason'), + result.get('terminal_reason'), + result.get('status'), + result.get('state'), + result.get('error'), + ] + haystack = " ".join(str(value or '') for value in fields).lower() + if ( + 'max_iterations_reached' in haystack + or 'maximum number of tool-calling iterations' in haystack + or 'tool-calling iterations' in haystack and 'maximum' in haystack + ): + return True + return any( + _is_synthetic_max_iteration_summary_request(msg) + for msg in result.get('messages') or [] + ) + + +def _mark_latest_assistant_tool_limit_status(messages) -> bool: + """Annotate the latest usable assistant final answer as limit-stopped.""" + for msg in reversed(list(messages or [])): + if not isinstance(msg, dict): + continue + if msg.get('_error') or msg.get('role') != 'assistant': + continue + content = msg.get('content') + if isinstance(content, list): + text = '\n'.join( + str(part.get('text') or part.get('content') or '') + for part in content + if isinstance(part, dict) + ) + else: + text = str(content or '') + if msg.get('tool_calls') or not text.strip(): + continue + msg['_terminal_state'] = 'tool_limit_reached' + msg['_terminal_reason'] = 'max_iterations' + msg.setdefault('_statusCard', { + 'title': 'Tool iteration limit reached', + 'subtitle': 'Stopped because the tool iteration limit was reached.', + 'rows': [ + {'label': 'State', 'value': 'Limit reached'}, + {'label': 'Next step', 'value': 'Start a new turn to continue.'}, + ], + }) + return True + return False + + def _session_has_cancel_marker(session) -> bool: """Return True if a visible cancel/interrupted marker is already persisted.""" for msg in reversed(getattr(session, 'messages', None) or []): @@ -6527,7 +6610,12 @@ def _run_agent_streaming( getattr(s, 'active_stream_id', None), ) return + _tool_limit_reached = _agent_result_tool_limit_reached(result) _result_messages = result.get('messages') or _previous_context_messages + _result_messages = _drop_synthetic_max_iteration_summary_requests( + _result_messages, + enabled=_tool_limit_reached, + ) if cancel_event.is_set(): _finalize_cancelled_turn(s, ephemeral=False) try: @@ -6575,6 +6663,8 @@ def _run_agent_streaming( for _part in _raw_content: if isinstance(_part, dict) and isinstance(_part.get('text'), str): _part['text'] = _strip_xml_tool_calls(_part['text']) + if _tool_limit_reached and not _session_lacks_final_assistant_answer(s.messages): + _mark_latest_assistant_tool_limit_status(s.messages) # ── Handle context compression side effects ── # If compression fired inside run_conversation, the agent may have @@ -6709,7 +6799,7 @@ def _run_agent_streaming( # workspace-aware helper from this branch while still # preserving the pre-turn length for downstream self-heal # checks introduced on master. - _all_result_messages = result.get('messages') or [] + _all_result_messages = _result_messages or [] _prev_len = len(_previous_context_messages) _assistant_added = _assistant_reply_added_after_current_turn( _all_result_messages, @@ -6718,6 +6808,10 @@ def _run_agent_streaming( ) _terminal_failure = ( _agent_result_terminal_failure(result) + or ( + _tool_limit_reached + and _session_lacks_final_assistant_answer(_all_result_messages) + ) or ( not _token_sent and _session_lacks_final_assistant_answer(_all_result_messages) @@ -6824,6 +6918,10 @@ def _run_agent_streaming( # Since we're in a flat block, directly run the # post-result merge logic here. _result_messages = result.get('messages') or _previous_context_messages + _result_messages = _drop_synthetic_max_iteration_summary_requests( + _result_messages, + enabled=_agent_result_tool_limit_reached(result), + ) _next_context_messages = _restore_reasoning_metadata( _previous_context_messages, _result_messages, @@ -6861,6 +6959,17 @@ def _run_agent_streaming( 'your API key is invalid. Run `hermes model` in your terminal to ' 'update credentials, then restart the WebUI.' ) + elif _tool_limit_reached: + _err_label = 'Tool iteration limit reached' + _err_type = 'tool_limit_reached' + _err_hint = ( + 'The agent reached its configured tool iteration limit before producing ' + 'a final answer. Start a narrower follow-up or increase agent.max_turns.' + ) + _err_str = ( + 'The agent reached its configured tool iteration limit before producing ' + 'a final answer.' + ) else: _err_label = _classification['label'] _err_type = _classification['type'] @@ -6898,6 +7007,8 @@ def _run_agent_streaming( _error_message['provider_details_label'] = 'Cancellation details' elif _err_type == 'interrupted': _error_message['provider_details_label'] = 'Interruption details' + elif _err_type == 'tool_limit_reached': + _error_message['provider_details_label'] = 'Terminal state details' s.messages.append(_error_message) try: s.save() @@ -6911,6 +7022,9 @@ def _run_agent_streaming( if _compression_continuation_session_id is not None: _error_payload['new_session_id'] = _compression_continuation_session_id _error_payload['continuation_session_id'] = _compression_continuation_session_id + if _err_type == 'tool_limit_reached': + _error_payload['terminal_state'] = 'tool_limit_reached' + _error_payload['terminal_reason'] = 'max_iterations' put('apperror', _error_payload) # Legacy #373 source tests and clients look for the # no_response type; #1765 keeps that type but improves @@ -7601,7 +7715,11 @@ def _run_agent_streaming( except Exception as _goal_exc: logger.debug("Goal continuation hook failed for session %s: %s", session_id, _goal_exc) raw_session = s.compact() | {'messages': s.messages, 'tool_calls': tool_calls} - put('done', {'session': redact_session_data(raw_session), 'usage': usage}) + _done_payload = {'session': redact_session_data(raw_session), 'usage': usage} + if _tool_limit_reached: + _done_payload['terminal_state'] = 'tool_limit_reached' + _done_payload['terminal_reason'] = 'max_iterations' + put('done', _done_payload) # Emit one last metering packet for the live message-header TPS label. meter_stats = meter().get_stats() meter_stats['session_id'] = session_id diff --git a/static/messages.js b/static/messages.js index c9973fb8..cd03618a 100644 --- a/static/messages.js +++ b/static/messages.js @@ -3504,12 +3504,13 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ const isCancelled=d.type==='cancelled'; const isInterrupted=d.type==='interrupted'; const isCompressionExhausted=d.type==='compression_exhausted'; + const isToolLimitReached=d.type==='tool_limit_reached'; isRecoveryControlMessage=isInterrupted && (d.recovery_control===true || _streamRecoveryControlMessageText(d.message)); const isNoResponse=d.type==='no_response'||d.type==='silent_failure'; - const label=isCancelled?'Task cancelled':isInterrupted?'Response interrupted':isCompressionExhausted?'Context compression exhausted':isQuotaExhausted?'Out of credits':isRateLimit?'Rate limit reached':isGatewayAuthError?(typeof t==='function'?t('gateway_auth_label'):'Gateway authentication failed'):isAuthMismatch?(typeof t==='function'?t('provider_mismatch_label'):'Provider mismatch'):isModelNotFound?(typeof t==='function'?t('model_not_found_label'):'Model not found'):isNoResponse?'No response from provider':'Error'; + const label=isCancelled?'Task cancelled':isInterrupted?'Response interrupted':isCompressionExhausted?'Context compression exhausted':isToolLimitReached?'Tool iteration limit reached':isQuotaExhausted?'Out of credits':isRateLimit?'Rate limit reached':isGatewayAuthError?(typeof t==='function'?t('gateway_auth_label'):'Gateway authentication failed'):isAuthMismatch?(typeof t==='function'?t('provider_mismatch_label'):'Provider mismatch'):isModelNotFound?(typeof t==='function'?t('model_not_found_label'):'Model not found'):isNoResponse?'No response from provider':'Error'; const hint=d.hint?`\n\n*${d.hint}*`:''; const details=d.details?String(d.details).replace(/```/g,'`\u200b``'):''; - const detailsLabel=isCancelled?'Cancellation details':isInterrupted?'Interruption details':undefined; + const detailsLabel=isCancelled?'Cancellation details':isInterrupted?'Interruption details':isToolLimitReached?'Terminal state details':undefined; window._compressionUi=null; if(typeof clearCompressionUi==='function') clearCompressionUi(); if(isRecoveryControlMessage){ diff --git a/tests/test_run_journal.py b/tests/test_run_journal.py index 84f30e09..b1d32c08 100644 --- a/tests/test_run_journal.py +++ b/tests/test_run_journal.py @@ -108,11 +108,15 @@ def test_terminal_state_classification_distinguishes_crash_from_user_cancel(tmp_ append_run_event("session_1", "run_cancelled", "cancel", {"message": "Cancelled by user"}, session_dir=tmp_path) append_run_event("session_1", "run_crashed", "apperror", {"type": "interrupted"}, session_dir=tmp_path) append_run_event("session_1", "run_failed", "apperror", {"type": "auth_mismatch"}, session_dir=tmp_path) + append_run_event("session_1", "run_tool_limit", "apperror", {"type": "tool_limit_reached"}, session_dir=tmp_path) + append_run_event("session_1", "run_tool_limit_done", "done", {"terminal_state": "tool_limit_reached"}, session_dir=tmp_path) append_run_event("session_1", "run_done", "done", {"session": {}}, session_dir=tmp_path) assert latest_run_summary("session_1", "run_cancelled", session_dir=tmp_path)["terminal_state"] == "interrupted-by-user" assert latest_run_summary("session_1", "run_crashed", session_dir=tmp_path)["terminal_state"] == "interrupted-by-crash" assert latest_run_summary("session_1", "run_failed", session_dir=tmp_path)["terminal_state"] == "errored" + assert latest_run_summary("session_1", "run_tool_limit", session_dir=tmp_path)["terminal_state"] == "tool_limit_reached" + assert latest_run_summary("session_1", "run_tool_limit_done", session_dir=tmp_path)["terminal_state"] == "tool_limit_reached" assert latest_run_summary("session_1", "run_done", session_dir=tmp_path)["terminal_state"] == "completed" diff --git a/tests/test_tool_limit_terminal_state.py b/tests/test_tool_limit_terminal_state.py new file mode 100644 index 00000000..0984c0a3 --- /dev/null +++ b/tests/test_tool_limit_terminal_state.py @@ -0,0 +1,254 @@ +import json +import queue +import sys +import types +from pathlib import Path + +from api import models +from api import streaming +from api.models import Session + + +ROOT = Path(__file__).resolve().parents[1] + + +def _run_streaming_with_fake_agent(tmp_path, monkeypatch, agent_result): + session_dir = tmp_path / "sessions" + session_dir.mkdir() + monkeypatch.setattr(models, "SESSION_DIR", session_dir) + monkeypatch.setattr(models, "SESSION_INDEX_FILE", session_dir / "_index.json") + monkeypatch.setattr(streaming, "SESSION_DIR", session_dir) + models.SESSIONS.clear() + streaming.SESSIONS.clear() + streaming.STREAMS.clear() + streaming.AGENT_INSTANCES.clear() + streaming.SESSION_AGENT_LOCKS.clear() + streaming.PENDING_GOAL_CONTINUATION.clear() + try: + from api.config import SESSION_AGENT_CACHE + + SESSION_AGENT_CACHE.clear() + except Exception: + pass + + session_id = "tool_limit_session" + stream_id = "stream-tool-limit" + session = Session( + session_id=session_id, + title="Tool limit test", + workspace=str(tmp_path), + model="gpt-4o", + messages=[], + context_messages=[], + ) + session.active_stream_id = stream_id + session.pending_user_message = "Do the long task." + session.pending_started_at = 1.0 + session.save() + models.SESSIONS[session_id] = session + streaming.SESSIONS[session_id] = session + event_queue = queue.Queue() + streaming.STREAMS[stream_id] = event_queue + + class FakeAgent: + def __init__(self, **kwargs): + self.session_id = kwargs.get("session_id") + self.stream_delta_callback = kwargs.get("stream_delta_callback") + self.context_compressor = None + self.session_prompt_tokens = 0 + self.session_completion_tokens = 0 + self.session_estimated_cost_usd = None + self.session_cache_read_tokens = 0 + self.session_cache_write_tokens = 0 + self.reasoning_config = None + self.ephemeral_system_prompt = None + self._last_error = None + + def run_conversation(self, **kwargs): + return agent_result + + def interrupt(self, _message): + return None + + fake_hermes_state = types.ModuleType("hermes_state") + fake_hermes_state.SessionDB = lambda *_args, **_kwargs: object() + + with monkeypatch.context() as m: + m.setattr(streaming, "get_session", lambda _sid: session) + m.setattr(streaming, "_get_ai_agent", lambda: FakeAgent) + m.setattr(streaming, "resolve_model_provider", lambda *_args, **_kwargs: ("gpt-4o", "openai", None)) + m.setattr("api.config.get_config", lambda *_args, **_kwargs: {}) + m.setattr("api.config._resolve_cli_toolsets", lambda *_args, **_kwargs: []) + m.setitem(sys.modules, "hermes_state", fake_hermes_state) + streaming._run_agent_streaming( + session_id=session_id, + msg_text="Do the long task.", + model="gpt-4o", + workspace=str(tmp_path), + stream_id=stream_id, + ) + + events = [] + while not event_queue.empty(): + events.append(event_queue.get_nowait()) + payload = json.loads((session_dir / f"{session_id}.json").read_text(encoding="utf-8")) + return events, payload + + +def test_synthetic_max_iteration_summary_request_is_dropped_from_agent_result(): + synthetic = { + "role": "user", + "content": streaming._MAX_ITERATION_SUMMARY_REQUEST, + } + messages = [ + {"role": "user", "content": "Do the long task."}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "call_1"}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "result"}, + synthetic, + {"role": "assistant", "content": "I reached the limit; here is the summary."}, + ] + result = { + "turn_exit_reason": "max_iterations_reached(30/30)", + "messages": messages, + } + + assert streaming._agent_result_tool_limit_reached(result) is True + + cleaned = streaming._drop_synthetic_max_iteration_summary_requests( + result["messages"], + enabled=streaming._agent_result_tool_limit_reached(result), + ) + + assert synthetic not in cleaned + assert cleaned[-1]["role"] == "assistant" + assert "here is the summary" in cleaned[-1]["content"] + + +def test_tool_limit_with_final_answer_marks_latest_assistant_status_card(): + messages = [ + {"role": "user", "content": "Do the long task."}, + {"role": "assistant", "content": "I reached the limit; here is the summary."}, + ] + + assert streaming._session_lacks_final_assistant_answer(messages) is False + assert streaming._mark_latest_assistant_tool_limit_status(messages) is True + + assistant = messages[-1] + assert assistant["_terminal_state"] == "tool_limit_reached" + assert assistant["_terminal_reason"] == "max_iterations" + assert assistant["_statusCard"]["title"] == "Tool iteration limit reached" + + +def test_tool_limit_without_final_answer_is_no_final_terminal_state_after_filtering(): + messages = [ + {"role": "user", "content": "Do the long task."}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "call_1"}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "result"}, + {"role": "user", "content": streaming._MAX_ITERATION_SUMMARY_REQUEST}, + ] + + cleaned = streaming._drop_synthetic_max_iteration_summary_requests(messages) + + assert all( + not streaming._is_synthetic_max_iteration_summary_request(message) + for message in cleaned + ) + assert streaming._session_lacks_final_assistant_answer(cleaned) is True + + +def test_display_merge_does_not_render_synthetic_summary_prompt(): + previous_display = [{"role": "user", "content": "Do the long task."}] + previous_context = [{"role": "user", "content": "Do the long task."}] + result_messages = previous_context + [ + {"role": "assistant", "content": "", "tool_calls": [{"id": "call_1"}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "result"}, + {"role": "user", "content": streaming._MAX_ITERATION_SUMMARY_REQUEST}, + {"role": "assistant", "content": "I reached the limit; here is the summary."}, + ] + result_messages = streaming._drop_synthetic_max_iteration_summary_requests( + result_messages, + enabled=True, + ) + + merged = streaming._merge_display_messages_after_agent_result( + previous_display, + previous_context, + result_messages, + "Do the long task.", + ) + + assert all( + message.get("content") != streaming._MAX_ITERATION_SUMMARY_REQUEST + for message in merged + ) + assert merged[-1]["role"] == "assistant" + assert "here is the summary" in merged[-1]["content"] + + +def test_frontend_handles_tool_limit_apperror_label(): + messages_js = (ROOT / "static" / "messages.js").read_text(encoding="utf-8") + start = messages_js.find("source.addEventListener('apperror'") + end = messages_js.find("source.addEventListener('warning'", start) + assert start != -1 and end != -1 + block = messages_js[start:end] + + assert "const isToolLimitReached=d.type==='tool_limit_reached';" in block + assert "Tool iteration limit reached" in block + assert "Terminal state details" in block + + +def test_streaming_tool_limit_with_final_answer_persists_clean_done_state(tmp_path, monkeypatch): + result = { + "turn_exit_reason": "max_iterations_reached(30/30)", + "messages": [ + {"role": "user", "content": "Do the long task."}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "call_1"}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "result"}, + {"role": "user", "content": streaming._MAX_ITERATION_SUMMARY_REQUEST}, + {"role": "assistant", "content": "I reached the limit; here is the summary."}, + ], + } + + events, payload = _run_streaming_with_fake_agent(tmp_path, monkeypatch, result) + + done_payloads = [payload for event, payload in events if event == "done"] + assert done_payloads, "expected done SSE payload" + assert done_payloads[-1]["terminal_state"] == "tool_limit_reached" + assert done_payloads[-1]["terminal_reason"] == "max_iterations" + assert all( + message.get("content") != streaming._MAX_ITERATION_SUMMARY_REQUEST + for message in payload["messages"] + ) + assert all( + message.get("content") != streaming._MAX_ITERATION_SUMMARY_REQUEST + for message in payload["context_messages"] + ) + assistant = payload["messages"][-1] + assert assistant["role"] == "assistant" + assert assistant["_terminal_state"] == "tool_limit_reached" + assert assistant["_statusCard"]["title"] == "Tool iteration limit reached" + + +def test_streaming_tool_limit_without_final_answer_emits_no_final_apperror(tmp_path, monkeypatch): + result = { + "turn_exit_reason": "max_iterations_reached(30/30)", + "messages": [ + {"role": "user", "content": "Do the long task."}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "call_1"}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "result"}, + {"role": "user", "content": streaming._MAX_ITERATION_SUMMARY_REQUEST}, + ], + } + + events, payload = _run_streaming_with_fake_agent(tmp_path, monkeypatch, result) + + apperror_payloads = [payload for event, payload in events if event == "apperror"] + assert apperror_payloads, "expected apperror SSE payload" + assert apperror_payloads[-1]["type"] == "tool_limit_reached" + assert apperror_payloads[-1]["terminal_state"] == "tool_limit_reached" + assert payload["messages"][-1]["_error"] is True + assert "Tool iteration limit reached" in payload["messages"][-1]["content"] + assert all( + message.get("content") != streaming._MAX_ITERATION_SUMMARY_REQUEST + for message in payload["messages"] + ) From 08ba5a6676245090d87be86b6523eca388690b1d Mon Sep 17 00:00:00 2001 From: Frank Song Date: Thu, 11 Jun 2026 15:31:34 +0800 Subject: [PATCH 2/4] Fix tool-limit terminal detection --- api/streaming.py | 7 ++---- tests/test_tool_limit_terminal_state.py | 33 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/api/streaming.py b/api/streaming.py index 2be1a0c0..e37a76e2 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -980,7 +980,7 @@ def _drop_synthetic_max_iteration_summary_requests(messages, *, enabled: bool = def _agent_result_tool_limit_reached(result) -> bool: - """Return True when Hermes Agent stopped because the tool iteration cap fired.""" + """Return True when current-turn metadata says the tool iteration cap fired.""" if not isinstance(result, dict): return False fields = [ @@ -997,10 +997,7 @@ def _agent_result_tool_limit_reached(result) -> bool: or 'tool-calling iterations' in haystack and 'maximum' in haystack ): return True - return any( - _is_synthetic_max_iteration_summary_request(msg) - for msg in result.get('messages') or [] - ) + return False def _mark_latest_assistant_tool_limit_status(messages) -> bool: diff --git a/tests/test_tool_limit_terminal_state.py b/tests/test_tool_limit_terminal_state.py index 0984c0a3..9bc0b1a7 100644 --- a/tests/test_tool_limit_terminal_state.py +++ b/tests/test_tool_limit_terminal_state.py @@ -124,6 +124,19 @@ def test_synthetic_max_iteration_summary_request_is_dropped_from_agent_result(): assert "here is the summary" in cleaned[-1]["content"] +def test_historical_synthetic_summary_prompt_does_not_mark_normal_result_as_tool_limit(): + result = { + "messages": [ + {"role": "user", "content": "Earlier task."}, + {"role": "user", "content": streaming._MAX_ITERATION_SUMMARY_REQUEST}, + {"role": "user", "content": "Current normal task."}, + {"role": "assistant", "content": "Current task completed normally."}, + ], + } + + assert streaming._agent_result_tool_limit_reached(result) is False + + def test_tool_limit_with_final_answer_marks_latest_assistant_status_card(): messages = [ {"role": "user", "content": "Do the long task."}, @@ -252,3 +265,23 @@ def test_streaming_tool_limit_without_final_answer_emits_no_final_apperror(tmp_p message.get("content") != streaming._MAX_ITERATION_SUMMARY_REQUEST for message in payload["messages"] ) + + +def test_streaming_historical_synthetic_prompt_normal_result_does_not_emit_tool_limit(tmp_path, monkeypatch): + result = { + "messages": [ + {"role": "user", "content": "Earlier task."}, + {"role": "user", "content": streaming._MAX_ITERATION_SUMMARY_REQUEST}, + {"role": "user", "content": "Do the long task."}, + {"role": "assistant", "content": "Current task completed normally."}, + ], + } + + events, payload = _run_streaming_with_fake_agent(tmp_path, monkeypatch, result) + + assert not [payload for event, payload in events if event == "apperror"] + done_payloads = [payload for event, payload in events if event == "done"] + assert done_payloads, "expected normal done SSE payload" + assert "terminal_state" not in done_payloads[-1] + assert payload["messages"][-1]["role"] == "assistant" + assert payload["messages"][-1]["content"] == "Current task completed normally." From 136568fe81e65bcedbdc5f5897fbb2920452c532 Mon Sep 17 00:00:00 2001 From: Frank Song Date: Thu, 11 Jun 2026 17:58:15 +0800 Subject: [PATCH 3/4] fix(streaming): tighten tool limit terminal guards --- api/run_journal.py | 2 +- api/streaming.py | 4 +-- tests/test_run_journal.py | 2 ++ tests/test_tool_limit_terminal_state.py | 41 +++++++++++++++++++++++-- 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/api/run_journal.py b/api/run_journal.py index 3ac639eb..1853de05 100644 --- a/api/run_journal.py +++ b/api/run_journal.py @@ -86,7 +86,7 @@ def _terminal_state_for_event(event_name: str, payload) -> str | None: if name == "done" or name == "stream_end": if isinstance(payload, dict): explicit_state = str(payload.get("terminal_state") or "").strip().lower() - if explicit_state: + if explicit_state in {"tool_limit_reached"}: return explicit_state return "completed" if name == "cancel": diff --git a/api/streaming.py b/api/streaming.py index e37a76e2..b5177d0d 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -994,7 +994,7 @@ def _agent_result_tool_limit_reached(result) -> bool: if ( 'max_iterations_reached' in haystack or 'maximum number of tool-calling iterations' in haystack - or 'tool-calling iterations' in haystack and 'maximum' in haystack + or ('tool-calling iterations' in haystack and 'maximum' in haystack) ): return True return False @@ -6796,7 +6796,7 @@ def _run_agent_streaming( # workspace-aware helper from this branch while still # preserving the pre-turn length for downstream self-heal # checks introduced on master. - _all_result_messages = _result_messages or [] + _all_result_messages = result.get('messages') or [] _prev_len = len(_previous_context_messages) _assistant_added = _assistant_reply_added_after_current_turn( _all_result_messages, diff --git a/tests/test_run_journal.py b/tests/test_run_journal.py index b1d32c08..6681e89f 100644 --- a/tests/test_run_journal.py +++ b/tests/test_run_journal.py @@ -110,6 +110,7 @@ def test_terminal_state_classification_distinguishes_crash_from_user_cancel(tmp_ append_run_event("session_1", "run_failed", "apperror", {"type": "auth_mismatch"}, session_dir=tmp_path) append_run_event("session_1", "run_tool_limit", "apperror", {"type": "tool_limit_reached"}, session_dir=tmp_path) append_run_event("session_1", "run_tool_limit_done", "done", {"terminal_state": "tool_limit_reached"}, session_dir=tmp_path) + append_run_event("session_1", "run_unknown_done", "done", {"terminal_state": "future_unknown_state"}, session_dir=tmp_path) append_run_event("session_1", "run_done", "done", {"session": {}}, session_dir=tmp_path) assert latest_run_summary("session_1", "run_cancelled", session_dir=tmp_path)["terminal_state"] == "interrupted-by-user" @@ -117,6 +118,7 @@ def test_terminal_state_classification_distinguishes_crash_from_user_cancel(tmp_ assert latest_run_summary("session_1", "run_failed", session_dir=tmp_path)["terminal_state"] == "errored" assert latest_run_summary("session_1", "run_tool_limit", session_dir=tmp_path)["terminal_state"] == "tool_limit_reached" assert latest_run_summary("session_1", "run_tool_limit_done", session_dir=tmp_path)["terminal_state"] == "tool_limit_reached" + assert latest_run_summary("session_1", "run_unknown_done", session_dir=tmp_path)["terminal_state"] == "completed" assert latest_run_summary("session_1", "run_done", session_dir=tmp_path)["terminal_state"] == "completed" diff --git a/tests/test_tool_limit_terminal_state.py b/tests/test_tool_limit_terminal_state.py index 9bc0b1a7..e675ba40 100644 --- a/tests/test_tool_limit_terminal_state.py +++ b/tests/test_tool_limit_terminal_state.py @@ -12,7 +12,14 @@ from api.models import Session ROOT = Path(__file__).resolve().parents[1] -def _run_streaming_with_fake_agent(tmp_path, monkeypatch, agent_result): +def _run_streaming_with_fake_agent( + tmp_path, + monkeypatch, + agent_result, + *, + prior_messages=None, + prior_context_messages=None, +): session_dir = tmp_path / "sessions" session_dir.mkdir() monkeypatch.setattr(models, "SESSION_DIR", session_dir) @@ -38,8 +45,8 @@ def _run_streaming_with_fake_agent(tmp_path, monkeypatch, agent_result): title="Tool limit test", workspace=str(tmp_path), model="gpt-4o", - messages=[], - context_messages=[], + messages=list(prior_messages or []), + context_messages=list(prior_context_messages or []), ) session.active_stream_id = stream_id session.pending_user_message = "Do the long task." @@ -124,6 +131,12 @@ def test_synthetic_max_iteration_summary_request_is_dropped_from_agent_result(): assert "here is the summary" in cleaned[-1]["content"] +def test_tool_limit_detection_uses_explicit_boolean_grouping(): + streaming_py = (ROOT / "api" / "streaming.py").read_text(encoding="utf-8") + + assert "or ('tool-calling iterations' in haystack and 'maximum' in haystack)" in streaming_py + + def test_historical_synthetic_summary_prompt_does_not_mark_normal_result_as_tool_limit(): result = { "messages": [ @@ -285,3 +298,25 @@ def test_streaming_historical_synthetic_prompt_normal_result_does_not_emit_tool_ assert "terminal_state" not in done_payloads[-1] assert payload["messages"][-1]["role"] == "assistant" assert payload["messages"][-1]["content"] == "Current task completed normally." + + +def test_streaming_empty_result_messages_do_not_treat_prior_assistant_as_current_answer(tmp_path, monkeypatch): + prior = [ + {"role": "user", "content": "Earlier task."}, + {"role": "assistant", "content": "Earlier answer."}, + ] + result = {"messages": []} + + events, payload = _run_streaming_with_fake_agent( + tmp_path, + monkeypatch, + result, + prior_messages=prior, + prior_context_messages=prior, + ) + + apperror_payloads = [payload for event, payload in events if event == "apperror"] + assert apperror_payloads, "expected silent-failure apperror" + assert apperror_payloads[-1]["type"] == "no_response" + assert not [payload for event, payload in events if event == "done"] + assert payload["messages"][-1]["_error"] is True From 02e72163a6229124223e25a125e94b7d6e1eaab0 Mon Sep 17 00:00:00 2001 From: Frank Song Date: Fri, 12 Jun 2026 10:25:18 +0800 Subject: [PATCH 4/4] Fix tool-limit terminal failure annotation --- api/streaming.py | 5 ++--- tests/test_tool_limit_terminal_state.py | 27 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/api/streaming.py b/api/streaming.py index b5177d0d..b1c1db36 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -6660,9 +6660,6 @@ def _run_agent_streaming( for _part in _raw_content: if isinstance(_part, dict) and isinstance(_part.get('text'), str): _part['text'] = _strip_xml_tool_calls(_part['text']) - if _tool_limit_reached and not _session_lacks_final_assistant_answer(s.messages): - _mark_latest_assistant_tool_limit_status(s.messages) - # ── Handle context compression side effects ── # If compression fired inside run_conversation, the agent may have # rotated its session_id. Detect and fix the mismatch before any @@ -6816,6 +6813,8 @@ def _run_agent_streaming( ) if _terminal_failure: _assistant_added = False + elif _tool_limit_reached and not _session_lacks_final_assistant_answer(s.messages): + _mark_latest_assistant_tool_limit_status(s.messages) # _token_sent tracks whether on_token() was called (any streamed text) if _terminal_failure or (not _assistant_added and not _token_sent): if cancel_event.is_set(): diff --git a/tests/test_tool_limit_terminal_state.py b/tests/test_tool_limit_terminal_state.py index e675ba40..80e4cd38 100644 --- a/tests/test_tool_limit_terminal_state.py +++ b/tests/test_tool_limit_terminal_state.py @@ -280,6 +280,33 @@ def test_streaming_tool_limit_without_final_answer_emits_no_final_apperror(tmp_p ) +def test_streaming_tool_limit_terminal_failure_does_not_mark_final_answer(tmp_path, monkeypatch): + result = { + "status": "partial", + "turn_exit_reason": "max_iterations_reached(30/30)", + "messages": [ + {"role": "user", "content": "Do the long task."}, + {"role": "assistant", "content": "I reached the limit; here is the summary."}, + ], + } + + events, payload = _run_streaming_with_fake_agent(tmp_path, monkeypatch, result) + + apperror_payloads = [payload for event, payload in events if event == "apperror"] + assert apperror_payloads, "expected terminal-failure apperror" + assert apperror_payloads[-1]["type"] == "tool_limit_reached" + assert not [payload for event, payload in events if event == "done"] + assistant = next( + message + for message in payload["messages"] + if message.get("role") == "assistant" + and message.get("content") == "I reached the limit; here is the summary." + ) + assert "_terminal_state" not in assistant + assert "_statusCard" not in assistant + assert payload["messages"][-1]["_error"] is True + + def test_streaming_historical_synthetic_prompt_normal_result_does_not_emit_tool_limit(tmp_path, monkeypatch): result = { "messages": [