Merge pull request #6503 from rodboev/pr/6481-verification-evidence-contamination
fix(#6481): settle verification turns from exact turn ownership
This commit is contained in:
11 files changed
+1706
-114
No files matched your search
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
@@ -137,7 +138,25 @@ def attach_wakeup_display_meta(msg: Any, source: Any) -> None:
|
||||
msg["_wakeup_meta"] = meta
|
||||
|
||||
|
||||
def stamp_message_source(msg: Any, source: Any) -> None:
|
||||
def build_active_turn_token(stream_id: Any, started_at: Any) -> str | None:
|
||||
"""Return the exact eager-row token for one active WebUI turn."""
|
||||
if not stream_id:
|
||||
return None
|
||||
try:
|
||||
started = float(started_at)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not math.isfinite(started) or started <= 0:
|
||||
return None
|
||||
return f"{str(stream_id).strip()}:{started:.17g}"
|
||||
|
||||
|
||||
def stamp_message_source(
|
||||
msg: Any,
|
||||
source: Any,
|
||||
*,
|
||||
active_turn_token: Any = None,
|
||||
) -> None:
|
||||
"""Stamp ``_source`` and any display metadata on a materialized user turn.
|
||||
|
||||
Single choke point for every path that persists a non-``webui`` user turn
|
||||
@@ -147,6 +166,8 @@ def stamp_message_source(msg: Any, source: Any) -> None:
|
||||
and the cancel outer-finally recovery. ``webui`` turns are left untouched to
|
||||
preserve the existing "``_source`` omitted for the default source" contract.
|
||||
"""
|
||||
if isinstance(msg, dict) and active_turn_token:
|
||||
msg["_active_turn_token"] = str(active_turn_token)
|
||||
if not isinstance(msg, dict) or not source or source == "webui":
|
||||
return
|
||||
msg["_source"] = source
|
||||
|
||||
+7
-3
@@ -20929,11 +20929,15 @@ def _checkpoint_user_message_for_eager_session_save(s, msg: str, attachments, st
|
||||
if latest_text == msg_text:
|
||||
return
|
||||
user_msg = {"role": "user", "content": msg}
|
||||
from api.process_event_utils import stamp_message_source
|
||||
from api.process_event_utils import build_active_turn_token, stamp_message_source
|
||||
|
||||
stamp_message_source(user_msg, source)
|
||||
stamp_message_source(
|
||||
user_msg,
|
||||
source,
|
||||
active_turn_token=build_active_turn_token(getattr(s, "active_stream_id", None), started_at),
|
||||
)
|
||||
if isinstance(started_at, (int, float)) and started_at > 0:
|
||||
user_msg["timestamp"] = int(started_at)
|
||||
user_msg["timestamp"] = float(started_at)
|
||||
if attachments:
|
||||
user_msg["attachments"] = list(attachments)
|
||||
s.messages.append(user_msg)
|
||||
|
||||
+561
-86
@@ -66,6 +66,7 @@ from api.models import (
|
||||
)
|
||||
from api.session_ops import mark_session_title_generated, session_has_manual_title
|
||||
from api.process_event_utils import (
|
||||
build_active_turn_token,
|
||||
claim_async_delegation_delivery,
|
||||
complete_async_delegation_delivery,
|
||||
completion_delivery_id,
|
||||
@@ -1429,6 +1430,380 @@ def _drop_synthetic_control_messages(messages):
|
||||
]
|
||||
|
||||
|
||||
def _clean_synthetic_control_messages_with_provenance(messages):
|
||||
"""Remove synthetic rows while retaining marked verification-nudge provenance."""
|
||||
raw_messages = list(messages or [])
|
||||
has_verification_nudge = any(
|
||||
isinstance(message, dict)
|
||||
and message.get('role') == 'user'
|
||||
and _is_synthetic_control_message(message)
|
||||
for message in raw_messages
|
||||
)
|
||||
return _drop_synthetic_control_messages(raw_messages), has_verification_nudge
|
||||
|
||||
|
||||
def _active_turn_authority(session, stream_id, msg_text):
|
||||
"""Capture the stream-owned pending turn before settlement mutates it."""
|
||||
pending_text = getattr(session, 'pending_user_message', None)
|
||||
return {
|
||||
'token': build_active_turn_token(stream_id, getattr(session, 'pending_started_at', None)),
|
||||
'text': pending_text if pending_text is not None else msg_text,
|
||||
'timestamp': getattr(session, 'pending_started_at', None),
|
||||
'source': getattr(session, 'pending_user_source', None) or 'webui',
|
||||
'attachments': copy.deepcopy(getattr(session, 'pending_attachments', None) or []),
|
||||
'current_turn_user_idx': None,
|
||||
'turn_id': '',
|
||||
}
|
||||
|
||||
|
||||
def _coerce_current_turn_user_idx(value):
|
||||
if isinstance(value, bool) or value is None:
|
||||
return None
|
||||
try:
|
||||
idx = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return idx if idx >= 0 else None
|
||||
|
||||
|
||||
def _resolve_active_turn_authority(identity, *, result=None, agent=None):
|
||||
if not isinstance(identity, dict):
|
||||
return identity
|
||||
resolved = dict(identity)
|
||||
if isinstance(result, dict):
|
||||
_result_idx = _coerce_current_turn_user_idx(result.get('current_turn_user_idx'))
|
||||
if _result_idx is not None:
|
||||
resolved['current_turn_user_idx'] = _result_idx
|
||||
_result_turn_id = str(result.get('turn_id') or '').strip()
|
||||
if _result_turn_id:
|
||||
resolved['turn_id'] = _result_turn_id
|
||||
if agent is not None:
|
||||
if resolved.get('current_turn_user_idx') is None:
|
||||
_agent_idx = _coerce_current_turn_user_idx(
|
||||
getattr(agent, '_persist_user_message_idx', None)
|
||||
)
|
||||
if _agent_idx is not None:
|
||||
resolved['current_turn_user_idx'] = _agent_idx
|
||||
if not resolved.get('turn_id'):
|
||||
_agent_turn_id = str(getattr(agent, '_current_turn_id', '') or '').strip()
|
||||
if _agent_turn_id:
|
||||
resolved['turn_id'] = _agent_turn_id
|
||||
return resolved
|
||||
|
||||
|
||||
def _active_turn_boundary_is_valid(identity):
|
||||
if not isinstance(identity, dict):
|
||||
return False
|
||||
if not str(identity.get('turn_id') or '').strip():
|
||||
return False
|
||||
return isinstance(identity.get('current_turn_user_idx'), int)
|
||||
|
||||
|
||||
def _active_turn_token_matches(message, identity):
|
||||
token = identity.get('token') if isinstance(identity, dict) else None
|
||||
if not token:
|
||||
return False
|
||||
return (
|
||||
isinstance(message, dict)
|
||||
and message.get('role') == 'user'
|
||||
and message.get('_active_turn_token') == token
|
||||
)
|
||||
|
||||
|
||||
def _active_turn_has_checkpoint(messages, identity):
|
||||
return any(_active_turn_token_matches(message, identity) for message in messages or [])
|
||||
|
||||
|
||||
def _mark_active_turn_checkpoint(message, identity):
|
||||
if (
|
||||
isinstance(message, dict)
|
||||
and message.get('role') == 'user'
|
||||
and isinstance(identity, dict)
|
||||
and identity.get('token')
|
||||
):
|
||||
message['_active_turn_token'] = identity['token']
|
||||
return message
|
||||
|
||||
|
||||
def _mark_active_turn_checkpoint_in_history(messages, identity, msg_text, *, allow_index_fallback=True):
|
||||
messages = list(messages or [])
|
||||
if not isinstance(identity, dict):
|
||||
return messages, False
|
||||
if _active_turn_has_checkpoint(messages, identity):
|
||||
return messages, True
|
||||
if not allow_index_fallback:
|
||||
return messages, False
|
||||
if not _active_turn_boundary_is_valid(identity):
|
||||
return messages, False
|
||||
idx = identity['current_turn_user_idx']
|
||||
if idx < 0 or idx >= len(messages):
|
||||
return messages, False
|
||||
message = messages[idx]
|
||||
expected_text = identity.get('text') if identity.get('text') is not None else msg_text
|
||||
if (
|
||||
not isinstance(message, dict)
|
||||
or message.get('role') != 'user'
|
||||
or _normalize_user_text(message.get('content')) != _normalize_user_text(expected_text)
|
||||
):
|
||||
return messages, False
|
||||
_mark_active_turn_checkpoint(message, identity)
|
||||
return messages, True
|
||||
|
||||
|
||||
def _owner_projection_current_turn_row(messages, identity):
|
||||
messages = list(messages or [])
|
||||
if not isinstance(identity, dict):
|
||||
return None
|
||||
for message in messages:
|
||||
if _active_turn_token_matches(message, identity):
|
||||
return copy.deepcopy(message)
|
||||
return None
|
||||
|
||||
|
||||
def _find_active_turn_checkpoint_index(result_messages, previous_context, identity, msg_text):
|
||||
result_messages = list(result_messages or [])
|
||||
if not isinstance(identity, dict):
|
||||
return None
|
||||
if _active_turn_has_checkpoint(result_messages, identity):
|
||||
for idx, message in enumerate(result_messages):
|
||||
if _active_turn_token_matches(message, identity):
|
||||
return idx
|
||||
if not _active_turn_boundary_is_valid(identity):
|
||||
return None
|
||||
expected_text = identity.get('text') if identity.get('text') is not None else msg_text
|
||||
candidates = []
|
||||
current_turn_user_idx = identity['current_turn_user_idx']
|
||||
previous_context = list(previous_context or [])
|
||||
if _messages_have_prefix(result_messages, previous_context):
|
||||
candidates.append(current_turn_user_idx)
|
||||
else:
|
||||
offset = current_turn_user_idx - len(previous_context)
|
||||
candidates.extend([offset, current_turn_user_idx])
|
||||
seen = set()
|
||||
for idx in candidates:
|
||||
if idx in seen or idx is None:
|
||||
continue
|
||||
seen.add(idx)
|
||||
if idx < 0 or idx >= len(result_messages):
|
||||
continue
|
||||
message = result_messages[idx]
|
||||
if (
|
||||
isinstance(message, dict)
|
||||
and message.get('role') == 'user'
|
||||
and _normalize_user_text(message.get('content')) == _normalize_user_text(expected_text)
|
||||
):
|
||||
return idx
|
||||
return None
|
||||
|
||||
|
||||
def _materialize_active_turn_user(identity, msg_text, source):
|
||||
message = {
|
||||
'role': 'user',
|
||||
'content': identity.get('text') if isinstance(identity, dict) else msg_text,
|
||||
}
|
||||
if isinstance(identity, dict):
|
||||
if identity.get('timestamp') is not None:
|
||||
message['timestamp'] = identity['timestamp']
|
||||
if identity.get('attachments'):
|
||||
message['attachments'] = copy.deepcopy(identity['attachments'])
|
||||
stamp_message_source(
|
||||
message,
|
||||
identity.get('source') or source or 'webui',
|
||||
active_turn_token=identity.get('token'),
|
||||
)
|
||||
else:
|
||||
stamp_message_source(message, source)
|
||||
return message
|
||||
|
||||
|
||||
def _settle_current_turn_boundary(previous_context, result_messages, identity, msg_text, source):
|
||||
"""Insert the pending turn before assistant/tool output when it is absent."""
|
||||
result_messages = list(result_messages or [])
|
||||
if not result_messages or not isinstance(identity, dict):
|
||||
return result_messages
|
||||
_checkpoint_idx = _find_active_turn_checkpoint_index(
|
||||
result_messages,
|
||||
previous_context,
|
||||
identity,
|
||||
msg_text,
|
||||
)
|
||||
if _checkpoint_idx is not None:
|
||||
_mark_active_turn_checkpoint(result_messages[_checkpoint_idx], identity)
|
||||
return result_messages
|
||||
previous_context = list(previous_context or [])
|
||||
if _messages_have_prefix(result_messages, previous_context):
|
||||
insert_at = len(previous_context)
|
||||
elif _active_turn_boundary_is_valid(identity):
|
||||
insert_at = identity['current_turn_user_idx'] - len(previous_context)
|
||||
if insert_at < 0 or insert_at > len(result_messages):
|
||||
insert_at = identity['current_turn_user_idx']
|
||||
if insert_at < 0 or insert_at > len(result_messages):
|
||||
insert_at = None
|
||||
else:
|
||||
insert_at = None
|
||||
if insert_at is None and all(
|
||||
_is_context_compression_marker(message)
|
||||
or (isinstance(message, dict) and message.get('role') in ('assistant', 'tool'))
|
||||
for message in result_messages
|
||||
):
|
||||
insert_at = 0
|
||||
while insert_at < len(result_messages) and _is_context_compression_marker(result_messages[insert_at]):
|
||||
insert_at += 1
|
||||
if insert_at is None:
|
||||
return result_messages
|
||||
return (
|
||||
result_messages[:insert_at]
|
||||
+ [_materialize_active_turn_user(identity, msg_text, source)]
|
||||
+ result_messages[insert_at:]
|
||||
)
|
||||
|
||||
|
||||
def _align_current_turn_display(previous_display, previous_context, identity):
|
||||
"""Make a context-only exact checkpoint visible before shared settlement."""
|
||||
display = list(previous_display or [])
|
||||
context = list(previous_context or [])
|
||||
if not isinstance(identity, dict):
|
||||
return display, context
|
||||
display, _ = _mark_active_turn_checkpoint_in_history(
|
||||
display,
|
||||
identity,
|
||||
identity.get('text'),
|
||||
allow_index_fallback=False,
|
||||
)
|
||||
if _active_turn_has_checkpoint(display, identity):
|
||||
return display, context
|
||||
checkpoint = _owner_projection_current_turn_row(
|
||||
context,
|
||||
identity,
|
||||
)
|
||||
if checkpoint is None and identity.get('token'):
|
||||
checkpoint = _materialize_active_turn_user(
|
||||
identity,
|
||||
identity.get('text'),
|
||||
identity.get('source') or 'webui',
|
||||
)
|
||||
context.append(copy.deepcopy(checkpoint))
|
||||
if checkpoint is not None and not _active_turn_has_checkpoint(display, identity):
|
||||
display.append(copy.deepcopy(checkpoint))
|
||||
return display, context
|
||||
|
||||
|
||||
def _prepare_marker_clean_writeback(
|
||||
previous_context_messages,
|
||||
result_messages,
|
||||
active_turn_identity=None,
|
||||
):
|
||||
"""Return marker-cleaned rows, next context rows, and nudge provenance."""
|
||||
cleaned, has_verification_nudge = _clean_synthetic_control_messages_with_provenance(
|
||||
result_messages
|
||||
)
|
||||
provenance = {
|
||||
'verification_nudge_seen': has_verification_nudge,
|
||||
'active_turn_identity': copy.deepcopy(active_turn_identity),
|
||||
}
|
||||
if not result_messages:
|
||||
return (
|
||||
cleaned,
|
||||
list(previous_context_messages or []),
|
||||
provenance,
|
||||
)
|
||||
if cleaned:
|
||||
return (
|
||||
cleaned,
|
||||
_restore_reasoning_metadata(previous_context_messages, cleaned),
|
||||
provenance,
|
||||
)
|
||||
return [], list(previous_context_messages or []), provenance
|
||||
|
||||
|
||||
def _settle_result_messages(
|
||||
session,
|
||||
previous_messages,
|
||||
previous_context_messages,
|
||||
result_messages,
|
||||
msg_text,
|
||||
source,
|
||||
active_turn_identity,
|
||||
):
|
||||
(
|
||||
result_messages,
|
||||
next_context_messages,
|
||||
verification_nudge_provenance,
|
||||
) = _prepare_marker_clean_writeback(
|
||||
previous_context_messages,
|
||||
result_messages,
|
||||
active_turn_identity,
|
||||
)
|
||||
if result_messages:
|
||||
_assign_stable_message_ids(
|
||||
result_messages,
|
||||
previous_messages,
|
||||
previous_context_messages,
|
||||
)
|
||||
next_context_messages = _dedupe_replayed_context_messages(
|
||||
previous_context_messages,
|
||||
next_context_messages,
|
||||
msg_text,
|
||||
)
|
||||
next_context_messages = _settle_current_turn_boundary(
|
||||
previous_context_messages,
|
||||
next_context_messages,
|
||||
active_turn_identity,
|
||||
msg_text,
|
||||
source,
|
||||
)
|
||||
session.context_messages = (
|
||||
_deduplicate_context_messages(next_context_messages)
|
||||
if result_messages
|
||||
else list(next_context_messages or [])
|
||||
)
|
||||
if result_messages:
|
||||
session.context_messages = _settle_current_turn_boundary(
|
||||
previous_context_messages,
|
||||
session.context_messages,
|
||||
active_turn_identity,
|
||||
msg_text,
|
||||
source,
|
||||
)
|
||||
previous_display_for_writeback, session.context_messages = _align_current_turn_display(
|
||||
previous_messages,
|
||||
session.context_messages,
|
||||
active_turn_identity,
|
||||
)
|
||||
session.messages = _merge_display_messages_after_agent_result(
|
||||
previous_display_for_writeback,
|
||||
previous_context_messages,
|
||||
_restore_display_reasoning_metadata(previous_messages, result_messages),
|
||||
msg_text,
|
||||
source=source,
|
||||
verification_nudge_provenance=verification_nudge_provenance,
|
||||
)
|
||||
_compact_session_image_parts_for_persistence(session)
|
||||
_advance_truncation_watermark_after_commit(session) # #3831
|
||||
return result_messages
|
||||
|
||||
|
||||
def _current_turn_already_has_visible_assistant_answer(messages, *, active_turn_identity=None):
|
||||
"""Return True only when the token-owned current turn already has visible assistant prose."""
|
||||
if not isinstance(active_turn_identity, dict) or not active_turn_identity.get('token'):
|
||||
return False
|
||||
current_turn_seen = False
|
||||
for msg in messages or []:
|
||||
if not isinstance(msg, dict) or _is_synthetic_control_message(msg):
|
||||
continue
|
||||
if _active_turn_token_matches(msg, active_turn_identity):
|
||||
current_turn_seen = True
|
||||
continue
|
||||
if not current_turn_seen:
|
||||
continue
|
||||
role = msg.get('role')
|
||||
if role == 'assistant' and _assistant_message_has_final_visible_text(msg) and not msg.get('_error'):
|
||||
return True
|
||||
if role == 'user':
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _agent_result_tool_limit_reached(result) -> bool:
|
||||
"""Return True when current-turn metadata says the tool iteration cap fired."""
|
||||
if not isinstance(result, dict):
|
||||
@@ -4653,6 +5028,7 @@ def _deduplicate_context_messages(messages):
|
||||
return messages
|
||||
seen = set()
|
||||
deduped = []
|
||||
user_exact_index = {}
|
||||
for msg in messages:
|
||||
if _is_context_compression_marker(msg):
|
||||
marker_key = (
|
||||
@@ -4671,6 +5047,26 @@ def _deduplicate_context_messages(messages):
|
||||
deduped.append(msg)
|
||||
continue
|
||||
key = _message_identity(msg)
|
||||
if isinstance(msg, dict) and msg.get('role') == 'user' and key is not None:
|
||||
user_exact_key = (
|
||||
key,
|
||||
msg.get('timestamp'),
|
||||
msg.get('_ts'),
|
||||
msg.get('_source') or 'webui',
|
||||
json.dumps(msg.get('attachments') or [], sort_keys=True, ensure_ascii=False),
|
||||
)
|
||||
prior_exact_idx = user_exact_index.get(user_exact_key)
|
||||
if msg.get('_active_turn_token'):
|
||||
if prior_exact_idx is not None:
|
||||
deduped[prior_exact_idx] = msg
|
||||
continue
|
||||
if key in seen:
|
||||
deduped.append(msg)
|
||||
user_exact_index[user_exact_key] = len(deduped) - 1
|
||||
continue
|
||||
elif prior_exact_idx is not None:
|
||||
continue
|
||||
user_exact_index[user_exact_key] = len(deduped)
|
||||
if key is not None and key in seen:
|
||||
continue
|
||||
if key is not None:
|
||||
@@ -5174,6 +5570,19 @@ def _dedupe_replayed_context_messages(previous_context, result_messages, msg_tex
|
||||
if candidates:
|
||||
candidates = _strip_replayed_context_items(previous_context, candidates)
|
||||
return previous_context + candidates
|
||||
assistant_or_tool_only_result = bool(result_messages) and all(
|
||||
_is_context_compression_marker(m)
|
||||
or (
|
||||
isinstance(m, dict)
|
||||
and m.get('role') in ('assistant', 'tool')
|
||||
)
|
||||
for m in result_messages
|
||||
)
|
||||
if assistant_or_tool_only_result:
|
||||
candidates = _strip_replayed_prefix(previous_context, result_messages)
|
||||
if candidates:
|
||||
candidates = _strip_replayed_context_items(previous_context, candidates)
|
||||
return previous_context + candidates
|
||||
return result_messages
|
||||
candidates = result_messages[len(previous_context):]
|
||||
# Strip stale merges only from the new-turn candidate slice so that
|
||||
@@ -5681,7 +6090,14 @@ def _advance_truncation_watermark_after_commit(session) -> None:
|
||||
session.truncation_watermark = time.time()
|
||||
|
||||
|
||||
def _merge_display_messages_after_agent_result(previous_display, previous_context, result_messages, msg_text, source: str = "webui"):
|
||||
def _merge_display_messages_after_agent_result(
|
||||
previous_display,
|
||||
previous_context,
|
||||
result_messages,
|
||||
msg_text,
|
||||
source: str = "webui",
|
||||
verification_nudge_provenance=None,
|
||||
):
|
||||
"""Keep UI transcript durable while allowing model context to compact.
|
||||
|
||||
If Hermes Agent returns a normal append-only history, append that delta to
|
||||
@@ -5730,6 +6146,25 @@ def _merge_display_messages_after_agent_result(previous_display, previous_contex
|
||||
previous_display = _deduped
|
||||
previous_context = list(previous_context or [])
|
||||
result_messages = list(result_messages or [])
|
||||
if isinstance(verification_nudge_provenance, dict):
|
||||
_writeback_provenance = verification_nudge_provenance
|
||||
else:
|
||||
_writeback_provenance = {
|
||||
'verification_nudge_seen': bool(verification_nudge_provenance),
|
||||
'active_turn_identity': None,
|
||||
}
|
||||
_verification_nudge_seen = _writeback_provenance.get('verification_nudge_seen', False) or any(
|
||||
isinstance(message, dict)
|
||||
and message.get('role') == 'user'
|
||||
and _is_synthetic_control_message(message)
|
||||
for message in result_messages
|
||||
)
|
||||
_active_turn_identity = _writeback_provenance.get('active_turn_identity')
|
||||
previous_display, _ = _align_current_turn_display(
|
||||
previous_display,
|
||||
previous_context,
|
||||
_active_turn_identity,
|
||||
)
|
||||
# Same marker filter for the model-history inputs: the synthetic verify-loop
|
||||
# answer/nudge live in the agent's returned messages and prior context, and
|
||||
# would otherwise slip into the merged transcript as a real delta. (#5334)
|
||||
@@ -5887,7 +6322,19 @@ def _merge_display_messages_after_agent_result(previous_display, previous_contex
|
||||
candidates = _strip_replayed_prefix(previous_context, candidates)
|
||||
else:
|
||||
current_user_idx = _find_current_user_turn(result_messages, msg_text)
|
||||
turn_candidates = result_messages[current_user_idx:] if current_user_idx is not None else []
|
||||
assistant_or_tool_only_result = bool(result_messages) and all(
|
||||
_is_context_compression_marker(m)
|
||||
or (
|
||||
isinstance(m, dict)
|
||||
and m.get('role') in ('assistant', 'tool')
|
||||
)
|
||||
for m in result_messages
|
||||
)
|
||||
turn_candidates = (
|
||||
result_messages[current_user_idx:]
|
||||
if current_user_idx is not None
|
||||
else result_messages if assistant_or_tool_only_result else []
|
||||
)
|
||||
# Normalize stale merges only in the current-turn slice.
|
||||
if msg_text and previous_user_tail:
|
||||
turn_candidates = _strip_stale_user_merge_from_messages(
|
||||
@@ -5905,17 +6352,24 @@ def _merge_display_messages_after_agent_result(previous_display, previous_contex
|
||||
_message_identity(m) == current_user_key or _looks_like_current_user_turn(m, msg_text)
|
||||
for m in candidates
|
||||
)
|
||||
current_user_already_checkpointed = bool(
|
||||
merged
|
||||
current_user_already_checkpointed = (
|
||||
_active_turn_has_checkpoint(merged, _active_turn_identity)
|
||||
if _active_turn_identity
|
||||
else False
|
||||
)
|
||||
verification_user_already_checkpointed = bool(
|
||||
_verification_nudge_seen
|
||||
and _active_turn_identity
|
||||
and (
|
||||
_message_identity(merged[-1]) == current_user_key
|
||||
or _looks_like_current_user_turn(merged[-1], msg_text)
|
||||
_active_turn_has_checkpoint(previous_display, _active_turn_identity)
|
||||
or _active_turn_has_checkpoint(previous_context, _active_turn_identity)
|
||||
)
|
||||
)
|
||||
if (
|
||||
current_user_key is not None
|
||||
and not current_user_in_candidates
|
||||
and not current_user_already_checkpointed
|
||||
and not verification_user_already_checkpointed
|
||||
and any(
|
||||
isinstance(m, dict) and m.get('role') in ('assistant', 'tool')
|
||||
for m in candidates
|
||||
@@ -5927,8 +6381,9 @@ def _merge_display_messages_after_agent_result(previous_display, previous_contex
|
||||
# directly would make the assistant bubble appear attached to the prior
|
||||
# exchange and then clear the pending prompt. Materialize the current
|
||||
# turn at the transcript boundary before the assistant/tool response.
|
||||
current_user_msg = {'role': 'user', 'content': msg_text}
|
||||
stamp_message_source(current_user_msg, source)
|
||||
current_user_msg = _materialize_active_turn_user(
|
||||
_active_turn_identity, msg_text, source
|
||||
)
|
||||
insert_at = 0
|
||||
while insert_at < len(candidates) and _is_context_compression_marker(candidates[insert_at]):
|
||||
insert_at += 1
|
||||
@@ -6057,12 +6512,25 @@ def _turn_transcript_lacks_final_assistant_answer(
|
||||
msg_text,
|
||||
source: str = "webui",
|
||||
drop_replayed_assistant: bool = False,
|
||||
active_turn_identity=None,
|
||||
) -> bool:
|
||||
"""Return True when an already-merged transcript still lacks a final assistant answer."""
|
||||
merged_messages = list(merged_messages or [])
|
||||
previous_display = list(previous_display or [])
|
||||
current_user_idx = _find_current_user_turn(merged_messages, msg_text)
|
||||
if current_user_idx is None or current_user_idx < len(previous_display):
|
||||
current_user_token_idx = next(
|
||||
(
|
||||
idx for idx, message in enumerate(merged_messages)
|
||||
if _active_turn_token_matches(message, active_turn_identity)
|
||||
),
|
||||
None,
|
||||
)
|
||||
current_user_idx = current_user_token_idx
|
||||
if current_user_idx is None:
|
||||
current_user_idx = _find_current_user_turn(merged_messages, msg_text)
|
||||
if current_user_idx is None or (
|
||||
current_user_token_idx is None
|
||||
and current_user_idx < len(previous_display)
|
||||
):
|
||||
# The active turn lives after the durable transcript boundary. If the
|
||||
# merged display only exposes an older user row, materialize the pending
|
||||
# prompt so a replayed assistant row cannot satisfy the wrong turn.
|
||||
@@ -6109,15 +6577,26 @@ def _merged_transcript_lacks_final_assistant_answer(
|
||||
msg_text,
|
||||
source: str = "webui",
|
||||
drop_replayed_assistant: bool = False,
|
||||
active_turn_identity=None,
|
||||
) -> bool:
|
||||
"""Return True when the current turn still lacks a final assistant answer."""
|
||||
previous_display = list(previous_display or [])
|
||||
_verification_nudge_seen = any(
|
||||
isinstance(message, dict)
|
||||
and message.get('role') == 'user'
|
||||
and _is_synthetic_control_message(message)
|
||||
for message in result_messages or []
|
||||
)
|
||||
merged_messages = _merge_display_messages_after_agent_result(
|
||||
previous_display,
|
||||
previous_context,
|
||||
_restore_reasoning_metadata(previous_display, result_messages),
|
||||
msg_text,
|
||||
source=source,
|
||||
verification_nudge_provenance={
|
||||
'verification_nudge_seen': _verification_nudge_seen,
|
||||
'active_turn_identity': active_turn_identity,
|
||||
},
|
||||
)
|
||||
return _turn_transcript_lacks_final_assistant_answer(
|
||||
merged_messages,
|
||||
@@ -6125,6 +6604,7 @@ def _merged_transcript_lacks_final_assistant_answer(
|
||||
msg_text,
|
||||
source=source,
|
||||
drop_replayed_assistant=drop_replayed_assistant,
|
||||
active_turn_identity=active_turn_identity,
|
||||
)
|
||||
|
||||
|
||||
@@ -7607,6 +8087,7 @@ def _run_agent_streaming(
|
||||
_turn_session_identity_tokens = _set_turn_session_identity(session_id)
|
||||
s = get_session(session_id)
|
||||
_turn_pending_source = getattr(s, 'pending_user_source', None) or 'webui'
|
||||
_active_turn_identity = _active_turn_authority(s, stream_id, msg_text)
|
||||
update_active_run(stream_id, phase="running", session_id=session_id)
|
||||
s.workspace = str(Path(workspace).expanduser().resolve())
|
||||
_last_persisted_model = None
|
||||
@@ -8988,12 +9469,18 @@ def _run_agent_streaming(
|
||||
state_messages=_external_state_messages,
|
||||
) or []
|
||||
)
|
||||
_previous_context_messages = _new_turn_context_from_messages(
|
||||
_previous_owner_context_messages = list(
|
||||
reconciled_state_db_messages_for_session(
|
||||
s,
|
||||
prefer_context=True,
|
||||
state_messages=_external_state_messages,
|
||||
),
|
||||
) or []
|
||||
)
|
||||
_previous_owner_context_messages = _deduplicate_context_messages(
|
||||
_previous_owner_context_messages
|
||||
)
|
||||
_previous_context_messages = _new_turn_context_from_messages(
|
||||
_previous_owner_context_messages,
|
||||
msg_text,
|
||||
)
|
||||
# Dedup before feeding to agent — merge_session_messages_append_only
|
||||
@@ -9066,6 +9553,7 @@ def _run_agent_streaming(
|
||||
),
|
||||
task_id=session_id,
|
||||
persist_user_message=msg_text,
|
||||
persist_user_timestamp=getattr(s, 'pending_started_at', None),
|
||||
)
|
||||
# Only pass moa_config when a /moa override is actually active, so a
|
||||
# normal send never trips a TypeError on an older hermes-agent whose
|
||||
@@ -9101,6 +9589,11 @@ def _run_agent_streaming(
|
||||
)
|
||||
_run_conversation_kwargs["user_message"] = user_message
|
||||
result = agent.run_conversation(**_run_conversation_kwargs)
|
||||
_active_turn_identity = _resolve_active_turn_authority(
|
||||
_active_turn_identity,
|
||||
result=result,
|
||||
agent=agent,
|
||||
)
|
||||
# #4729: the run is done — flush any reasoning tail still in the coalescing
|
||||
# buffer (the agent never calls reasoning_callback(None), and a turn can end on
|
||||
# reasoning with no trailing token/tool boundary to trigger a flush) so the last
|
||||
@@ -9192,7 +9685,9 @@ def _run_agent_streaming(
|
||||
return
|
||||
with _stream_writeback_stage(_writeback_timings, "merge_result"):
|
||||
_tool_limit_reached = _agent_result_tool_limit_reached(result)
|
||||
_result_messages = result.get('messages') or _previous_context_messages
|
||||
_result_messages = result.get('messages')
|
||||
if _result_messages is None:
|
||||
_result_messages = _previous_context_messages
|
||||
_result_messages = _drop_synthetic_max_iteration_summary_requests(
|
||||
_result_messages,
|
||||
enabled=_tool_limit_reached,
|
||||
@@ -9231,35 +9726,15 @@ def _run_agent_streaming(
|
||||
logger.debug("Failed to append cancelled turn journal event", exc_info=True)
|
||||
put('cancel', _cancel_event_payload('Cancelled by user'))
|
||||
return
|
||||
_next_context_messages = _restore_reasoning_metadata(
|
||||
_previous_context_messages,
|
||||
_result_messages,
|
||||
)
|
||||
# Stamp stable ids on the shared result rows AFTER the context
|
||||
# restore (so carried-forward ids survive) and BEFORE the
|
||||
# dedupe/merge build both arrays — including before
|
||||
# _dedupe_replayed_context_messages deep-copies any
|
||||
# stale-user repaired boundary row — so display and
|
||||
# model-context copies of each row share an id for the
|
||||
# fork/truncate aligner.
|
||||
_assign_stable_message_ids(
|
||||
_result_messages, _previous_messages, _previous_context_messages
|
||||
)
|
||||
_next_context_messages = _dedupe_replayed_context_messages(
|
||||
_previous_context_messages,
|
||||
_next_context_messages,
|
||||
msg_text,
|
||||
)
|
||||
s.context_messages = _deduplicate_context_messages(_next_context_messages)
|
||||
s.messages = _merge_display_messages_after_agent_result(
|
||||
_result_messages = _settle_result_messages(
|
||||
s,
|
||||
_previous_messages,
|
||||
_previous_context_messages,
|
||||
_restore_display_reasoning_metadata(_previous_messages, _result_messages),
|
||||
_previous_owner_context_messages,
|
||||
_result_messages,
|
||||
msg_text,
|
||||
source=getattr(s, 'pending_user_source', None) or 'webui',
|
||||
_turn_pending_source,
|
||||
_active_turn_identity,
|
||||
)
|
||||
_compact_session_image_parts_for_persistence(s)
|
||||
_advance_truncation_watermark_after_commit(s) # #3831
|
||||
# Strip XML tool-call blocks from assistant message content.
|
||||
# DeepSeek and some other providers emit <function_calls>...</function_calls>
|
||||
# in the raw response text; this must be removed before the content is
|
||||
@@ -9403,7 +9878,7 @@ def _run_agent_streaming(
|
||||
# workspace-aware helper from this branch while still
|
||||
# preserving the pre-turn length for downstream self-heal
|
||||
# checks introduced on master.
|
||||
_all_result_messages = result.get('messages') or []
|
||||
_all_result_messages = _drop_synthetic_control_messages(result.get('messages') or [])
|
||||
_prev_len = len(_previous_context_messages)
|
||||
_assistant_added = _assistant_reply_added_after_current_turn(
|
||||
_all_result_messages,
|
||||
@@ -9434,12 +9909,27 @@ def _run_agent_streaming(
|
||||
)
|
||||
_saved_transcript_lacks_final_answer = _merged_transcript_lacks_final_assistant_answer(
|
||||
_previous_messages,
|
||||
_previous_context_messages,
|
||||
_previous_owner_context_messages,
|
||||
_all_result_messages,
|
||||
msg_text,
|
||||
source=getattr(s, 'pending_user_source', None) or 'webui',
|
||||
drop_replayed_assistant=_drop_replayed_assistant,
|
||||
active_turn_identity=_active_turn_identity,
|
||||
)
|
||||
if (
|
||||
not _all_result_messages
|
||||
and _current_turn_already_has_visible_assistant_answer(
|
||||
_align_current_turn_display(
|
||||
_previous_messages,
|
||||
_previous_owner_context_messages,
|
||||
_active_turn_identity,
|
||||
)[0],
|
||||
active_turn_identity=_active_turn_identity,
|
||||
)
|
||||
):
|
||||
_saved_transcript_lacks_final_answer = False
|
||||
if not _assistant_added and not _saved_transcript_lacks_final_answer:
|
||||
_assistant_added = True
|
||||
_is_agent_result_terminal = _agent_result_terminal_failure(result)
|
||||
_terminal_failure = (
|
||||
_captured_terminal_failure
|
||||
@@ -9547,10 +10037,16 @@ def _run_agent_streaming(
|
||||
),
|
||||
task_id=session_id,
|
||||
persist_user_message=msg_text,
|
||||
persist_user_timestamp=getattr(s, 'pending_started_at', None),
|
||||
)
|
||||
if moa_config is not None:
|
||||
_heal_kwargs["moa_config"] = moa_config
|
||||
_heal_result = agent.run_conversation(**_heal_kwargs)
|
||||
_active_turn_identity = _resolve_active_turn_authority(
|
||||
_active_turn_identity,
|
||||
result=_heal_result,
|
||||
agent=agent,
|
||||
)
|
||||
_heal_all_msgs = _heal_result.get('messages') or []
|
||||
_heal_ok = _has_new_assistant_reply(_heal_all_msgs, _prev_len) or _token_sent
|
||||
except Exception as _retry_exc:
|
||||
@@ -9571,36 +10067,22 @@ def _run_agent_streaming(
|
||||
# evaluates False on next conceptual pass.
|
||||
# Since we're in a flat block, directly run the
|
||||
# post-result merge logic here.
|
||||
_result_messages = result.get('messages') or _previous_context_messages
|
||||
_result_messages = result.get('messages')
|
||||
if _result_messages is None:
|
||||
_result_messages = _previous_context_messages
|
||||
_result_messages = _drop_synthetic_max_iteration_summary_requests(
|
||||
_result_messages,
|
||||
enabled=_agent_result_tool_limit_reached(result),
|
||||
)
|
||||
_next_context_messages = _restore_reasoning_metadata(
|
||||
_previous_context_messages,
|
||||
_result_messages,
|
||||
)
|
||||
# Mint ids on the shared result rows BEFORE dedupe
|
||||
# deep-copies any stale-user boundary row, so both
|
||||
# arrays share the id (#5564).
|
||||
_assign_stable_message_ids(
|
||||
_result_messages, _previous_messages, _previous_context_messages
|
||||
)
|
||||
_next_context_messages = _dedupe_replayed_context_messages(
|
||||
_previous_context_messages,
|
||||
_next_context_messages,
|
||||
msg_text,
|
||||
)
|
||||
s.context_messages = _deduplicate_context_messages(_next_context_messages)
|
||||
s.messages = _merge_display_messages_after_agent_result(
|
||||
_result_messages = _settle_result_messages(
|
||||
s,
|
||||
_previous_messages,
|
||||
_previous_context_messages,
|
||||
_restore_reasoning_metadata(_previous_messages, _result_messages),
|
||||
_previous_owner_context_messages,
|
||||
_result_messages,
|
||||
msg_text,
|
||||
source=getattr(s, 'pending_user_source', None) or 'webui',
|
||||
_turn_pending_source,
|
||||
_active_turn_identity,
|
||||
)
|
||||
_compact_session_image_parts_for_persistence(s)
|
||||
_advance_truncation_watermark_after_commit(s) # #3831
|
||||
# normal post-result persistence path by
|
||||
# leaving _assistant_added truthy (set below).
|
||||
_assistant_added = True # prevent re-entering guard
|
||||
@@ -10779,10 +11261,16 @@ def _run_agent_streaming(
|
||||
),
|
||||
task_id=session_id,
|
||||
persist_user_message=msg_text,
|
||||
persist_user_timestamp=getattr(s, 'pending_started_at', None),
|
||||
)
|
||||
if moa_config is not None:
|
||||
_heal_kwargs2["moa_config"] = moa_config
|
||||
_heal_result = _heal_agent.run_conversation(**_heal_kwargs2)
|
||||
_active_turn_identity = _resolve_active_turn_authority(
|
||||
_active_turn_identity,
|
||||
result=_heal_result,
|
||||
agent=_heal_agent,
|
||||
)
|
||||
# Retry succeeded — persist the result normally
|
||||
if s is not None:
|
||||
if _checkpoint_stop is not None:
|
||||
@@ -10799,31 +11287,18 @@ def _run_agent_streaming(
|
||||
getattr(s, 'active_stream_id', None),
|
||||
)
|
||||
return
|
||||
_result_messages = _heal_result.get('messages') or _previous_context_messages
|
||||
_next_context_messages = _restore_reasoning_metadata(
|
||||
_previous_context_messages, _result_messages,
|
||||
)
|
||||
# Mint ids on the shared result rows BEFORE dedupe
|
||||
# deep-copies any stale-user boundary row, so both
|
||||
# arrays share the id (#5564).
|
||||
_assign_stable_message_ids(
|
||||
_result_messages, _previous_messages, _previous_context_messages
|
||||
)
|
||||
_next_context_messages = _dedupe_replayed_context_messages(
|
||||
_previous_context_messages,
|
||||
_next_context_messages,
|
||||
msg_text,
|
||||
)
|
||||
s.context_messages = _deduplicate_context_messages(_next_context_messages)
|
||||
s.messages = _merge_display_messages_after_agent_result(
|
||||
_result_messages = _heal_result.get('messages')
|
||||
if _result_messages is None:
|
||||
_result_messages = _previous_context_messages
|
||||
_result_messages = _settle_result_messages(
|
||||
s,
|
||||
_previous_messages,
|
||||
_previous_context_messages,
|
||||
_restore_reasoning_metadata(_previous_messages, _result_messages),
|
||||
_previous_owner_context_messages,
|
||||
_result_messages,
|
||||
msg_text,
|
||||
source=getattr(s, 'pending_user_source', None) or 'webui',
|
||||
_turn_pending_source,
|
||||
_active_turn_identity,
|
||||
)
|
||||
_compact_session_image_parts_for_persistence(s)
|
||||
_advance_truncation_watermark_after_commit(s) # #3831
|
||||
s.save()
|
||||
logger.info('[webui] self-heal (except path): retry succeeded')
|
||||
return # skip error emission
|
||||
|
||||
@@ -97,10 +97,16 @@ def test_reasoning_buffer_flushed_at_every_boundary():
|
||||
# terminal catch-all: flush right after the agent run returns, AND a guaranteed-exit
|
||||
# flush in the inner finally so exception / retry paths (which bypass the post-run
|
||||
# flush) still emit the tail before the outer apperror.
|
||||
assert re.search(
|
||||
r"agent\.run_conversation\(\*\*_run_conversation_kwargs\)\n\s*# #4729",
|
||||
STREAMING,
|
||||
), "must flush after agent.run_conversation() returns (success path, before done)"
|
||||
run_idx = STREAMING.index("result = agent.run_conversation(**_run_conversation_kwargs)")
|
||||
resolve_idx = STREAMING.index(
|
||||
"_active_turn_identity = _resolve_active_turn_authority(",
|
||||
run_idx,
|
||||
)
|
||||
flush_idx = STREAMING.index("_flush_reasoning_buffer()", resolve_idx)
|
||||
assert run_idx < resolve_idx < flush_idx, (
|
||||
"must flush after agent.run_conversation() returns, after turn authority is resolved, "
|
||||
"and before terminal SSE delivery"
|
||||
)
|
||||
# the inner finally must also flush (covers exception/retry exits)
|
||||
fin = STREAMING[STREAMING.index(" finally:\n # #4729"):]
|
||||
fin = fin[:700]
|
||||
|
||||
@@ -93,7 +93,14 @@ def test_turn_evaluator_materializes_pending_user_after_display_boundary():
|
||||
def test_merged_wrapper_delegates_to_turn_evaluator():
|
||||
calls = []
|
||||
|
||||
def _fake_evaluator(merged_messages, previous_display, msg_text, source="webui", drop_replayed_assistant=False):
|
||||
def _fake_evaluator(
|
||||
merged_messages,
|
||||
previous_display,
|
||||
msg_text,
|
||||
source="webui",
|
||||
drop_replayed_assistant=False,
|
||||
active_turn_identity=None,
|
||||
):
|
||||
calls.append(
|
||||
{
|
||||
"merged_len": len(list(merged_messages or [])),
|
||||
@@ -101,6 +108,7 @@ def test_merged_wrapper_delegates_to_turn_evaluator():
|
||||
"msg_text": msg_text,
|
||||
"source": source,
|
||||
"drop_replayed_assistant": drop_replayed_assistant,
|
||||
"active_turn_identity": active_turn_identity,
|
||||
}
|
||||
)
|
||||
return True
|
||||
@@ -125,4 +133,5 @@ def test_merged_wrapper_delegates_to_turn_evaluator():
|
||||
assert calls[0]["msg_text"] == "hello"
|
||||
assert calls[0]["source"] == "cli"
|
||||
assert calls[0]["drop_replayed_assistant"] is True
|
||||
assert calls[0]["active_turn_identity"] is None
|
||||
assert calls[0]["merged_len"] >= 1
|
||||
@@ -0,0 +1,475 @@
|
||||
"""Regression coverage for issue #6481 verification follow-up contracts."""
|
||||
|
||||
import types
|
||||
|
||||
from api import streaming
|
||||
|
||||
|
||||
ATTEMPTED_ANSWER = "The failing test is fixed."
|
||||
CORRECTIVE_ANSWER = "Verification failed. I fixed the parser and reran the tests."
|
||||
VERIFICATION_EVIDENCE = {
|
||||
"status": "passed",
|
||||
"kind": "terminal",
|
||||
"scope": "workspace",
|
||||
"canonical_command": "pytest tests/test_app.py -q",
|
||||
}
|
||||
|
||||
|
||||
def _current_turn_prefix(*, include_attempted_answer):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Fix the failing test.",
|
||||
"_active_turn_token": "stream:turn",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "tool_use", "name": "terminal", "input": {"cmd": "pytest"}}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "terminal",
|
||||
"content": "Tests passed.",
|
||||
"verification_evidence": VERIFICATION_EVIDENCE,
|
||||
},
|
||||
]
|
||||
if include_attempted_answer:
|
||||
messages.append({"role": "assistant", "content": ATTEMPTED_ANSWER})
|
||||
return messages
|
||||
|
||||
|
||||
def _verification_followup(marker, corrective=CORRECTIVE_ANSWER):
|
||||
return [
|
||||
{"role": "user", "content": "[System: verify the workspace]", marker: True},
|
||||
{"role": "assistant", "content": corrective},
|
||||
]
|
||||
|
||||
|
||||
def _writeback_provenance(*, prompt, timestamp, current_turn_user_idx, turn_id="turn-current", token="stream:turn"):
|
||||
return {
|
||||
"verification_nudge_seen": True,
|
||||
"active_turn_identity": {
|
||||
"token": token,
|
||||
"text": prompt,
|
||||
"timestamp": timestamp,
|
||||
"source": "webui",
|
||||
"attachments": [],
|
||||
"current_turn_user_idx": current_turn_user_idx,
|
||||
"turn_id": turn_id,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _merge(
|
||||
result_messages,
|
||||
previous=None,
|
||||
previous_context=None,
|
||||
msg_text="Fix the failing test.",
|
||||
writeback_provenance=None,
|
||||
):
|
||||
previous = list(previous or [{"role": "user", "content": "Fix the failing test."}])
|
||||
previous_context = list(previous_context if previous_context is not None else previous)
|
||||
return streaming._merge_display_messages_after_agent_result(
|
||||
previous,
|
||||
previous_context,
|
||||
result_messages,
|
||||
msg_text,
|
||||
verification_nudge_provenance=writeback_provenance,
|
||||
)
|
||||
|
||||
|
||||
def _role_content_sequence(messages):
|
||||
return [(message["role"], message.get("content")) for message in messages]
|
||||
|
||||
|
||||
def _expected_current_turn(*, include_attempted_answer):
|
||||
expected = _current_turn_prefix(include_attempted_answer=include_attempted_answer)
|
||||
return expected + [{"role": "assistant", "content": CORRECTIVE_ANSWER}]
|
||||
|
||||
|
||||
def test_corrective_followup_survives_after_attempted_answer_for_both_markers():
|
||||
for marker in ("_verification_stop_synthetic", "_pre_verify_synthetic"):
|
||||
previous = _current_turn_prefix(include_attempted_answer=True)
|
||||
merged = _merge(
|
||||
previous + _verification_followup(marker),
|
||||
previous=previous,
|
||||
writeback_provenance=_writeback_provenance(
|
||||
prompt="Fix the failing test.",
|
||||
timestamp=1.0,
|
||||
current_turn_user_idx=0,
|
||||
),
|
||||
)
|
||||
|
||||
assert _role_content_sequence(merged) == _role_content_sequence(
|
||||
_expected_current_turn(include_attempted_answer=True)
|
||||
)
|
||||
assert [message.get("content") for message in merged].count("Fix the failing test.") == 1
|
||||
assert all(not streaming._is_synthetic_control_message(message) for message in merged)
|
||||
|
||||
|
||||
def test_delta_only_followup_preserves_corrective_answer():
|
||||
for marker in ("_verification_stop_synthetic", "_pre_verify_synthetic"):
|
||||
previous = _current_turn_prefix(include_attempted_answer=True)
|
||||
merged = _merge(
|
||||
_verification_followup(marker),
|
||||
previous=previous,
|
||||
writeback_provenance=_writeback_provenance(
|
||||
prompt="Fix the failing test.",
|
||||
timestamp=1.0,
|
||||
current_turn_user_idx=0,
|
||||
),
|
||||
)
|
||||
|
||||
assert _role_content_sequence(merged) == _role_content_sequence(
|
||||
_expected_current_turn(include_attempted_answer=True)
|
||||
)
|
||||
assert [message.get("content") for message in merged].count("Fix the failing test.") == 1
|
||||
|
||||
|
||||
def test_first_post_nudge_answer_survives_when_no_attempted_answer_exists():
|
||||
for marker in ("_verification_stop_synthetic", "_pre_verify_synthetic"):
|
||||
previous = _current_turn_prefix(include_attempted_answer=False)
|
||||
merged = _merge(
|
||||
previous + _verification_followup(marker),
|
||||
previous=previous,
|
||||
writeback_provenance=_writeback_provenance(
|
||||
prompt="Fix the failing test.",
|
||||
timestamp=1.0,
|
||||
current_turn_user_idx=0,
|
||||
),
|
||||
)
|
||||
|
||||
assert _role_content_sequence(merged) == _role_content_sequence(
|
||||
_expected_current_turn(include_attempted_answer=False)
|
||||
)
|
||||
assert [message.get("content") for message in merged].count("Fix the failing test.") == 1
|
||||
|
||||
|
||||
def test_unmarked_adjacent_assistant_rows_remain_unchanged():
|
||||
messages = [
|
||||
{"role": "user", "content": "Give me two alternatives."},
|
||||
{"role": "assistant", "content": "First alternative."},
|
||||
{"role": "assistant", "content": "Second alternative."},
|
||||
]
|
||||
|
||||
assert streaming._drop_synthetic_control_messages(messages) == messages
|
||||
|
||||
|
||||
def test_unmarked_assistant_only_new_turn_materializes_current_user_row():
|
||||
previous = [
|
||||
{"role": "user", "content": "Earlier request."},
|
||||
{"role": "assistant", "content": "Earlier answer."},
|
||||
]
|
||||
result = [{"role": "assistant", "content": "New answer."}]
|
||||
|
||||
merged = _merge(result, previous=previous, msg_text="New request.")
|
||||
|
||||
assert _role_content_sequence(merged) == [
|
||||
("user", "Earlier request."),
|
||||
("assistant", "Earlier answer."),
|
||||
("user", "New request."),
|
||||
("assistant", "New answer."),
|
||||
]
|
||||
|
||||
|
||||
def test_unmarked_repeated_identical_prompt_materializes_new_user_row():
|
||||
previous = [
|
||||
{"role": "user", "content": "Fix the failing test."},
|
||||
{"role": "assistant", "content": "The first attempt."},
|
||||
]
|
||||
result = [{"role": "assistant", "content": "The second attempt."}]
|
||||
|
||||
merged = _merge(result, previous=previous)
|
||||
|
||||
assert _role_content_sequence(merged) == [
|
||||
("user", "Fix the failing test."),
|
||||
("assistant", "The first attempt."),
|
||||
("user", "Fix the failing test."),
|
||||
("assistant", "The second attempt."),
|
||||
]
|
||||
|
||||
|
||||
def test_marked_repeated_old_prompt_materializes_deferred_current_turn():
|
||||
prompt = "Fix the failing test."
|
||||
previous = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.0},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
corrective = "Verification failed. I fixed the parser and reran the tests."
|
||||
|
||||
for marker in ("_verification_stop_synthetic", "_pre_verify_synthetic"):
|
||||
merged = _merge(
|
||||
previous
|
||||
+ [
|
||||
{"role": "user", "content": "[System: verify the workspace]", marker: True},
|
||||
{"role": "assistant", "content": corrective},
|
||||
],
|
||||
previous=previous,
|
||||
msg_text=prompt,
|
||||
writeback_provenance=_writeback_provenance(
|
||||
prompt=prompt,
|
||||
timestamp=2.0,
|
||||
current_turn_user_idx=len(previous),
|
||||
token="direct-stream:2",
|
||||
),
|
||||
)
|
||||
|
||||
assert _role_content_sequence(merged) == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", corrective),
|
||||
]
|
||||
assert [message.get("content") for message in merged].count(prompt) == 2
|
||||
|
||||
|
||||
def test_context_only_exact_checkpoint_does_not_suppress_display_boundary():
|
||||
prompt = "Fix the failing test."
|
||||
previous_display = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.0},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
previous_context = previous_display + [
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
"timestamp": 2.0,
|
||||
"_source": "webui",
|
||||
"attachments": [],
|
||||
"_active_turn_token": "direct-stream:2",
|
||||
},
|
||||
]
|
||||
corrective = "Verification failed. I fixed the parser and reran the tests."
|
||||
aligned_display, _ = streaming._align_current_turn_display(
|
||||
previous_display,
|
||||
previous_context,
|
||||
_writeback_provenance(
|
||||
prompt=prompt,
|
||||
timestamp=2.0,
|
||||
current_turn_user_idx=2,
|
||||
token="direct-stream:2",
|
||||
)["active_turn_identity"],
|
||||
)
|
||||
|
||||
merged = _merge(
|
||||
[
|
||||
{"role": "user", "content": "[System: verify the workspace]", "_verification_stop_synthetic": True},
|
||||
{"role": "assistant", "content": corrective},
|
||||
],
|
||||
previous=aligned_display,
|
||||
previous_context=previous_context,
|
||||
msg_text=prompt,
|
||||
writeback_provenance=_writeback_provenance(
|
||||
prompt=prompt,
|
||||
timestamp=2.0,
|
||||
current_turn_user_idx=2,
|
||||
token="direct-stream:2",
|
||||
),
|
||||
)
|
||||
|
||||
assert _role_content_sequence(merged) == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", corrective),
|
||||
]
|
||||
|
||||
|
||||
def test_divergent_context_index_does_not_relabel_same_text_display_row():
|
||||
prompt = "Fix the failing test."
|
||||
token = "direct-stream:2"
|
||||
previous_display = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.0},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
previous_context = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.0, "attachments": []},
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
"timestamp": 2.0,
|
||||
"attachments": [],
|
||||
"_active_turn_token": token,
|
||||
},
|
||||
]
|
||||
|
||||
aligned_display, _ = streaming._align_current_turn_display(
|
||||
previous_display,
|
||||
previous_context,
|
||||
_writeback_provenance(
|
||||
prompt=prompt,
|
||||
timestamp=2.0,
|
||||
current_turn_user_idx=0,
|
||||
token=token,
|
||||
)["active_turn_identity"],
|
||||
)
|
||||
|
||||
assert _role_content_sequence(aligned_display) == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
]
|
||||
assert aligned_display[0].get("_active_turn_token") is None
|
||||
assert aligned_display[-1]["_active_turn_token"] == token
|
||||
|
||||
|
||||
def test_missing_exact_checkpoint_materializes_current_turn_without_retokening_history():
|
||||
prompt = "Fix the failing test."
|
||||
token = "direct-stream:1.9"
|
||||
previous_display = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
previous_context = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1, "attachments": []},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
|
||||
aligned_display, aligned_context = streaming._align_current_turn_display(
|
||||
previous_display,
|
||||
previous_context,
|
||||
_writeback_provenance(
|
||||
prompt=prompt,
|
||||
timestamp=1.9,
|
||||
current_turn_user_idx=0,
|
||||
token=token,
|
||||
)["active_turn_identity"],
|
||||
)
|
||||
|
||||
expected = [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
]
|
||||
assert _role_content_sequence(aligned_display) == expected
|
||||
assert _role_content_sequence(aligned_context) == expected
|
||||
assert aligned_display[0].get("_active_turn_token") is None
|
||||
assert aligned_context[0].get("_active_turn_token") is None
|
||||
assert aligned_display[-1]["_active_turn_token"] == token
|
||||
assert aligned_context[-1]["_active_turn_token"] == token
|
||||
|
||||
|
||||
def test_shared_settlement_owner_contract():
|
||||
prompt = "Fix the failing test."
|
||||
previous = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
result_messages = previous + [
|
||||
{"role": "user", "content": "[System: verify the workspace]", "_verification_stop_synthetic": True},
|
||||
{"role": "assistant", "content": CORRECTIVE_ANSWER},
|
||||
]
|
||||
session = types.SimpleNamespace(messages=list(previous), context_messages=list(previous))
|
||||
|
||||
streaming._settle_result_messages(
|
||||
session,
|
||||
previous,
|
||||
previous,
|
||||
result_messages,
|
||||
prompt,
|
||||
"webui",
|
||||
_writeback_provenance(
|
||||
prompt=prompt,
|
||||
timestamp=1.9,
|
||||
current_turn_user_idx=len(previous),
|
||||
token="direct-stream:1.9",
|
||||
)["active_turn_identity"],
|
||||
)
|
||||
|
||||
expected = [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", CORRECTIVE_ANSWER),
|
||||
]
|
||||
assert _role_content_sequence(session.messages) == expected
|
||||
assert _role_content_sequence(session.context_messages) == expected
|
||||
|
||||
|
||||
def test_untokened_legacy_row_fails_closed():
|
||||
prompt = "Fix the failing test."
|
||||
previous_display = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
previous_context = previous_display + [
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
"timestamp": 1.9,
|
||||
"_source": "webui",
|
||||
"attachments": [],
|
||||
},
|
||||
]
|
||||
|
||||
merged = _merge(
|
||||
[
|
||||
{"role": "user", "content": "[System: verify the workspace]", "_verification_stop_synthetic": True},
|
||||
{"role": "assistant", "content": CORRECTIVE_ANSWER},
|
||||
],
|
||||
previous=previous_display,
|
||||
previous_context=previous_context,
|
||||
msg_text=prompt,
|
||||
writeback_provenance=_writeback_provenance(
|
||||
prompt=prompt,
|
||||
timestamp=1.9,
|
||||
current_turn_user_idx=2,
|
||||
token="direct-stream:1.9",
|
||||
),
|
||||
)
|
||||
|
||||
assert _role_content_sequence(merged) == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", CORRECTIVE_ANSWER),
|
||||
]
|
||||
|
||||
|
||||
def test_materialized_webui_user_omits_source():
|
||||
message = streaming._materialize_active_turn_user(
|
||||
_writeback_provenance(
|
||||
prompt="Fix the failing test.",
|
||||
timestamp=1.9,
|
||||
current_turn_user_idx=2,
|
||||
token="direct-stream:1.9",
|
||||
)["active_turn_identity"],
|
||||
"Fix the failing test.",
|
||||
"webui",
|
||||
)
|
||||
|
||||
assert message["role"] == "user"
|
||||
assert message["content"] == "Fix the failing test."
|
||||
assert message["_active_turn_token"] == "direct-stream:1.9"
|
||||
assert "_source" not in message
|
||||
|
||||
|
||||
def test_materialized_process_wakeup_user_has_wakeup_meta():
|
||||
wakeup_text = (
|
||||
"[IMPORTANT: Background process bg-1 completed (exit_code=0).\n"
|
||||
"Command: python worker.py\n"
|
||||
"Output:\n"
|
||||
"done]"
|
||||
)
|
||||
message = streaming._materialize_active_turn_user(
|
||||
{
|
||||
"token": "wakeup-stream:1.9",
|
||||
"text": wakeup_text,
|
||||
"timestamp": 1.9,
|
||||
"source": "process_wakeup",
|
||||
"attachments": [],
|
||||
"current_turn_user_idx": 0,
|
||||
"turn_id": "turn-wakeup",
|
||||
},
|
||||
wakeup_text,
|
||||
"process_wakeup",
|
||||
)
|
||||
|
||||
assert message["_source"] == "process_wakeup"
|
||||
assert message["_active_turn_token"] == "wakeup-stream:1.9"
|
||||
assert message["_wakeup_meta"] == {
|
||||
"type": "completion",
|
||||
"task_id": "bg-1",
|
||||
"command": "python worker.py",
|
||||
"exit_code": 0,
|
||||
}
|
||||
@@ -314,10 +314,10 @@ class TestIssue765FollowupHardening:
|
||||
t2.join(timeout=5)
|
||||
|
||||
assert not errors, f"Concurrent same-session saves should not fail: {errors}"
|
||||
assert len(replace_sources) == 2, f"Expected 2 replace calls, got {replace_sources}"
|
||||
assert len(replace_sources) >= 2, f"Expected replace calls, got {replace_sources}"
|
||||
assert len(set(replace_sources)) == 2, (
|
||||
"Concurrent same-session saves must use distinct temp files; "
|
||||
f"got {replace_sources}"
|
||||
"Concurrent same-session saves must use distinct temp files even if Windows-safe "
|
||||
f"replace retries one of them; got {replace_sources}"
|
||||
)
|
||||
data = json.loads(s.path.read_text(encoding="utf-8"))
|
||||
assert data["session_id"] == "same_sid"
|
||||
@@ -337,14 +337,14 @@ class TestIssue765FollowupHardening:
|
||||
"with _agent_lock:\n"
|
||||
" if not ephemeral and not _stream_writeback_is_current(s, stream_id):"
|
||||
)
|
||||
save_idx = src.find("_deduplicate_context_messages(_next_context_messages)")
|
||||
save_idx = src.find("_result_messages = _settle_result_messages(")
|
||||
|
||||
assert stop_idx != -1, "Success path must stop the checkpoint thread"
|
||||
assert join_idx != -1, "Success path must join the checkpoint thread"
|
||||
assert lock_idx != -1, "Success path must serialize mutation with _agent_lock"
|
||||
assert save_idx != -1, "Success path restore/mutation block not found"
|
||||
assert save_idx != -1, "Success path settlement block not found"
|
||||
assert stop_idx < join_idx < lock_idx <= save_idx, (
|
||||
"Checkpoint stop/join must happen before the success-path session mutation block"
|
||||
"Checkpoint stop/join must happen before the success-path settlement block"
|
||||
)
|
||||
|
||||
def test_silent_failure_path_does_not_reacquire_agent_lock(self):
|
||||
|
||||
@@ -29,11 +29,14 @@ def test_streaming_journals_sse_events_before_queue_delivery():
|
||||
def test_streaming_compacts_all_successful_agent_result_writebacks():
|
||||
src = Path("api/streaming.py").read_text(encoding="utf-8")
|
||||
run_src = src[src.index("def _run_agent_streaming("):]
|
||||
settle_src = src[src.index("def _settle_result_messages("):]
|
||||
settle_src = settle_src[: settle_src.index("\n\ndef _current_turn_already_has_visible_assistant_answer(")]
|
||||
|
||||
# Normal completion plus both credential self-heal retry-success paths must
|
||||
# each compact after committing their own result shape. The retries use
|
||||
# different local variable names, so guard the durable invariant itself.
|
||||
assert run_src.count("_compact_session_image_parts_for_persistence(s)") == 3
|
||||
# Normal completion plus both credential self-heal retry-success paths now
|
||||
# route through one shared settlement helper, which owns the compaction.
|
||||
assert "def _settle_result_messages(" in src
|
||||
assert "_compact_session_image_parts_for_persistence(session)" in settle_src
|
||||
assert run_src.count("_settle_result_messages(") == 3
|
||||
|
||||
|
||||
def test_visible_process_echo_compare_ignores_all_whitespace():
|
||||
|
||||
@@ -813,9 +813,9 @@ def test_streaming_restores_prior_reasoning_metadata_after_followup():
|
||||
src = (REPO / 'api' / 'streaming.py').read_text(encoding="utf-8")
|
||||
assert "def _restore_reasoning_metadata(" in src, \
|
||||
"streaming.py must define a helper to restore prior reasoning metadata"
|
||||
assert "_next_context_messages" in src and "s.context_messages" in src, \
|
||||
assert "next_context_messages" in src and "_deduplicate_context_messages(next_context_messages)" in src, \
|
||||
"streaming.py must restore prior reasoning metadata into model context"
|
||||
assert "s.messages = _merge_display_messages_after_agent_result(" in src, \
|
||||
assert "session.messages = _merge_display_messages_after_agent_result(" in src, \
|
||||
"streaming.py must merge restored result messages into the visible transcript"
|
||||
assert "updated_messages.insert(safe_pos, copy.deepcopy(prev_msg))" in src, \
|
||||
"streaming.py must reinsert dropped reasoning-only assistant messages"
|
||||
|
||||
@@ -174,7 +174,7 @@ def test_success_path_checks_stream_ownership_before_persisting_result():
|
||||
src = Path("api/streaming.py").read_text(encoding="utf-8")
|
||||
guard = "if not ephemeral and not _stream_writeback_is_current(s, stream_id):"
|
||||
guard_pos = src.find(guard)
|
||||
result_merge_pos = src.find("_result_messages = result.get('messages') or _previous_context_messages")
|
||||
result_merge_pos = src.find("_result_messages = result.get('messages')")
|
||||
compression_pos = src.find("Handle context compression side effects")
|
||||
|
||||
assert guard_pos != -1
|
||||
@@ -192,7 +192,7 @@ def test_self_heal_retry_success_checks_stream_ownership_before_writeback():
|
||||
guard = "if not ephemeral and not _stream_writeback_is_current(s, stream_id):"
|
||||
|
||||
assert guard in block
|
||||
assert block.index(guard) < block.index("_result_messages = _heal_result.get('messages') or _previous_context_messages")
|
||||
assert block.index(guard) < block.index("_result_messages = _heal_result.get('messages')")
|
||||
assert block.index(guard) < block.index("s.save()")
|
||||
|
||||
|
||||
|
||||
@@ -4,7 +4,10 @@ import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from api import models
|
||||
from api.process_event_utils import build_active_turn_token
|
||||
from api import streaming
|
||||
from api.models import Session
|
||||
|
||||
@@ -19,6 +22,12 @@ def _run_streaming_with_fake_agent(
|
||||
*,
|
||||
prior_messages=None,
|
||||
prior_context_messages=None,
|
||||
msg_text="Do the long task.",
|
||||
pending_started_at=1.0,
|
||||
current_turn_user_idx=None,
|
||||
turn_id="turn-current",
|
||||
agent_results=None,
|
||||
enable_auth_retry=False,
|
||||
):
|
||||
session_dir = tmp_path / "sessions"
|
||||
session_dir.mkdir()
|
||||
@@ -40,22 +49,31 @@ def _run_streaming_with_fake_agent(
|
||||
|
||||
session_id = "tool_limit_session"
|
||||
stream_id = "stream-tool-limit"
|
||||
session_messages = list(prior_messages or [])
|
||||
session_context_messages = list(prior_context_messages or [])
|
||||
for history in (session_messages, session_context_messages):
|
||||
for message in history:
|
||||
if isinstance(message, dict) and message.get("role") == "user":
|
||||
message.setdefault("timestamp", 1.0)
|
||||
message.setdefault("_source", "webui")
|
||||
message.setdefault("attachments", [])
|
||||
session = Session(
|
||||
session_id=session_id,
|
||||
title="Tool limit test",
|
||||
workspace=str(tmp_path),
|
||||
model="gpt-4o",
|
||||
messages=list(prior_messages or []),
|
||||
context_messages=list(prior_context_messages or []),
|
||||
messages=session_messages,
|
||||
context_messages=session_context_messages,
|
||||
)
|
||||
session.active_stream_id = stream_id
|
||||
session.pending_user_message = "Do the long task."
|
||||
session.pending_started_at = 1.0
|
||||
session.pending_user_message = msg_text
|
||||
session.pending_started_at = pending_started_at
|
||||
session.save()
|
||||
models.SESSIONS[session_id] = session
|
||||
streaming.SESSIONS[session_id] = session
|
||||
event_queue = queue.Queue()
|
||||
streaming.STREAMS[stream_id] = event_queue
|
||||
result_queue = list(agent_results or [])
|
||||
|
||||
class FakeAgent:
|
||||
def __init__(self, **kwargs):
|
||||
@@ -70,8 +88,15 @@ def _run_streaming_with_fake_agent(
|
||||
self.reasoning_config = None
|
||||
self.ephemeral_system_prompt = None
|
||||
self._last_error = None
|
||||
self._persist_user_message_idx = current_turn_user_idx
|
||||
self._current_turn_id = turn_id if current_turn_user_idx is not None else ""
|
||||
|
||||
def run_conversation(self, **kwargs):
|
||||
if result_queue:
|
||||
next_result = result_queue.pop(0)
|
||||
if isinstance(next_result, BaseException):
|
||||
raise next_result
|
||||
return next_result
|
||||
return agent_result
|
||||
|
||||
def interrupt(self, _message):
|
||||
@@ -87,9 +112,15 @@ def _run_streaming_with_fake_agent(
|
||||
m.setattr("api.config.get_config", lambda *_args, **_kwargs: {})
|
||||
m.setattr("api.config._resolve_cli_toolsets", lambda *_args, **_kwargs: [])
|
||||
m.setitem(sys.modules, "hermes_state", fake_hermes_state)
|
||||
if enable_auth_retry:
|
||||
m.setattr(
|
||||
streaming,
|
||||
"_attempt_credential_self_heal",
|
||||
lambda *_args, **_kwargs: {"api_key": "retry-key", "provider": "openai"},
|
||||
)
|
||||
streaming._run_agent_streaming(
|
||||
session_id=session_id,
|
||||
msg_text="Do the long task.",
|
||||
msg_text=msg_text,
|
||||
model="gpt-4o",
|
||||
workspace=str(tmp_path),
|
||||
stream_id=stream_id,
|
||||
@@ -255,6 +286,408 @@ def test_streaming_tool_limit_with_final_answer_persists_clean_done_state(tmp_pa
|
||||
assert assistant["_statusCard"]["title"] == "Tool iteration limit reached"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", ["_verification_stop_synthetic", "_pre_verify_synthetic"])
|
||||
@pytest.mark.parametrize("delta_only", [False, True])
|
||||
def test_verification_nudge_is_removed_while_corrective_followup_persists(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
marker,
|
||||
delta_only,
|
||||
):
|
||||
corrective = "Verification failed. I fixed the parser and reran the tests."
|
||||
prior_turn = [
|
||||
{"role": "user", "content": "Fix the failing test."},
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "name": "terminal"}]},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "terminal",
|
||||
"content": "Tests passed.",
|
||||
"verification_evidence": {
|
||||
"status": "passed",
|
||||
"kind": "terminal",
|
||||
"scope": "workspace",
|
||||
"canonical_command": "pytest tests/test_app.py -q",
|
||||
},
|
||||
},
|
||||
{"role": "assistant", "content": "The failing test is fixed."},
|
||||
]
|
||||
result_messages = [
|
||||
{"role": "user", "content": "[System: verify the workspace]", marker: True},
|
||||
{"role": "assistant", "content": corrective},
|
||||
]
|
||||
if not delta_only:
|
||||
result_messages = prior_turn + result_messages
|
||||
result = {"messages": result_messages}
|
||||
|
||||
_events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=prior_turn,
|
||||
prior_context_messages=prior_turn,
|
||||
msg_text="Fix the failing test.",
|
||||
current_turn_user_idx=0,
|
||||
)
|
||||
|
||||
expected_sequence = [
|
||||
(message["role"], message.get("content"))
|
||||
for message in prior_turn
|
||||
] + [("assistant", corrective)]
|
||||
for field in ("messages", "context_messages"):
|
||||
serialized = json.dumps(payload[field])
|
||||
assert marker not in serialized
|
||||
assert "[System: verify the workspace]" not in serialized
|
||||
assert [
|
||||
(message["role"], message.get("content"))
|
||||
for message in payload[field]
|
||||
] == expected_sequence
|
||||
assert [message.get("content") for message in payload[field]].count(
|
||||
"Fix the failing test."
|
||||
) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", ["_verification_stop_synthetic", "_pre_verify_synthetic"])
|
||||
def test_deferred_repeated_old_prompt_marked_followup_materializes_current_turn(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
marker,
|
||||
):
|
||||
prompt = "Fix the failing test."
|
||||
old_turn = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.0},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
corrective = "Verification failed. I fixed the parser and reran the tests."
|
||||
result = {
|
||||
"messages": old_turn + [
|
||||
{"role": "user", "content": "[System: verify the workspace]", marker: True},
|
||||
{"role": "assistant", "content": corrective},
|
||||
],
|
||||
}
|
||||
|
||||
_events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=old_turn,
|
||||
prior_context_messages=old_turn,
|
||||
msg_text=prompt,
|
||||
pending_started_at=2.0,
|
||||
current_turn_user_idx=len(old_turn),
|
||||
)
|
||||
|
||||
expected = [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", corrective),
|
||||
]
|
||||
for field in ("messages", "context_messages"):
|
||||
assert [(message["role"], message.get("content")) for message in payload[field]] == expected
|
||||
assert [message.get("content") for message in payload[field]].count(prompt) == 2
|
||||
assert marker not in json.dumps(payload[field])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", ["_verification_stop_synthetic", "_pre_verify_synthetic"])
|
||||
def test_same_second_repeated_prompt_keeps_current_user_turn(tmp_path, monkeypatch, marker):
|
||||
prompt = "Fix the failing test."
|
||||
old_turn = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
corrective = "Verification failed. I fixed the parser and reran the tests."
|
||||
result = {
|
||||
"messages": old_turn + [
|
||||
{"role": "user", "content": "[System: verify the workspace]", marker: True},
|
||||
{"role": "assistant", "content": corrective},
|
||||
],
|
||||
}
|
||||
|
||||
_events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=old_turn,
|
||||
prior_context_messages=old_turn,
|
||||
msg_text=prompt,
|
||||
pending_started_at=1.9,
|
||||
current_turn_user_idx=len(old_turn),
|
||||
)
|
||||
|
||||
expected = [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", corrective),
|
||||
]
|
||||
for field in ("messages", "context_messages"):
|
||||
assert [(message["role"], message.get("content")) for message in payload[field]] == expected
|
||||
assert [message.get("content") for message in payload[field]].count(prompt) == 2
|
||||
assert marker not in json.dumps(payload[field])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", ["_verification_stop_synthetic", "_pre_verify_synthetic"])
|
||||
def test_eager_exact_checkpoint_does_not_duplicate_repeated_prompt(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
marker,
|
||||
):
|
||||
prompt = "Fix the failing test."
|
||||
old_turn = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.0},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
current_checkpoint = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
"timestamp": 2.0,
|
||||
"_source": "webui",
|
||||
"attachments": [],
|
||||
"_active_turn_token": "stream-tool-limit:2",
|
||||
},
|
||||
]
|
||||
corrective = "Verification failed. I fixed the parser and reran the tests."
|
||||
result = {
|
||||
"messages": [
|
||||
{"role": "user", "content": "[System: verify the workspace]", marker: True},
|
||||
{"role": "assistant", "content": corrective},
|
||||
]
|
||||
}
|
||||
|
||||
_events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=old_turn,
|
||||
prior_context_messages=old_turn + current_checkpoint,
|
||||
msg_text=prompt,
|
||||
pending_started_at=2.0,
|
||||
current_turn_user_idx=2,
|
||||
)
|
||||
|
||||
expected = [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", corrective),
|
||||
]
|
||||
for field in ("messages", "context_messages"):
|
||||
assert [(message["role"], message.get("content")) for message in payload[field]] == expected
|
||||
assert [message.get("content") for message in payload[field]].count(prompt) == 2
|
||||
assert marker not in json.dumps(payload[field])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", ["_verification_stop_synthetic", "_pre_verify_synthetic"])
|
||||
def test_eager_checkpoint_reused_by_exact_token(tmp_path, monkeypatch, marker):
|
||||
prompt = "Fix the failing test."
|
||||
old_turn = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
current_checkpoint = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
"timestamp": 1.9,
|
||||
"_source": "webui",
|
||||
"attachments": [],
|
||||
"_active_turn_token": build_active_turn_token("stream-tool-limit", 1.9),
|
||||
},
|
||||
]
|
||||
corrective = "Verification failed. I fixed the parser and reran the tests."
|
||||
result = {
|
||||
"messages": [
|
||||
{"role": "user", "content": "[System: verify the workspace]", marker: True},
|
||||
{"role": "assistant", "content": corrective},
|
||||
]
|
||||
}
|
||||
|
||||
_events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=old_turn,
|
||||
prior_context_messages=old_turn + current_checkpoint,
|
||||
msg_text=prompt,
|
||||
pending_started_at=1.9,
|
||||
current_turn_user_idx=2,
|
||||
)
|
||||
|
||||
expected = [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", corrective),
|
||||
]
|
||||
for field in ("messages", "context_messages"):
|
||||
assert [(message["role"], message.get("content")) for message in payload[field]] == expected
|
||||
assert [message.get("content") for message in payload[field]].count(prompt) == 2
|
||||
assert marker not in json.dumps(payload[field])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", ["_verification_stop_synthetic", "_pre_verify_synthetic"])
|
||||
def test_tool_limit_exit_uses_shared_settlement_owner(tmp_path, monkeypatch, marker):
|
||||
prompt = "Fix the failing test."
|
||||
old_turn = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
corrective = "Verification failed. I fixed the parser and reran the tests."
|
||||
result = {
|
||||
"turn_exit_reason": "max_iterations_reached(30/30)",
|
||||
"messages": old_turn + [
|
||||
{"role": "user", "content": "[System: verify the workspace]", marker: True},
|
||||
{"role": "assistant", "content": corrective},
|
||||
],
|
||||
}
|
||||
settle_calls = []
|
||||
original = streaming._settle_result_messages
|
||||
|
||||
def _spy(*args, **kwargs):
|
||||
settle_calls.append(True)
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(streaming, "_settle_result_messages", _spy)
|
||||
_events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=old_turn,
|
||||
prior_context_messages=old_turn,
|
||||
msg_text=prompt,
|
||||
pending_started_at=1.9,
|
||||
current_turn_user_idx=len(old_turn),
|
||||
)
|
||||
|
||||
assert len(settle_calls) == 1
|
||||
assert [(message["role"], message.get("content")) for message in payload["messages"]] == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", corrective),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", ["_verification_stop_synthetic", "_pre_verify_synthetic"])
|
||||
def test_shared_settlement_owner_handles_exception_style_delta(tmp_path, monkeypatch, marker):
|
||||
prompt = "Fix the failing test."
|
||||
old_turn = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
corrective = "Verification failed. I fixed the parser and reran the tests."
|
||||
heal_result = {
|
||||
"messages": old_turn + [
|
||||
{"role": "user", "content": "[System: verify the workspace]", marker: True},
|
||||
{"role": "assistant", "content": corrective},
|
||||
],
|
||||
}
|
||||
session = types.SimpleNamespace(messages=list(old_turn), context_messages=list(old_turn))
|
||||
streaming._settle_result_messages(
|
||||
session,
|
||||
old_turn,
|
||||
old_turn,
|
||||
heal_result["messages"],
|
||||
prompt,
|
||||
"webui",
|
||||
{
|
||||
"token": "stream-tool-limit:1.9",
|
||||
"text": prompt,
|
||||
"timestamp": 1.9,
|
||||
"source": "webui",
|
||||
"attachments": [],
|
||||
"current_turn_user_idx": len(old_turn),
|
||||
"turn_id": "turn-current",
|
||||
},
|
||||
)
|
||||
|
||||
assert [(message["role"], message.get("content")) for message in session.messages] == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", corrective),
|
||||
]
|
||||
|
||||
|
||||
def test_missing_agent_authority_fails_closed(tmp_path, monkeypatch):
|
||||
prompt = "Fix the failing test."
|
||||
old_turn = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
result = {
|
||||
"messages": old_turn + [
|
||||
{"role": "user", "content": "[System: verify the workspace]", "_verification_stop_synthetic": True},
|
||||
{"role": "assistant", "content": "Verification failed. I fixed the parser and reran the tests."},
|
||||
],
|
||||
}
|
||||
|
||||
_events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=old_turn,
|
||||
prior_context_messages=old_turn,
|
||||
msg_text=prompt,
|
||||
pending_started_at=1.9,
|
||||
current_turn_user_idx=None,
|
||||
)
|
||||
|
||||
assert [(message["role"], message.get("content")) for message in payload["messages"]] == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
("assistant", "Verification failed. I fixed the parser and reran the tests."),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", ["_verification_stop_synthetic", "_pre_verify_synthetic"])
|
||||
def test_synthetic_only_verification_delta_preserves_prior_histories(tmp_path, monkeypatch, marker):
|
||||
prior_turn = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Fix the failing test.",
|
||||
"_active_turn_token": build_active_turn_token("stream-tool-limit", 1.0),
|
||||
},
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "name": "terminal"}]},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "terminal",
|
||||
"content": "Tests passed.",
|
||||
"verification_evidence": {
|
||||
"status": "passed",
|
||||
"kind": "terminal",
|
||||
"scope": "workspace",
|
||||
"canonical_command": "pytest tests/test_app.py -q",
|
||||
},
|
||||
},
|
||||
{"role": "assistant", "content": "The failing test is fixed."},
|
||||
]
|
||||
result = {
|
||||
"messages": [
|
||||
{"role": "user", "content": "[System: verify the workspace]", marker: True},
|
||||
],
|
||||
}
|
||||
|
||||
_events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=prior_turn,
|
||||
prior_context_messages=prior_turn,
|
||||
msg_text="Fix the failing test.",
|
||||
current_turn_user_idx=0,
|
||||
)
|
||||
|
||||
for field in ("messages", "context_messages"):
|
||||
assert payload[field] == payload["messages"]
|
||||
serialized = json.dumps(payload[field])
|
||||
assert marker not in serialized
|
||||
assert "[System: verify the workspace]" not in serialized
|
||||
assert "The failing test is fixed." in serialized
|
||||
|
||||
|
||||
def test_streaming_tool_limit_partial_without_final_answer_emits_no_final_apperror(tmp_path, monkeypatch):
|
||||
result = {
|
||||
"status": "partial",
|
||||
@@ -479,3 +912,169 @@ def test_streaming_empty_result_messages_do_not_treat_prior_assistant_as_current
|
||||
assert apperror_payloads[-1]["type"] == "no_response"
|
||||
assert not [payload for event, payload in events if event == "done"]
|
||||
assert payload["messages"][-1]["_error"] is True
|
||||
|
||||
|
||||
def test_streaming_empty_result_same_text_does_not_treat_prior_assistant_as_current_answer(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
prompt = "Fix the failing test."
|
||||
prior = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
result = {"messages": []}
|
||||
|
||||
events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=prior,
|
||||
prior_context_messages=prior,
|
||||
msg_text=prompt,
|
||||
pending_started_at=1.9,
|
||||
current_turn_user_idx=len(prior),
|
||||
)
|
||||
|
||||
apperror_payloads = [event_payload for event, event_payload in events if event == "apperror"]
|
||||
assert apperror_payloads, "expected silent-failure apperror"
|
||||
assert apperror_payloads[-1]["type"] == "no_response"
|
||||
assert not [event_payload for event, event_payload in events if event == "done"]
|
||||
assert [(message["role"], message.get("content")) for message in payload["messages"][:-1]] == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
]
|
||||
assert payload["messages"][-1]["role"] == "assistant"
|
||||
assert payload["messages"][-1]["_error"] is True
|
||||
|
||||
|
||||
def test_streaming_empty_result_without_exact_checkpoint_materializes_current_user(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
prompt = "Fix the failing test."
|
||||
prior = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
result = {"messages": []}
|
||||
|
||||
events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=prior,
|
||||
prior_context_messages=prior,
|
||||
msg_text=prompt,
|
||||
pending_started_at=1.9,
|
||||
current_turn_user_idx=0,
|
||||
)
|
||||
|
||||
apperror_payloads = [event_payload for event, event_payload in events if event == "apperror"]
|
||||
assert apperror_payloads, "expected silent-failure apperror"
|
||||
assert apperror_payloads[-1]["type"] == "no_response"
|
||||
assert not [event_payload for event, event_payload in events if event == "done"]
|
||||
assert [(message["role"], message.get("content")) for message in payload["messages"][:-1]] == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
]
|
||||
assert payload["messages"][0].get("_active_turn_token") is None
|
||||
assert payload["messages"][2]["_active_turn_token"] == build_active_turn_token(
|
||||
"stream-tool-limit",
|
||||
1.9,
|
||||
)
|
||||
assert payload["messages"][-1]["role"] == "assistant"
|
||||
assert payload["messages"][-1]["_error"] is True
|
||||
|
||||
|
||||
def test_streaming_empty_result_divergent_context_index_does_not_reuse_historical_assistant(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
prompt = "Fix the failing test."
|
||||
current_token = build_active_turn_token("stream-tool-limit", 1.9)
|
||||
prior = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.0},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
prior_context = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.0, "attachments": []},
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
"timestamp": 1.9,
|
||||
"attachments": [],
|
||||
"_active_turn_token": current_token,
|
||||
},
|
||||
]
|
||||
result = {"messages": []}
|
||||
|
||||
events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=prior,
|
||||
prior_context_messages=prior_context,
|
||||
msg_text=prompt,
|
||||
pending_started_at=1.9,
|
||||
current_turn_user_idx=0,
|
||||
)
|
||||
|
||||
apperror_payloads = [event_payload for event, event_payload in events if event == "apperror"]
|
||||
assert apperror_payloads, "expected silent-failure apperror"
|
||||
assert apperror_payloads[-1]["type"] == "no_response"
|
||||
assert not [event_payload for event, event_payload in events if event == "done"]
|
||||
assert [(message["role"], message.get("content")) for message in payload["messages"][:-1]] == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
]
|
||||
assert payload["messages"][0].get("_active_turn_token") is None
|
||||
assert payload["messages"][-1]["role"] == "assistant"
|
||||
assert payload["messages"][-1]["_error"] is True
|
||||
|
||||
|
||||
def test_streaming_tool_limit_empty_result_without_exact_checkpoint_materializes_current_user(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
prompt = "Fix the failing test."
|
||||
prior = [
|
||||
{"role": "user", "content": prompt, "timestamp": 1.1},
|
||||
{"role": "assistant", "content": "The earlier attempt is complete."},
|
||||
]
|
||||
result = {
|
||||
"turn_exit_reason": "max_iterations_reached(30/30)",
|
||||
"messages": [],
|
||||
}
|
||||
|
||||
events, payload = _run_streaming_with_fake_agent(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
result,
|
||||
prior_messages=prior,
|
||||
prior_context_messages=prior,
|
||||
msg_text=prompt,
|
||||
pending_started_at=1.9,
|
||||
current_turn_user_idx=0,
|
||||
)
|
||||
|
||||
apperror_payloads = [event_payload for event, event_payload in events if event == "apperror"]
|
||||
assert apperror_payloads, "expected tool-limit apperror"
|
||||
assert apperror_payloads[-1]["type"] == "tool_limit_reached"
|
||||
assert apperror_payloads[-1]["terminal_state"] == "tool_limit_reached"
|
||||
assert not [event_payload for event, event_payload in events if event == "done"]
|
||||
assert [(message["role"], message.get("content")) for message in payload["messages"][:-1]] == [
|
||||
("user", prompt),
|
||||
("assistant", "The earlier attempt is complete."),
|
||||
("user", prompt),
|
||||
]
|
||||
assert payload["messages"][0].get("_active_turn_token") is None
|
||||
assert payload["messages"][2]["_active_turn_token"] == build_active_turn_token(
|
||||
"stream-tool-limit",
|
||||
1.9,
|
||||
)
|
||||
assert payload["messages"][-1]["role"] == "assistant"
|
||||
assert payload["messages"][-1]["_error"] is True
|
||||
Reference in new issue
Block a user