240 lines
10 KiB
Python
240 lines
10 KiB
Python
"""Regression coverage for #6068 per-turn used-model footer instrumentation."""
|
|
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from api.models import Session
|
|
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
NODE = shutil.which("node")
|
|
UI_JS_PATH = REPO / "static" / "ui.js"
|
|
STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8")
|
|
MODELS_PY = (REPO / "api" / "models.py").read_text(encoding="utf-8")
|
|
UI_JS = (REPO / "static" / "ui.js").read_text(encoding="utf-8")
|
|
STYLE_CSS = (REPO / "static" / "style.css").read_text(encoding="utf-8")
|
|
|
|
|
|
def _run_node(source: str) -> str:
|
|
result = subprocess.run(
|
|
[NODE],
|
|
input=source,
|
|
cwd=str(REPO),
|
|
capture_output=True,
|
|
encoding="utf-8",
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(result.stderr)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def _eval_used_model_turn_chip_label_cases() -> dict:
|
|
ui_js = UI_JS_PATH.read_text(encoding="utf-8")
|
|
source = f"""
|
|
const src = {ui_js!r};
|
|
function extractFunc(name) {{
|
|
const re = new RegExp('function\\\\s+' + name + '\\\\s*\\\\(');
|
|
const start = src.search(re);
|
|
if (start < 0) throw new Error(name + ' not found');
|
|
let i = src.indexOf('{{', start);
|
|
let depth = 1; i++;
|
|
while (depth > 0 && i < src.length) {{
|
|
if (src[i] === '{{') depth++;
|
|
else if (src[i] === '}}') depth--;
|
|
i++;
|
|
}}
|
|
return src.slice(start, i);
|
|
}}
|
|
function getModelLabel(modelId) {{
|
|
return String(modelId || 'Unknown');
|
|
}}
|
|
eval(extractFunc('_compactComposerModelChipLabel'));
|
|
eval(extractFunc('_usedModelTurnChipLabel'));
|
|
eval(extractFunc('_gatewayProviderName'));
|
|
eval(extractFunc('_gatewayRoutingLabel'));
|
|
eval(extractFunc('_formatGatewayModelLabel'));
|
|
const modelId = 'gpt-5-mini';
|
|
const expectedPresent = _compactComposerModelChipLabel(modelId, getModelLabel(modelId));
|
|
const cases = {{
|
|
present: _usedModelTurnChipLabel({{ _usedModel: modelId }}),
|
|
expectedPresent,
|
|
suppressed: _usedModelTurnChipLabel({{
|
|
_usedModel: modelId,
|
|
_gatewayRouting: {{ used_model: 'deepseek-v3.2' }},
|
|
}}),
|
|
suppressedRoutingOnly: _usedModelTurnChipLabel({{
|
|
_usedModel: modelId,
|
|
_gatewayRouting: {{ used_provider: 'openrouter' }},
|
|
}}),
|
|
absent: _usedModelTurnChipLabel({{}}),
|
|
nullMsg: _usedModelTurnChipLabel(null),
|
|
gatewayFallback: _formatGatewayModelLabel(modelId, getModelLabel(modelId), {{ used_provider: 'openrouter' }}),
|
|
}};
|
|
console.log(JSON.stringify(cases));
|
|
"""
|
|
return json.loads(_run_node(source))
|
|
|
|
|
|
def _eval_transparent_multi_segment_footer_cases() -> dict:
|
|
"""Drive the transparent footer for a realistic multi-segment tool turn.
|
|
|
|
Turn shape: user -> assistant tool/activity segment (no metadata) -> tool
|
|
result -> final assistant carrying _usedModel/_turnDuration. The footer must
|
|
resolve metadata from the FINAL segment (not querySelector's first match) so
|
|
the model label renders exactly once as .lf-model. Regression for the #6068
|
|
round-2 gate where multi-segment turns dropped the label entirely.
|
|
"""
|
|
ui_js = UI_JS_PATH.read_text(encoding="utf-8")
|
|
source = f"""
|
|
const src = {ui_js!r};
|
|
function extractFunc(name) {{
|
|
const re = new RegExp('function\\\\s+' + name + '\\\\s*\\\\(');
|
|
const start = src.search(re);
|
|
if (start < 0) throw new Error(name + ' not found');
|
|
let i = src.indexOf('{{', start);
|
|
let depth = 1; i++;
|
|
while (depth > 0 && i < src.length) {{
|
|
if (src[i] === '{{') depth++;
|
|
else if (src[i] === '}}') depth--;
|
|
i++;
|
|
}}
|
|
return src.slice(start, i);
|
|
}}
|
|
function getModelLabel(modelId) {{ return String(modelId || 'Unknown'); }}
|
|
function esc(s) {{ return String(s == null ? '' : s); }}
|
|
function t() {{ return ''; }}
|
|
// Multi-segment tool turn: idx 1 is the activity segment (no metadata),
|
|
// idx 2 is the final answer that carries the stamped turn metadata.
|
|
const S = {{ messages: [
|
|
{{ role: 'user', content: 'hi' }},
|
|
{{ role: 'assistant', content: '', _toolCalls: [{{}}] }},
|
|
{{ role: 'assistant', content: 'answer', _usedModel: 'anthropic/claude-sonnet-4.5', _turnDuration: 1.2 }},
|
|
] }};
|
|
function FakeSeg(idx) {{ return {{ getAttribute(name) {{ return name === 'data-msg-idx' ? String(idx) : null; }} }}; }}
|
|
const turn = {{ querySelectorAll(sel) {{ return [FakeSeg(1), FakeSeg(2)]; }} }};
|
|
eval(extractFunc('_compactComposerModelChipLabel'));
|
|
eval(extractFunc('_usedModelTurnChipLabel'));
|
|
eval(extractFunc('_transparentTurnMetaMessage'));
|
|
eval(extractFunc('_transparentTurnFooterHtml'));
|
|
const picked = _transparentTurnMetaMessage(turn);
|
|
const modelText = _usedModelTurnChipLabel(picked || {{}});
|
|
const modelTitle = picked ? String(picked._usedModel || '') : '';
|
|
const html = _transparentTurnFooterHtml('1.2s', modelText, '', '', 'Done', modelTitle);
|
|
const lfModelCount = (html.match(/lf-model/g) || []).length;
|
|
console.log(JSON.stringify({{
|
|
pickedUsedModel: picked ? picked._usedModel : null,
|
|
pickedContent: picked ? picked.content : null,
|
|
modelText,
|
|
lfModelCount,
|
|
hasTitle: html.indexOf('title="anthropic/claude-sonnet-4.5"') >= 0,
|
|
}}));
|
|
"""
|
|
return json.loads(_run_node(source))
|
|
|
|
|
|
def test_streaming_stamps_used_model_on_assistant_message_and_usage_payload():
|
|
assert "_dm['_usedModel'] = _used_model" in STREAMING_PY
|
|
# Read the served model from the agent after the run (including fallback),
|
|
# without treating a requested/configured selection as proof of service.
|
|
assert "_observed_model = getattr(agent, 'model', None)" in STREAMING_PY
|
|
assert "_used_model = _observed_model.strip() if isinstance(_observed_model, str) else None" in STREAMING_PY
|
|
assert "_used_model = getattr(agent, 'model', None) or resolved_model or model" not in STREAMING_PY
|
|
assert "usage['used_model'] = _used_model" in STREAMING_PY
|
|
|
|
|
|
def test_models_allowlist_round_trips_used_model_across_save_reload():
|
|
assert '"_usedModel"' in MODELS_PY
|
|
assert "_usedModel" in MODELS_PY.split("_SESSION_MESSAGE_DISPLAY_METADATA_KEYS")[1].split(")")[0]
|
|
|
|
session = Session(session_id="6068usedmodel", title="Used model")
|
|
session.messages = [
|
|
{
|
|
"role": "assistant",
|
|
"content": "done",
|
|
"_firstTokenMs": 250,
|
|
"_usedModel": "gpt-5-mini",
|
|
},
|
|
]
|
|
session.save()
|
|
|
|
reloaded = Session.load("6068usedmodel")
|
|
assert reloaded.messages[-1]["_usedModel"] == "gpt-5-mini"
|
|
assert reloaded.messages[-1]["_firstTokenMs"] == 250
|
|
|
|
|
|
@pytest.mark.skipif(NODE is None, reason="node not on PATH")
|
|
def test_used_model_turn_chip_label_renders_and_suppresses_gateway_duplicate():
|
|
"""Behavioral footer chip cases from #6068 (not source-string greps)."""
|
|
cases = _eval_used_model_turn_chip_label_cases()
|
|
assert cases["present"] == cases["expectedPresent"]
|
|
assert cases["present"] # non-empty label when _usedModel is set
|
|
assert cases["suppressed"] == ""
|
|
# Routing metadata WITHOUT used_model must also suppress the additive chip:
|
|
# the gateway formatter owns the label (falling back to _usedModel), so a
|
|
# provider-only routing payload must not render two model labels.
|
|
assert cases["suppressedRoutingOnly"] == ""
|
|
assert cases["absent"] == ""
|
|
assert cases["nullMsg"] == ""
|
|
# When routing omits used_model, the gateway label falls back to the
|
|
# caller-provided model id (the settled footer passes msg._usedModel).
|
|
assert cases["gatewayFallback"].startswith(cases["expectedPresent"])
|
|
assert "via" in cases["gatewayFallback"]
|
|
|
|
|
|
def test_settled_footer_wires_used_model_chip_in_dom_paths():
|
|
assert "msg-used-model-inline" in UI_JS
|
|
assert "_usedModelTurnChipLabel(msg)" in UI_JS
|
|
# The settled footer must hand the served model to the gateway formatter as
|
|
# its fallback, and skip the generic chip when the transparent turn footer
|
|
# owns the model label (one model label per turn).
|
|
assert "_formatGatewayModelLabel(String(msg._usedModel||'').trim()||(S.session&&S.session.model)||'', '', routing)" in UI_JS
|
|
assert "_transparentFooterOwnsModel" in UI_JS
|
|
assert ".msg-used-model-inline" in STYLE_CSS
|
|
|
|
|
|
def test_settled_footer_orders_model_chip_after_duration():
|
|
# Match the transparent footer order (elapsed · model · …): the duration
|
|
# fragment must be pushed before the used-model fragment.
|
|
footer_block = UI_JS.split("const usedModelText=_usedModelTurnChipLabel(msg);", 1)[1]
|
|
duration_at = footer_block.index("duration.className='msg-duration-inline'")
|
|
used_model_at = footer_block.index("usedModel.className='msg-used-model-inline'")
|
|
assert duration_at < used_model_at
|
|
|
|
|
|
def test_transparent_turn_footer_includes_model_between_duration_and_ttft():
|
|
assert "function _transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText, modelTitle)" in UI_JS
|
|
assert 'class="lf-model"' in UI_JS
|
|
assert "modelText=_usedModelTurnChipLabel(msg)" in UI_JS
|
|
assert ".transparent-turn-footer .lf-model" in STYLE_CSS
|
|
|
|
|
|
def test_transparent_footer_reads_metadata_from_final_segment_not_first():
|
|
# The transparent footer must resolve turn metadata via the dedicated
|
|
# last→first segment scan, never querySelector's first-match, which would
|
|
# read the activity segment and drop the model label on tool turns.
|
|
assert "const msg=_transparentTurnMetaMessage(turn);" in UI_JS
|
|
assert "function _transparentTurnMetaMessage(turn)" in UI_JS
|
|
# The generic footer yields the model chip to the transparent footer.
|
|
assert "_transparentFooterOwnsModel" in UI_JS
|
|
|
|
|
|
@pytest.mark.skipif(NODE is None, reason="node not on PATH")
|
|
def test_transparent_multi_segment_tool_turn_renders_exactly_one_model_label():
|
|
"""#6068 gate round 2: multi-segment tool turns must show the model once."""
|
|
cases = _eval_transparent_multi_segment_footer_cases()
|
|
# Metadata resolved from the FINAL assistant segment, not the first.
|
|
assert cases["pickedUsedModel"] == "anthropic/claude-sonnet-4.5"
|
|
assert cases["pickedContent"] == "answer"
|
|
assert cases["modelText"] # non-empty label
|
|
# Exactly one .lf-model in the transparent footer (the generic chip is
|
|
# suppressed by _transparentFooterOwnsModel), so no duplicate and no loss.
|
|
assert cases["lfModelCount"] == 1
|
|
# Non-blocking UX: full model id preserved in a hover title.
|
|
assert cases["hasTitle"]
|