diff --git a/api/models.py b/api/models.py index e32228c5..3c0f660b 100644 --- a/api/models.py +++ b/api/models.py @@ -7989,6 +7989,7 @@ _SESSION_MESSAGE_DISPLAY_METADATA_KEYS = ( "_turnTps", "_turnUsage", "_firstTokenMs", + "_usedModel", "_gatewayRouting", "_statusCard", "_anchor_stream_id", diff --git a/api/streaming.py b/api/streaming.py index 2528baf4..b668f57f 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -9931,6 +9931,11 @@ def _run_agent_streaming( requested_model=resolved_model or model, requested_provider=resolved_provider, ) + # #6068: the served model must be read AFTER agent.run — the agent + # mutates agent.model when a fallback fires, so the pre-run + # resolved_model would mis-attribute exactly the turns where + # attribution matters most. + _used_model = getattr(agent, 'model', None) or resolved_model or model if _gateway_routing: s.gateway_routing = _gateway_routing _history = list(getattr(s, 'gateway_routing_history', None) or []) @@ -9947,6 +9952,8 @@ def _run_agent_streaming( _ttft_ms = meter().get_ttft_ms(stream_id) if _ttft_ms is not None: _dm['_firstTokenMs'] = _ttft_ms + if _used_model: + _dm['_usedModel'] = _used_model break # Persist context window data on the session so the context-ring # indicator survives a page reload (#1318). Must run BEFORE @@ -10334,6 +10341,8 @@ def _run_agent_streaming( _ttft_ms = meter().get_ttft_ms(stream_id) if _ttft_ms is not None: usage['ttft_ms'] = _ttft_ms + if _used_model: + usage['used_model'] = _used_model # Include context window data from the agent's compressor for the UI indicator. # The session-level persistence happens above (before s.save()) so the values # survive a page reload; this block only populates the live SSE usage payload. diff --git a/static/style.css b/static/style.css index a2538e28..2516a88a 100644 --- a/static/style.css +++ b/static/style.css @@ -4411,6 +4411,7 @@ body.resizing .sidebar{transition:none!important;} .transparent-turn-footer .lf-time{color:var(--text);} .transparent-turn-footer .lf-sep{opacity:.4;} .transparent-turn-footer .lf-status{color:var(--muted);opacity:.85;} +.transparent-turn-footer .lf-model{color:var(--muted);opacity:.85;} .transparent-turn-footer .lf-ttft{color:var(--accent);opacity:.9;} .transparent-turn-footer .lf-tokens{font-variant-numeric:tabular-nums;} @@ -6110,6 +6111,7 @@ main.main > #mainPlugin{display:none;} .msg-usage-inline, .msg-duration-inline, .msg-gateway-inline, +.msg-used-model-inline, .gateway-failover-inline, .msg-model-warning-inline { font-size: 11px; diff --git a/static/ui.js b/static/ui.js index 06896d03..a2ff17d8 100644 --- a/static/ui.js +++ b/static/ui.js @@ -7030,6 +7030,17 @@ function _formatGatewayModelLabel(modelId,labelText,routing){ const via=_gatewayRoutingLabel(routing); return via?`${base} ${via}`:base; } +function _usedModelTurnChipLabel(msg){ + if(!msg)return''; + // Gateway turns own their model label via _formatGatewayModelLabel (which + // falls back to msg._usedModel when routing omits used_model), so suppress + // the additive chip whenever routing metadata is present — not only when + // routing.used_model is set — to guarantee one model label per turn. + if(msg._gatewayRouting)return''; + const usedModel=String(msg._usedModel||'').trim(); + if(!usedModel)return''; + return _compactComposerModelChipLabel(usedModel,getModelLabel(usedModel)); +} function _gatewayRoutingFailoverText(routing){ if(!routing||!routing.has_failover)return''; const attempts=Array.isArray(routing.routing)?routing.routing:[]; @@ -11750,13 +11761,40 @@ function _applyTransparentRowFading(turn){ row.setAttribute('data-transparent-fade',String(step)); } } +// Resolve the assistant message that carries a transparent turn's settled +// metadata (duration / used model / TTFT / usage). A tool-using turn renders +// multiple assistant segments — earlier activity segment(s) then the final +// answer — and the metadata is stamped only on the LAST assistant message +// (see api/streaming.py finalization). `turn.querySelector('.assistant-segment')` +// returns the FIRST segment, which has no metadata, so the footer (model chip +// included) would render empty exactly on tool turns (#6068 gate round 2). +// Scan segments last→first for the final metadata-bearing assistant message, +// falling back to the last assistant segment when none carries metadata yet. +function _transparentTurnMetaMessage(turn){ + if(!turn)return null; + const segs=turn.querySelectorAll('.assistant-segment[data-msg-idx]'); + let fallback=null; + for(let si=segs.length-1;si>=0;si--){ + const mi=segs[si].getAttribute('data-msg-idx'); + if(mi==null)continue; + const candidate=S.messages[Number(mi)]; + if(!candidate||candidate.role!=='assistant')continue; + if(!fallback)fallback=candidate; + if(candidate._turnDuration!=null||candidate._usedModel||candidate._firstTokenMs!=null||candidate._turnUsage)return candidate; + } + return fallback; +} // ── Transparent turn footer (elapsed · tokens · TTFT · status) ─────────── // Mirrors the live run-status line for settled turns in transparent // mode. Shows duration, first-token time, token usage, and final status. // Only rendered for turns that have transparent event rows. -function _transparentTurnFooterHtml(durationText, ttftText, tokensText, statusText){ +function _transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText, modelTitle){ const parts=[]; if(durationText) parts.push(`${esc(durationText)}`); + if(modelText){ + const titleAttr=(modelTitle&&modelTitle!==modelText)?` title="${esc(modelTitle)}"`:''; + parts.push(`${esc(modelText)}`); + } if(ttftText) parts.push(`TTFT ${esc(ttftText)}`); if(tokensText) parts.push(`${esc(tokensText)}`); if(statusText) parts.push(`${esc(statusText)}`); @@ -11775,10 +11813,12 @@ function _renderTransparentTurnFooter(turn, opts){ return; } const durationText=opts&&opts.durationText||''; + const modelText=opts&&opts.modelText||''; + const modelTitle=opts&&opts.modelTitle||''; const ttftText=opts&&opts.ttftText||''; const tokensText=opts&&opts.tokensText||''; const statusText=opts&&opts.statusText||(t('done')||'Done'); - const html=_transparentTurnFooterHtml(durationText, ttftText, tokensText, statusText); + const html=_transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText, modelTitle); let footer=turn.querySelector('.transparent-turn-footer'); if(!html){ if(footer) footer.remove(); @@ -16817,7 +16857,7 @@ function renderMessages(options){ const msg=S.messages[mi]||{}; if(msg.role!=='assistant') continue; const routing=msg._gatewayRouting||null; - const gatewayText=_formatGatewayModelLabel(S.session&&S.session.model||'', '', routing); + const gatewayText=_formatGatewayModelLabel(String(msg._usedModel||'').trim()||(S.session&&S.session.model)||'', '', routing); const failoverText=_gatewayRoutingFailoverText(routing); const modelWarningText=_gatewayModelWarningText(routing); const hasTurnUsage=!!msg._turnUsage; @@ -16826,12 +16866,13 @@ function renderMessages(options){ // Worklog above the final answer. const compactWorklogForMessage=isCompactWorklogMode()&&(toolCallAssistantIdxs.has(mi)||assistantThinking.has(mi)); const durationText=compactWorklogForMessage?'':_formatTurnDuration(msg._turnDuration); - if(!hasTurnUsage&&!durationText&&!gatewayText&&!failoverText&&!modelWarningText) continue; + const usedModelText=_usedModelTurnChipLabel(msg); + if(!hasTurnUsage&&!durationText&&!gatewayText&&!failoverText&&!modelWarningText&&!usedModelText) continue; const seg=assistantSegments.get(mi); const row=seg?seg.closest('.assistant-turn'):null; const footerRows=row?row.querySelectorAll('.msg-foot'):[]; const targetFoot=footerRows.length?footerRows[footerRows.length-1]:null; - if(!targetFoot||targetFoot.querySelector('.msg-usage-inline,.msg-duration-inline,.msg-gateway-inline,.gateway-failover-inline,.msg-model-warning-inline')) continue; + if(!targetFoot||targetFoot.querySelector('.msg-usage-inline,.msg-duration-inline,.msg-gateway-inline,.gateway-failover-inline,.msg-model-warning-inline,.msg-used-model-inline')) continue; const fragments=[]; if(modelWarningText){ const warning=document.createElement('span'); @@ -16857,6 +16898,23 @@ function renderMessages(options){ duration.textContent=`Done in ${durationText}`; fragments.push(duration); } + // The transparent turn footer owns the model label (.lf-model) whenever + // the turn has transparent event rows — skip the generic chip there so + // exactly one model label renders per turn. Model sits after duration to + // match the transparent footer order (elapsed · model · …). + const _transparentFooterOwnsModel=usedModelText&&isTransparentStream()&&row&&(()=>{ + const blocks=_assistantTurnBlocks(row); + return !!(blocks&&blocks.querySelector(':scope > .transparent-event-row')); + })(); + if(usedModelText&&!_transparentFooterOwnsModel){ + const usedModel=document.createElement('span'); + usedModel.className='msg-used-model-inline'; + usedModel.textContent=usedModelText; + // Preserve the full (uncompacted) model id on hover where available. + const usedModelFull=String(msg._usedModel||'').trim(); + if(usedModelFull&&usedModelFull!==usedModelText) usedModel.title=usedModelFull; + fragments.push(usedModel); + } if(window._showTokenUsage&&hasTurnUsage){ const usage=document.createElement('span'); usage.className='msg-usage-inline'; @@ -16904,26 +16962,31 @@ function renderMessages(options){ } _applyTransparentRowFading(turn); if(hasTransparentRows){ - // Find the corresponding message to read duration/usage. - const seg=turn.querySelector('.assistant-segment'); + // Read turn metadata from the final metadata-bearing assistant segment, + // not querySelector's first match — a tool turn's activity segment + // precedes the answer, and the metadata lives on the last message + // (#6068 gate round 2: multi-segment turns lost the model label). + const msg=_transparentTurnMetaMessage(turn); let durationText=''; + let modelText=''; + let modelTitle=''; let ttftText=''; let tokensText=''; - if(seg){ - const mi=seg.getAttribute('data-msg-idx'); - if(mi!=null){ - const msg=S.messages[Number(mi)]||{}; - if(msg._turnDuration!=null) durationText=_formatTurnDuration(msg._turnDuration); - if(msg._firstTokenMs!=null) ttftText=_formatFirstToken(msg._firstTokenMs); - if(msg._turnUsage){ - const inTok=msg._turnUsage.input_tokens||0; - const outTok=msg._turnUsage.output_tokens||0; - tokensText=`${_fmtTokens(inTok)} in · ${_fmtTokens(outTok)} out`; - } + if(msg){ + if(msg._turnDuration!=null) durationText=_formatTurnDuration(msg._turnDuration); + modelText=_usedModelTurnChipLabel(msg); + if(modelText) modelTitle=String(msg._usedModel||'').trim(); + if(msg._firstTokenMs!=null) ttftText=_formatFirstToken(msg._firstTokenMs); + if(msg._turnUsage){ + const inTok=msg._turnUsage.input_tokens||0; + const outTok=msg._turnUsage.output_tokens||0; + tokensText=`${_fmtTokens(inTok)} in · ${_fmtTokens(outTok)} out`; } } _renderTransparentTurnFooter(turn,{ durationText, + modelText, + modelTitle, ttftText, tokensText, statusText: t('done')||'Done', diff --git a/tests/test_anchor_fallback_ownership.py b/tests/test_anchor_fallback_ownership.py index 700e59c4..6cc6d677 100644 --- a/tests/test_anchor_fallback_ownership.py +++ b/tests/test_anchor_fallback_ownership.py @@ -680,6 +680,7 @@ def test_render_messages_keeps_anchor_owned_turn_out_of_legacy_activity_rebuilds function _formatGatewayModelLabel() {{ return ''; }} function _gatewayRoutingFailoverText() {{ return ''; }} function _gatewayModelWarningText() {{ return ''; }} + function _usedModelTurnChipLabel() {{ return ''; }} function _formatTurnDuration() {{ return ''; }} function _renderSettledAnchorSceneForMessage(message, segment, rawIdx) {{ const group = new FakeElement('div'); diff --git a/tests/test_issue6068_used_model_footer.py b/tests/test_issue6068_used_model_footer.py new file mode 100644 index 00000000..7ba04c53 --- /dev/null +++ b/tests/test_issue6068_used_model_footer.py @@ -0,0 +1,238 @@ +"""Regression coverage for #6068 per-turn used-model footer instrumentation.""" + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from api.models import Session + + +REPO = Path(__file__).resolve().parents[1] +NODE = shutil.which("node") +UI_JS_PATH = REPO / "static" / "ui.js" +STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8") +MODELS_PY = (REPO / "api" / "models.py").read_text(encoding="utf-8") +UI_JS = (REPO / "static" / "ui.js").read_text(encoding="utf-8") +STYLE_CSS = (REPO / "static" / "style.css").read_text(encoding="utf-8") + + +def _run_node(source: str) -> str: + result = subprocess.run( + [NODE], + input=source, + cwd=str(REPO), + capture_output=True, + encoding="utf-8", + text=True, + timeout=30, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr) + return result.stdout.strip() + + +def _eval_used_model_turn_chip_label_cases() -> dict: + ui_js = UI_JS_PATH.read_text(encoding="utf-8") + source = f""" +const src = {ui_js!r}; +function extractFunc(name) {{ + const re = new RegExp('function\\\\s+' + name + '\\\\s*\\\\('); + const start = src.search(re); + if (start < 0) throw new Error(name + ' not found'); + let i = src.indexOf('{{', start); + let depth = 1; i++; + while (depth > 0 && i < src.length) {{ + if (src[i] === '{{') depth++; + else if (src[i] === '}}') depth--; + i++; + }} + return src.slice(start, i); +}} +function getModelLabel(modelId) {{ + return String(modelId || 'Unknown'); +}} +eval(extractFunc('_compactComposerModelChipLabel')); +eval(extractFunc('_usedModelTurnChipLabel')); +eval(extractFunc('_gatewayProviderName')); +eval(extractFunc('_gatewayRoutingLabel')); +eval(extractFunc('_formatGatewayModelLabel')); +const modelId = 'gpt-5-mini'; +const expectedPresent = _compactComposerModelChipLabel(modelId, getModelLabel(modelId)); +const cases = {{ + present: _usedModelTurnChipLabel({{ _usedModel: modelId }}), + expectedPresent, + suppressed: _usedModelTurnChipLabel({{ + _usedModel: modelId, + _gatewayRouting: {{ used_model: 'deepseek-v3.2' }}, + }}), + suppressedRoutingOnly: _usedModelTurnChipLabel({{ + _usedModel: modelId, + _gatewayRouting: {{ used_provider: 'openrouter' }}, + }}), + absent: _usedModelTurnChipLabel({{}}), + nullMsg: _usedModelTurnChipLabel(null), + gatewayFallback: _formatGatewayModelLabel(modelId, getModelLabel(modelId), {{ used_provider: 'openrouter' }}), +}}; +console.log(JSON.stringify(cases)); +""" + return json.loads(_run_node(source)) + + +def _eval_transparent_multi_segment_footer_cases() -> dict: + """Drive the transparent footer for a realistic multi-segment tool turn. + + Turn shape: user -> assistant tool/activity segment (no metadata) -> tool + result -> final assistant carrying _usedModel/_turnDuration. The footer must + resolve metadata from the FINAL segment (not querySelector's first match) so + the model label renders exactly once as .lf-model. Regression for the #6068 + round-2 gate where multi-segment turns dropped the label entirely. + """ + ui_js = UI_JS_PATH.read_text(encoding="utf-8") + source = f""" +const src = {ui_js!r}; +function extractFunc(name) {{ + const re = new RegExp('function\\\\s+' + name + '\\\\s*\\\\('); + const start = src.search(re); + if (start < 0) throw new Error(name + ' not found'); + let i = src.indexOf('{{', start); + let depth = 1; i++; + while (depth > 0 && i < src.length) {{ + if (src[i] === '{{') depth++; + else if (src[i] === '}}') depth--; + i++; + }} + return src.slice(start, i); +}} +function getModelLabel(modelId) {{ return String(modelId || 'Unknown'); }} +function esc(s) {{ return String(s == null ? '' : s); }} +function t() {{ return ''; }} +// Multi-segment tool turn: idx 1 is the activity segment (no metadata), +// idx 2 is the final answer that carries the stamped turn metadata. +const S = {{ messages: [ + {{ role: 'user', content: 'hi' }}, + {{ role: 'assistant', content: '', _toolCalls: [{{}}] }}, + {{ role: 'assistant', content: 'answer', _usedModel: 'anthropic/claude-sonnet-4.5', _turnDuration: 1.2 }}, +] }}; +function FakeSeg(idx) {{ return {{ getAttribute(name) {{ return name === 'data-msg-idx' ? String(idx) : null; }} }}; }} +const turn = {{ querySelectorAll(sel) {{ return [FakeSeg(1), FakeSeg(2)]; }} }}; +eval(extractFunc('_compactComposerModelChipLabel')); +eval(extractFunc('_usedModelTurnChipLabel')); +eval(extractFunc('_transparentTurnMetaMessage')); +eval(extractFunc('_transparentTurnFooterHtml')); +const picked = _transparentTurnMetaMessage(turn); +const modelText = _usedModelTurnChipLabel(picked || {{}}); +const modelTitle = picked ? String(picked._usedModel || '') : ''; +const html = _transparentTurnFooterHtml('1.2s', modelText, '', '', 'Done', modelTitle); +const lfModelCount = (html.match(/lf-model/g) || []).length; +console.log(JSON.stringify({{ + pickedUsedModel: picked ? picked._usedModel : null, + pickedContent: picked ? picked.content : null, + modelText, + lfModelCount, + hasTitle: html.indexOf('title="anthropic/claude-sonnet-4.5"') >= 0, +}})); +""" + return json.loads(_run_node(source)) + + +def test_streaming_stamps_used_model_on_assistant_message_and_usage_payload(): + assert "_dm['_usedModel'] = _used_model" in STREAMING_PY + # The served model must be read from the agent AFTER the run — the agent + # mutates agent.model when a fallback fires, so the pre-run resolved_model + # would mis-attribute fallback turns. + assert "_used_model = getattr(agent, 'model', None) or resolved_model or model" in STREAMING_PY + assert "usage['used_model'] = _used_model" in STREAMING_PY + + +def test_models_allowlist_round_trips_used_model_across_save_reload(): + assert '"_usedModel"' in MODELS_PY + assert "_usedModel" in MODELS_PY.split("_SESSION_MESSAGE_DISPLAY_METADATA_KEYS")[1].split(")")[0] + + session = Session(session_id="6068usedmodel", title="Used model") + session.messages = [ + { + "role": "assistant", + "content": "done", + "_firstTokenMs": 250, + "_usedModel": "gpt-5-mini", + }, + ] + session.save() + + reloaded = Session.load("6068usedmodel") + assert reloaded.messages[-1]["_usedModel"] == "gpt-5-mini" + assert reloaded.messages[-1]["_firstTokenMs"] == 250 + + +@pytest.mark.skipif(NODE is None, reason="node not on PATH") +def test_used_model_turn_chip_label_renders_and_suppresses_gateway_duplicate(): + """Behavioral footer chip cases from #6068 (not source-string greps).""" + cases = _eval_used_model_turn_chip_label_cases() + assert cases["present"] == cases["expectedPresent"] + assert cases["present"] # non-empty label when _usedModel is set + assert cases["suppressed"] == "" + # Routing metadata WITHOUT used_model must also suppress the additive chip: + # the gateway formatter owns the label (falling back to _usedModel), so a + # provider-only routing payload must not render two model labels. + assert cases["suppressedRoutingOnly"] == "" + assert cases["absent"] == "" + assert cases["nullMsg"] == "" + # When routing omits used_model, the gateway label falls back to the + # caller-provided model id (the settled footer passes msg._usedModel). + assert cases["gatewayFallback"].startswith(cases["expectedPresent"]) + assert "via" in cases["gatewayFallback"] + + +def test_settled_footer_wires_used_model_chip_in_dom_paths(): + assert "msg-used-model-inline" in UI_JS + assert "_usedModelTurnChipLabel(msg)" in UI_JS + # The settled footer must hand the served model to the gateway formatter as + # its fallback, and skip the generic chip when the transparent turn footer + # owns the model label (one model label per turn). + assert "_formatGatewayModelLabel(String(msg._usedModel||'').trim()||(S.session&&S.session.model)||'', '', routing)" in UI_JS + assert "_transparentFooterOwnsModel" in UI_JS + assert ".msg-used-model-inline" in STYLE_CSS + + +def test_settled_footer_orders_model_chip_after_duration(): + # Match the transparent footer order (elapsed · model · …): the duration + # fragment must be pushed before the used-model fragment. + footer_block = UI_JS.split("const usedModelText=_usedModelTurnChipLabel(msg);", 1)[1] + duration_at = footer_block.index("duration.className='msg-duration-inline'") + used_model_at = footer_block.index("usedModel.className='msg-used-model-inline'") + assert duration_at < used_model_at + + +def test_transparent_turn_footer_includes_model_between_duration_and_ttft(): + assert "function _transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText, modelTitle)" in UI_JS + assert 'class="lf-model"' in UI_JS + assert "modelText=_usedModelTurnChipLabel(msg)" in UI_JS + assert ".transparent-turn-footer .lf-model" in STYLE_CSS + + +def test_transparent_footer_reads_metadata_from_final_segment_not_first(): + # The transparent footer must resolve turn metadata via the dedicated + # last→first segment scan, never querySelector's first-match, which would + # read the activity segment and drop the model label on tool turns. + assert "const msg=_transparentTurnMetaMessage(turn);" in UI_JS + assert "function _transparentTurnMetaMessage(turn)" in UI_JS + # The generic footer yields the model chip to the transparent footer. + assert "_transparentFooterOwnsModel" in UI_JS + + +@pytest.mark.skipif(NODE is None, reason="node not on PATH") +def test_transparent_multi_segment_tool_turn_renders_exactly_one_model_label(): + """#6068 gate round 2: multi-segment tool turns must show the model once.""" + cases = _eval_transparent_multi_segment_footer_cases() + # Metadata resolved from the FINAL assistant segment, not the first. + assert cases["pickedUsedModel"] == "anthropic/claude-sonnet-4.5" + assert cases["pickedContent"] == "answer" + assert cases["modelText"] # non-empty label + # Exactly one .lf-model in the transparent footer (the generic chip is + # suppressed by _transparentFooterOwnsModel), so no duplicate and no loss. + assert cases["lfModelCount"] == 1 + # Non-blocking UX: full model id preserved in a hover title. + assert cases["hasTitle"]