feat: stamp per-turn used model in footer stats (#6068)
Record resolved_model on assistant messages as _usedModel, expose used_model in the usage SSE payload, and render model chips in transparent and settled turn footers while suppressing duplicates when gateway routing already shows used_model.
This commit is contained in:
b3nw committed
1 parent
1dd5cc43ce
commit
30aef85b18
5 files changed
+88
-4
No files matched your search
@@ -7989,6 +7989,7 @@ _SESSION_MESSAGE_DISPLAY_METADATA_KEYS = (
|
||||
"_turnTps",
|
||||
"_turnUsage",
|
||||
"_firstTokenMs",
|
||||
"_usedModel",
|
||||
"_gatewayRouting",
|
||||
"_statusCard",
|
||||
"_anchor_stream_id",
|
||||
|
||||
@@ -9936,6 +9936,9 @@ def _run_agent_streaming(
|
||||
_ttft_ms = meter().get_ttft_ms(stream_id)
|
||||
if _ttft_ms is not None:
|
||||
_dm['_firstTokenMs'] = _ttft_ms
|
||||
_used_model = resolved_model or model
|
||||
if _used_model:
|
||||
_dm['_usedModel'] = _used_model
|
||||
break
|
||||
# Persist context window data on the session so the context-ring
|
||||
# indicator survives a page reload (#1318). Must run BEFORE
|
||||
@@ -10323,6 +10326,9 @@ def _run_agent_streaming(
|
||||
_ttft_ms = meter().get_ttft_ms(stream_id)
|
||||
if _ttft_ms is not None:
|
||||
usage['ttft_ms'] = _ttft_ms
|
||||
_used_model = resolved_model or model
|
||||
if _used_model:
|
||||
usage['used_model'] = _used_model
|
||||
# Include context window data from the agent's compressor for the UI indicator.
|
||||
# The session-level persistence happens above (before s.save()) so the values
|
||||
# survive a page reload; this block only populates the live SSE usage payload.
|
||||
|
||||
@@ -4411,6 +4411,7 @@ body.resizing .sidebar{transition:none!important;}
|
||||
.transparent-turn-footer .lf-time{color:var(--text);}
|
||||
.transparent-turn-footer .lf-sep{opacity:.4;}
|
||||
.transparent-turn-footer .lf-status{color:var(--muted);opacity:.85;}
|
||||
.transparent-turn-footer .lf-model{color:var(--muted);opacity:.85;}
|
||||
.transparent-turn-footer .lf-ttft{color:var(--accent);opacity:.9;}
|
||||
.transparent-turn-footer .lf-tokens{font-variant-numeric:tabular-nums;}
|
||||
|
||||
@@ -6110,6 +6111,7 @@ main.main > #mainPlugin{display:none;}
|
||||
.msg-usage-inline,
|
||||
.msg-duration-inline,
|
||||
.msg-gateway-inline,
|
||||
.msg-used-model-inline,
|
||||
.gateway-failover-inline,
|
||||
.msg-model-warning-inline {
|
||||
font-size: 11px;
|
||||
|
||||
+24
-4
@@ -7030,6 +7030,14 @@ function _formatGatewayModelLabel(modelId,labelText,routing){
|
||||
const via=_gatewayRoutingLabel(routing);
|
||||
return via?`${base} ${via}`:base;
|
||||
}
|
||||
function _usedModelTurnChipLabel(msg){
|
||||
if(!msg)return'';
|
||||
const routing=msg._gatewayRouting||null;
|
||||
if(routing&&String(routing.used_model||'').trim())return'';
|
||||
const usedModel=String(msg._usedModel||'').trim();
|
||||
if(!usedModel)return'';
|
||||
return _compactComposerModelChipLabel(usedModel,getModelLabel(usedModel));
|
||||
}
|
||||
function _gatewayRoutingFailoverText(routing){
|
||||
if(!routing||!routing.has_failover)return'';
|
||||
const attempts=Array.isArray(routing.routing)?routing.routing:[];
|
||||
@@ -11754,9 +11762,10 @@ function _applyTransparentRowFading(turn){
|
||||
// Mirrors the live run-status line for settled turns in transparent
|
||||
// mode. Shows duration, first-token time, token usage, and final status.
|
||||
// Only rendered for turns that have transparent event rows.
|
||||
function _transparentTurnFooterHtml(durationText, ttftText, tokensText, statusText){
|
||||
function _transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText){
|
||||
const parts=[];
|
||||
if(durationText) parts.push(`<span class="lf-time">${esc(durationText)}</span>`);
|
||||
if(modelText) parts.push(`<span class="lf-model">${esc(modelText)}</span>`);
|
||||
if(ttftText) parts.push(`<span class="lf-ttft" title="${esc(t('first_token_time')||'Time to first token')}">TTFT ${esc(ttftText)}</span>`);
|
||||
if(tokensText) parts.push(`<span class="lf-tokens">${esc(tokensText)}</span>`);
|
||||
if(statusText) parts.push(`<span class="lf-status">${esc(statusText)}</span>`);
|
||||
@@ -11775,10 +11784,11 @@ function _renderTransparentTurnFooter(turn, opts){
|
||||
return;
|
||||
}
|
||||
const durationText=opts&&opts.durationText||'';
|
||||
const modelText=opts&&opts.modelText||'';
|
||||
const ttftText=opts&&opts.ttftText||'';
|
||||
const tokensText=opts&&opts.tokensText||'';
|
||||
const statusText=opts&&opts.statusText||(t('done')||'Done');
|
||||
const html=_transparentTurnFooterHtml(durationText, ttftText, tokensText, statusText);
|
||||
const html=_transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText);
|
||||
let footer=turn.querySelector('.transparent-turn-footer');
|
||||
if(!html){
|
||||
if(footer) footer.remove();
|
||||
@@ -16595,12 +16605,13 @@ function renderMessages(options){
|
||||
// Worklog above the final answer.
|
||||
const compactWorklogForMessage=isCompactWorklogMode()&&(toolCallAssistantIdxs.has(mi)||assistantThinking.has(mi));
|
||||
const durationText=compactWorklogForMessage?'':_formatTurnDuration(msg._turnDuration);
|
||||
if(!hasTurnUsage&&!durationText&&!gatewayText&&!failoverText&&!modelWarningText) continue;
|
||||
const usedModelText=_usedModelTurnChipLabel(msg);
|
||||
if(!hasTurnUsage&&!durationText&&!gatewayText&&!failoverText&&!modelWarningText&&!usedModelText) continue;
|
||||
const seg=assistantSegments.get(mi);
|
||||
const row=seg?seg.closest('.assistant-turn'):null;
|
||||
const footerRows=row?row.querySelectorAll('.msg-foot'):[];
|
||||
const targetFoot=footerRows.length?footerRows[footerRows.length-1]:null;
|
||||
if(!targetFoot||targetFoot.querySelector('.msg-usage-inline,.msg-duration-inline,.msg-gateway-inline,.gateway-failover-inline,.msg-model-warning-inline')) continue;
|
||||
if(!targetFoot||targetFoot.querySelector('.msg-usage-inline,.msg-duration-inline,.msg-gateway-inline,.gateway-failover-inline,.msg-model-warning-inline,.msg-used-model-inline')) continue;
|
||||
const fragments=[];
|
||||
if(modelWarningText){
|
||||
const warning=document.createElement('span');
|
||||
@@ -16620,6 +16631,12 @@ function renderMessages(options){
|
||||
gateway.textContent=gatewayText;
|
||||
fragments.push(gateway);
|
||||
}
|
||||
if(usedModelText){
|
||||
const usedModel=document.createElement('span');
|
||||
usedModel.className='msg-used-model-inline';
|
||||
usedModel.textContent=usedModelText;
|
||||
fragments.push(usedModel);
|
||||
}
|
||||
if(durationText){
|
||||
const duration=document.createElement('span');
|
||||
duration.className='msg-duration-inline';
|
||||
@@ -16676,6 +16693,7 @@ function renderMessages(options){
|
||||
// Find the corresponding message to read duration/usage.
|
||||
const seg=turn.querySelector('.assistant-segment');
|
||||
let durationText='';
|
||||
let modelText='';
|
||||
let ttftText='';
|
||||
let tokensText='';
|
||||
if(seg){
|
||||
@@ -16683,6 +16701,7 @@ function renderMessages(options){
|
||||
if(mi!=null){
|
||||
const msg=S.messages[Number(mi)]||{};
|
||||
if(msg._turnDuration!=null) durationText=_formatTurnDuration(msg._turnDuration);
|
||||
modelText=_usedModelTurnChipLabel(msg);
|
||||
if(msg._firstTokenMs!=null) ttftText=_formatFirstToken(msg._firstTokenMs);
|
||||
if(msg._turnUsage){
|
||||
const inTok=msg._turnUsage.input_tokens||0;
|
||||
@@ -16693,6 +16712,7 @@ function renderMessages(options){
|
||||
}
|
||||
_renderTransparentTurnFooter(turn,{
|
||||
durationText,
|
||||
modelText,
|
||||
ttftText,
|
||||
tokensText,
|
||||
statusText: t('done')||'Done',
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Regression coverage for #6068 per-turn used-model footer instrumentation."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from api.models import Session
|
||||
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8")
|
||||
MODELS_PY = (REPO / "api" / "models.py").read_text(encoding="utf-8")
|
||||
UI_JS = (REPO / "static" / "ui.js").read_text(encoding="utf-8")
|
||||
STYLE_CSS = (REPO / "static" / "style.css").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_streaming_stamps_used_model_on_assistant_message_and_usage_payload():
|
||||
assert "_dm['_usedModel'] = _used_model" in STREAMING_PY
|
||||
assert "_used_model = resolved_model or model" in STREAMING_PY
|
||||
assert "usage['used_model'] = _used_model" in STREAMING_PY
|
||||
|
||||
|
||||
def test_models_allowlist_round_trips_used_model_across_save_reload():
|
||||
assert '"_usedModel"' in MODELS_PY
|
||||
assert "_usedModel" in MODELS_PY.split("_SESSION_MESSAGE_DISPLAY_METADATA_KEYS")[1].split(")")[0]
|
||||
|
||||
session = Session(session_id="6068usedmodel", title="Used model")
|
||||
session.messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "done",
|
||||
"_firstTokenMs": 250,
|
||||
"_usedModel": "gpt-5-mini",
|
||||
},
|
||||
]
|
||||
session.save()
|
||||
|
||||
reloaded = Session.load("6068usedmodel")
|
||||
assert reloaded.messages[-1]["_usedModel"] == "gpt-5-mini"
|
||||
assert reloaded.messages[-1]["_firstTokenMs"] == 250
|
||||
|
||||
|
||||
def test_settled_footer_renders_used_model_chip_and_suppresses_gateway_duplicate():
|
||||
assert "function _usedModelTurnChipLabel" in UI_JS
|
||||
assert "msg-used-model-inline" in UI_JS
|
||||
assert "_usedModelTurnChipLabel(msg)" in UI_JS
|
||||
assert "routing.used_model" in UI_JS
|
||||
assert "msg._usedModel" in UI_JS
|
||||
assert "_compactComposerModelChipLabel(usedModel,getModelLabel(usedModel))" in UI_JS
|
||||
assert ".msg-used-model-inline" in STYLE_CSS
|
||||
|
||||
|
||||
def test_transparent_turn_footer_includes_model_between_duration_and_ttft():
|
||||
assert "function _transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText)" in UI_JS
|
||||
assert 'class="lf-model"' in UI_JS
|
||||
assert "modelText=_usedModelTurnChipLabel(msg)" in UI_JS
|
||||
assert ".transparent-turn-footer .lf-model" in STYLE_CSS
|
||||
Reference in new issue
Block a user