hermes-webui/tests/test_issue6068_used_model_footer.py

"""Regression coverage for #6068 per-turn used-model footer instrumentation."""

import json
import shutil
import subprocess
from pathlib import Path

import pytest

from api.models import Session


REPO = Path(__file__).resolve().parents[1]
NODE = shutil.which("node")
UI_JS_PATH = REPO / "static" / "ui.js"
STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8")
MODELS_PY = (REPO / "api" / "models.py").read_text(encoding="utf-8")
UI_JS = (REPO / "static" / "ui.js").read_text(encoding="utf-8")
STYLE_CSS = (REPO / "static" / "style.css").read_text(encoding="utf-8")


def _run_node(source: str) -> str:
    result = subprocess.run(
        [NODE],
        input=source,
        cwd=str(REPO),
        capture_output=True,
        encoding="utf-8",
        text=True,
        timeout=30,
    )
    if result.returncode != 0:
        raise RuntimeError(result.stderr)
    return result.stdout.strip()


def _eval_used_model_turn_chip_label_cases() -> dict:
    ui_js = UI_JS_PATH.read_text(encoding="utf-8")
    source = f"""
const src = {ui_js!r};
function extractFunc(name) {{
  const re = new RegExp('function\\\\s+' + name + '\\\\s*\\\\(');
  const start = src.search(re);
  if (start < 0) throw new Error(name + ' not found');
  let i = src.indexOf('{{', start);
  let depth = 1; i++;
  while (depth > 0 && i < src.length) {{
    if (src[i] === '{{') depth++;
    else if (src[i] === '}}') depth--;
    i++;
  }}
  return src.slice(start, i);
}}
function getModelLabel(modelId) {{
  return String(modelId || 'Unknown');
}}
eval(extractFunc('_compactComposerModelChipLabel'));
eval(extractFunc('_usedModelTurnChipLabel'));
eval(extractFunc('_gatewayProviderName'));
eval(extractFunc('_gatewayRoutingLabel'));
eval(extractFunc('_formatGatewayModelLabel'));
const modelId = 'gpt-5-mini';
const expectedPresent = _compactComposerModelChipLabel(modelId, getModelLabel(modelId));
const cases = {{
  present: _usedModelTurnChipLabel({{ _usedModel: modelId }}),
  expectedPresent,
  suppressed: _usedModelTurnChipLabel({{
    _usedModel: modelId,
    _gatewayRouting: {{ used_model: 'deepseek-v3.2' }},
  }}),
  suppressedRoutingOnly: _usedModelTurnChipLabel({{
    _usedModel: modelId,
    _gatewayRouting: {{ used_provider: 'openrouter' }},
  }}),
  absent: _usedModelTurnChipLabel({{}}),
  nullMsg: _usedModelTurnChipLabel(null),
  gatewayFallback: _formatGatewayModelLabel(modelId, getModelLabel(modelId), {{ used_provider: 'openrouter' }}),
}};
console.log(JSON.stringify(cases));
"""
    return json.loads(_run_node(source))


def _eval_transparent_multi_segment_footer_cases() -> dict:
    """Drive the transparent footer for a realistic multi-segment tool turn.

    Turn shape: user -> assistant tool/activity segment (no metadata) -> tool
    result -> final assistant carrying _usedModel/_turnDuration. The footer must
    resolve metadata from the FINAL segment (not querySelector's first match) so
    the model label renders exactly once as .lf-model. Regression for the #6068
    round-2 gate where multi-segment turns dropped the label entirely.
    """
    ui_js = UI_JS_PATH.read_text(encoding="utf-8")
    source = f"""
const src = {ui_js!r};
function extractFunc(name) {{
  const re = new RegExp('function\\\\s+' + name + '\\\\s*\\\\(');
  const start = src.search(re);
  if (start < 0) throw new Error(name + ' not found');
  let i = src.indexOf('{{', start);
  let depth = 1; i++;
  while (depth > 0 && i < src.length) {{
    if (src[i] === '{{') depth++;
    else if (src[i] === '}}') depth--;
    i++;
  }}
  return src.slice(start, i);
}}
function getModelLabel(modelId) {{ return String(modelId || 'Unknown'); }}
function esc(s) {{ return String(s == null ? '' : s); }}
function t() {{ return ''; }}
// Multi-segment tool turn: idx 1 is the activity segment (no metadata),
// idx 2 is the final answer that carries the stamped turn metadata.
const S = {{ messages: [
  {{ role: 'user', content: 'hi' }},
  {{ role: 'assistant', content: '', _toolCalls: [{{}}] }},
  {{ role: 'assistant', content: 'answer', _usedModel: 'anthropic/claude-sonnet-4.5', _turnDuration: 1.2 }},
] }};
function FakeSeg(idx) {{ return {{ getAttribute(name) {{ return name === 'data-msg-idx' ? String(idx) : null; }} }}; }}
const turn = {{ querySelectorAll(sel) {{ return [FakeSeg(1), FakeSeg(2)]; }} }};
eval(extractFunc('_compactComposerModelChipLabel'));
eval(extractFunc('_usedModelTurnChipLabel'));
eval(extractFunc('_transparentTurnMetaMessage'));
eval(extractFunc('_transparentTurnFooterHtml'));
const picked = _transparentTurnMetaMessage(turn);
const modelText = _usedModelTurnChipLabel(picked || {{}});
const modelTitle = picked ? String(picked._usedModel || '') : '';
const html = _transparentTurnFooterHtml('1.2s', modelText, '', '', 'Done', modelTitle);
const lfModelCount = (html.match(/lf-model/g) || []).length;
console.log(JSON.stringify({{
  pickedUsedModel: picked ? picked._usedModel : null,
  pickedContent: picked ? picked.content : null,
  modelText,
  lfModelCount,
  hasTitle: html.indexOf('title="anthropic/claude-sonnet-4.5"') >= 0,
}}));
"""
    return json.loads(_run_node(source))


def test_streaming_stamps_used_model_on_assistant_message_and_usage_payload():
    assert "_dm['_usedModel'] = _used_model" in STREAMING_PY
    # The served model must be read from the agent AFTER the run — the agent
    # mutates agent.model when a fallback fires, so the pre-run resolved_model
    # would mis-attribute fallback turns.
    assert "_used_model = getattr(agent, 'model', None) or resolved_model or model" in STREAMING_PY
    assert "usage['used_model'] = _used_model" in STREAMING_PY


def test_models_allowlist_round_trips_used_model_across_save_reload():
    assert '"_usedModel"' in MODELS_PY
    assert "_usedModel" in MODELS_PY.split("_SESSION_MESSAGE_DISPLAY_METADATA_KEYS")[1].split(")")[0]

    session = Session(session_id="6068usedmodel", title="Used model")
    session.messages = [
        {
            "role": "assistant",
            "content": "done",
            "_firstTokenMs": 250,
            "_usedModel": "gpt-5-mini",
        },
    ]
    session.save()

    reloaded = Session.load("6068usedmodel")
    assert reloaded.messages[-1]["_usedModel"] == "gpt-5-mini"
    assert reloaded.messages[-1]["_firstTokenMs"] == 250


@pytest.mark.skipif(NODE is None, reason="node not on PATH")
def test_used_model_turn_chip_label_renders_and_suppresses_gateway_duplicate():
    """Behavioral footer chip cases from #6068 (not source-string greps)."""
    cases = _eval_used_model_turn_chip_label_cases()
    assert cases["present"] == cases["expectedPresent"]
    assert cases["present"]  # non-empty label when _usedModel is set
    assert cases["suppressed"] == ""
    # Routing metadata WITHOUT used_model must also suppress the additive chip:
    # the gateway formatter owns the label (falling back to _usedModel), so a
    # provider-only routing payload must not render two model labels.
    assert cases["suppressedRoutingOnly"] == ""
    assert cases["absent"] == ""
    assert cases["nullMsg"] == ""
    # When routing omits used_model, the gateway label falls back to the
    # caller-provided model id (the settled footer passes msg._usedModel).
    assert cases["gatewayFallback"].startswith(cases["expectedPresent"])
    assert "via" in cases["gatewayFallback"]


def test_settled_footer_wires_used_model_chip_in_dom_paths():
    assert "msg-used-model-inline" in UI_JS
    assert "_usedModelTurnChipLabel(msg)" in UI_JS
    # The settled footer must hand the served model to the gateway formatter as
    # its fallback, and skip the generic chip when the transparent turn footer
    # owns the model label (one model label per turn).
    assert "_formatGatewayModelLabel(String(msg._usedModel||'').trim()||(S.session&&S.session.model)||'', '', routing)" in UI_JS
    assert "_transparentFooterOwnsModel" in UI_JS
    assert ".msg-used-model-inline" in STYLE_CSS


def test_settled_footer_orders_model_chip_after_duration():
    # Match the transparent footer order (elapsed · model · …): the duration
    # fragment must be pushed before the used-model fragment.
    footer_block = UI_JS.split("const usedModelText=_usedModelTurnChipLabel(msg);", 1)[1]
    duration_at = footer_block.index("duration.className='msg-duration-inline'")
    used_model_at = footer_block.index("usedModel.className='msg-used-model-inline'")
    assert duration_at < used_model_at


def test_transparent_turn_footer_includes_model_between_duration_and_ttft():
    assert "function _transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText, modelTitle)" in UI_JS
    assert 'class="lf-model"' in UI_JS
    assert "modelText=_usedModelTurnChipLabel(msg)" in UI_JS
    assert ".transparent-turn-footer .lf-model" in STYLE_CSS


def test_transparent_footer_reads_metadata_from_final_segment_not_first():
    # The transparent footer must resolve turn metadata via the dedicated
    # last→first segment scan, never querySelector's first-match, which would
    # read the activity segment and drop the model label on tool turns.
    assert "const msg=_transparentTurnMetaMessage(turn);" in UI_JS
    assert "function _transparentTurnMetaMessage(turn)" in UI_JS
    # The generic footer yields the model chip to the transparent footer.
    assert "_transparentFooterOwnsModel" in UI_JS


@pytest.mark.skipif(NODE is None, reason="node not on PATH")
def test_transparent_multi_segment_tool_turn_renders_exactly_one_model_label():
    """#6068 gate round 2: multi-segment tool turns must show the model once."""
    cases = _eval_transparent_multi_segment_footer_cases()
    # Metadata resolved from the FINAL assistant segment, not the first.
    assert cases["pickedUsedModel"] == "anthropic/claude-sonnet-4.5"
    assert cases["pickedContent"] == "answer"
    assert cases["modelText"]  # non-empty label
    # Exactly one .lf-model in the transparent footer (the generic chip is
    # suppressed by _transparentFooterOwnsModel), so no duplicate and no loss.
    assert cases["lfModelCount"] == 1
    # Non-blocking UX: full model id preserved in a hover title.
    assert cases["hasTitle"]