feat: show LLM Gateway routing metadata

This commit is contained in:
Michael Lam authored and test committed 2026-05-05 02:26:55 +00:00
1 parent fcc83284e3
commit c94ec31dec
12 files changed
+376 -10

No files matched your search

+1 -1
View File
@@ -238,6 +238,7 @@ Remaining gaps and forward work live in [Forward Work](#forward-work) below.
- [x] Transcript-summary card at 10+ rounds
- [x] Sidebar dedup keying on per-conversation identity (distinct chats from same platform stay separate)
- [x] Gateway session sync skips dup / delete options for external sessions
- [x] LLM Gateway routing metadata display — assistant turns and session metadata show the served model/provider, failover path, and model-switch warnings when response metadata includes `used_provider`, `used_model`, or `routing` (#732)
### MCP integration
- [x] MCP server management UI (System Settings → MCP Servers)
@@ -284,7 +285,6 @@ Remaining gaps and forward work live in [Forward Work](#forward-work) below.
- **Insights / monitoring suite** — agent heartbeat + alerts (#716), quota / rate-limit display (#706), data tabs (#722), monitor dashboard concepts (#766, #721)
- **Native MCP server expose** — Hermes WebUI as an MCP server for direct agent integration (#733)
- **Provider failover status** — show active provider per model + LLM Gateway failover state (#732)
- **Teams / agents management panel** — editable names, roles, assignments (#719)
- **Web UI profile model alignment with Hermes runtime** — design parity (#749)
- **DOM windowing / message virtualization** — for sessions with hundreds of messages (#734)
+6
View File
@@ -331,6 +331,7 @@ class Session:
compression_anchor_message_key=None,
context_length=None, threshold_tokens=None,
last_prompt_tokens=None,
gateway_routing=None, gateway_routing_history=None,
parent_session_id: str=None,
enabled_toolsets=None,
**kwargs):
@@ -361,6 +362,8 @@ class Session:
self.context_length = context_length
self.threshold_tokens = threshold_tokens
self.last_prompt_tokens = last_prompt_tokens
self.gateway_routing = gateway_routing if isinstance(gateway_routing, dict) else None
self.gateway_routing_history = gateway_routing_history if isinstance(gateway_routing_history, list) else []
self.parent_session_id = parent_session_id
self.is_cli_session = bool(kwargs.get('is_cli_session', False))
self.source_tag = kwargs.get('source_tag')
@@ -405,6 +408,7 @@ class Session:
'pending_user_message', 'pending_attachments', 'pending_started_at',
'compression_anchor_visible_idx', 'compression_anchor_message_key',
'context_length', 'threshold_tokens', 'last_prompt_tokens',
'gateway_routing', 'gateway_routing_history',
'parent_session_id',
'is_cli_session', 'source_tag', 'raw_source', 'session_source', 'source_label',
'enabled_toolsets',
@@ -567,6 +571,8 @@ class Session:
'context_length': self.context_length,
'threshold_tokens': self.threshold_tokens,
'last_prompt_tokens': self.last_prompt_tokens,
'gateway_routing': self.gateway_routing,
'gateway_routing_history': self.gateway_routing_history,
# Only emit 'parent_session_id' when set (the /branch fork link, #1342).
# Sessions without a fork must not leak None — see test_session_lineage_metadata_api.
**({'parent_session_id': self.parent_session_id} if self.parent_session_id else {}),
+161
View File
@@ -69,6 +69,152 @@ _API_SAFE_MSG_KEYS = {'role', 'content', 'tool_calls', 'tool_call_id', 'name', '
_NATIVE_IMAGE_MAX_BYTES = 20 * 1024 * 1024
_GATEWAY_ROUTING_TOP_LEVEL_KEYS = {
'used_provider',
'used_model',
'requested_provider',
'requested_model',
}
_GATEWAY_ROUTING_CONTAINER_KEYS = (
'llm_gateway',
'gateway',
'metadata',
'response_metadata',
'routing_metadata',
'usage',
)
_GATEWAY_ROUTING_ATTEMPT_KEYS = {
'provider', 'model', 'status', 'reason', 'selection_reason', 'score',
'latency_ms', 'error', 'timestamp', 'selected', 'attempt', 'attempt_index',
}
def _clean_gateway_routing_scalar(value):
if value is None:
return None
if isinstance(value, (str, int, float, bool)):
text = str(value).strip()
if not text:
return None
return value if isinstance(value, (int, float, bool)) else text[:240]
return None
def _find_gateway_metadata_payload(payload):
if not isinstance(payload, dict):
return None
if any(k in payload for k in _GATEWAY_ROUTING_TOP_LEVEL_KEYS) or isinstance(payload.get('routing'), list):
return payload
for key in _GATEWAY_ROUTING_CONTAINER_KEYS:
nested = payload.get(key)
found = _find_gateway_metadata_payload(nested)
if found:
return found
return None
def _normalize_gateway_routing_metadata(payload, requested_model=None, requested_provider=None):
"""Return safe LLM Gateway routing metadata, or None when absent.
LLM Gateway response metadata can contain provider/model routing details,
but WebUI must only persist display-safe scalars and a bounded routing list.
Secrets or provider-specific request objects are deliberately ignored.
"""
src = _find_gateway_metadata_payload(payload)
if not src:
return None
normalized = {}
for key in _GATEWAY_ROUTING_TOP_LEVEL_KEYS:
value = _clean_gateway_routing_scalar(src.get(key))
if value is not None:
normalized[key] = value
if 'requested_model' not in normalized:
fallback_model = _clean_gateway_routing_scalar(requested_model)
if fallback_model is not None:
normalized['requested_model'] = fallback_model
if 'requested_provider' not in normalized:
fallback_provider = _clean_gateway_routing_scalar(requested_provider)
if fallback_provider is not None:
normalized['requested_provider'] = fallback_provider
routing = []
raw_routing = src.get('routing')
if isinstance(raw_routing, list):
for attempt in raw_routing[:12]:
if not isinstance(attempt, dict):
continue
clean_attempt = {}
for key in _GATEWAY_ROUTING_ATTEMPT_KEYS:
value = _clean_gateway_routing_scalar(attempt.get(key))
if value is not None:
clean_attempt[key] = value
if clean_attempt:
routing.append(clean_attempt)
if routing:
normalized['routing'] = routing
used_provider = str(normalized.get('used_provider') or '').strip().lower()
requested_provider_norm = str(normalized.get('requested_provider') or '').strip().lower()
used_model = str(normalized.get('used_model') or '').strip().lower()
requested_model_norm = str(normalized.get('requested_model') or '').strip().lower()
provider_changed = bool(used_provider and requested_provider_norm and used_provider != requested_provider_norm)
model_changed = bool(used_model and requested_model_norm and used_model != requested_model_norm)
attempted_providers = [
str(a.get('provider') or '').strip().lower()
for a in routing
if a.get('provider')
]
distinct_attempted_providers = {p for p in attempted_providers if p}
failed_before_selection = any(
str(a.get('status') or '').strip().lower() in {'failed', 'error', 'timeout', 'rejected'}
for a in routing
)
has_failover = bool(provider_changed or len(distinct_attempted_providers) > 1 or failed_before_selection)
if not (
normalized.get('used_provider') or normalized.get('used_model') or routing or provider_changed or model_changed
):
return None
normalized['provider_changed'] = provider_changed
normalized['model_changed'] = model_changed
normalized['has_failover'] = has_failover
return normalized
def _extract_gateway_routing_metadata(agent, result, requested_model=None, requested_provider=None):
candidates = []
if isinstance(result, dict):
candidates.extend([
result.get('llm_gateway'),
result.get('gateway'),
result.get('metadata'),
result.get('response_metadata'),
result.get('routing_metadata'),
result.get('usage'),
result,
])
for attr in (
'llm_gateway_metadata',
'gateway_metadata',
'last_response_metadata',
'response_metadata',
'routing_metadata',
'last_usage',
):
if agent is not None:
candidates.append(getattr(agent, attr, None))
for candidate in candidates:
normalized = _normalize_gateway_routing_metadata(
candidate,
requested_model=requested_model,
requested_provider=requested_provider,
)
if normalized:
return normalized
return None
def _build_agent_thread_env(profile_runtime_env: dict | None, workspace: str, session_id: str, profile_home: str) -> dict:
"""Build thread-local agent env with per-run values overriding profile defaults.
@@ -2497,12 +2643,25 @@ def _run_agent_streaming(
_turn_tps = None
if output_tokens and _turn_duration_seconds > 0:
_turn_tps = round(float(output_tokens) / _turn_duration_seconds, 1)
_gateway_routing = _extract_gateway_routing_metadata(
agent,
result,
requested_model=resolved_model or model,
requested_provider=resolved_provider,
)
if _gateway_routing:
s.gateway_routing = _gateway_routing
_history = list(getattr(s, 'gateway_routing_history', None) or [])
_history.append(_gateway_routing)
s.gateway_routing_history = _history[-50:]
if s.messages:
for _dm in reversed(s.messages):
if isinstance(_dm, dict) and _dm.get('role') == 'assistant':
_dm['_turnDuration'] = round(_turn_duration_seconds, 3)
if _turn_tps is not None:
_dm['_turnTps'] = _turn_tps
if _gateway_routing:
_dm['_gatewayRouting'] = _gateway_routing
break
# Persist context window data on the session so the context-ring
# indicator survives a page reload (#1318). Must run BEFORE
@@ -2559,6 +2718,8 @@ def _run_agent_streaming(
}
if _turn_tps is not None:
usage['tps'] = _turn_tps
if _gateway_routing:
usage['gateway_routing'] = _gateway_routing
# Include context window data from the agent's compressor for the UI indicator.
# The session-level persistence happens above (before s.save()) so the values
# survive a page reload; this block only populates the live SSE usage payload.
Binary file not shown.

After

Width:  |  Height:  |  Size: 52 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 60 KiB

+6
View File
@@ -910,6 +910,12 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
if(typeof d.usage.tps==='number'&&d.usage.tps>0){
lastAsst._turnTps=d.usage.tps;
}
if(d.usage.gateway_routing){
lastAsst._gatewayRouting=d.usage.gateway_routing;
if(S.session)S.session.gateway_routing=d.usage.gateway_routing;
if(S.session&&Array.isArray(S.session.gateway_routing_history))S.session.gateway_routing_history.push(d.usage.gateway_routing);
else if(S.session)S.session.gateway_routing_history=[d.usage.gateway_routing];
}
}
}
if(d.session.tool_calls&&d.session.tool_calls.length){
+11 -1
View File
@@ -25,6 +25,15 @@ let _sessionObservedStreaming = null;
const _sessionStreamingById = new Map();
const _sessionListSnapshotById = new Map();
function _formatSessionModelWithGateway(s){
if(!s||!s.model)return'';
const routing=(typeof _latestGatewayRoutingForSession==='function')?_latestGatewayRoutingForSession(s):(s.gateway_routing||null);
if(typeof _formatGatewayModelLabel==='function'){
return _formatGatewayModelLabel(s.model,s.model,routing)||s.model;
}
return s.model;
}
function _getSessionViewedCounts() {
if (_sessionViewedCounts !== null) return _sessionViewedCounts;
try {
@@ -2338,7 +2347,8 @@ function renderSessionListFromCache(){
: `${msgCount} msg${msgCount===1?'':'s'}`;
metaBits.push(msgLabel);
if(childCount>0) metaBits.push(t('session_meta_children', childCount));
if(s.model) metaBits.push(s.model);
const modelMeta=_formatSessionModelWithGateway(s);
if(modelMeta) metaBits.push(modelMeta);
const sourceLabel=_getChannelLabel(s);
if(s.is_cli_session&&sourceLabel) metaBits.push(sourceLabel);
if(readOnly) metaBits.push('read-only');
+13 -1
View File
@@ -2794,13 +2794,25 @@ main.main.showing-logs > #mainLogs{display:flex;}
gap: 8px;
}
.msg-usage-inline,
.msg-duration-inline {
.msg-duration-inline,
.msg-gateway-inline,
.gateway-failover-inline,
.msg-model-warning-inline {
font-size: 11px;
color: var(--muted);
opacity: .7;
flex: 0 0 auto;
font-variant-numeric: tabular-nums;
}
.gateway-failover-inline {
color: var(--accent);
opacity: .9;
}
.msg-model-warning-inline {
color: var(--error);
opacity: .95;
font-weight: 600;
}
.msg-foot-with-usage .msg-time,
.msg-foot-with-usage .msg-actions {
opacity: 0;
+70 -5
View File
@@ -739,9 +739,11 @@ function syncModelChip(){
}
const opt=_selectedModelOption();
const text=opt?opt.textContent:getModelLabel(sel.value||'');
label.textContent=text;
if(mobileLabel) mobileLabel.textContent=text;
chip.title=sel.value||'Conversation model';
const gatewayRouting=_latestGatewayRoutingForSession(S.session);
const displayText=_formatGatewayModelLabel(sel.value||'',text,gatewayRouting)||text;
label.textContent=displayText;
if(mobileLabel) mobileLabel.textContent=displayText;
chip.title=gatewayRouting?`${sel.value||'Conversation model'} ${_gatewayRoutingLabel(gatewayRouting)}`:(sel.value||'Conversation model');
chip.classList.toggle('active',!!(dd&&dd.classList.contains('open')));
if(mobileAction) mobileAction.classList.toggle('active',!!(dd&&dd.classList.contains('open')));
}
@@ -1647,6 +1649,47 @@ function getModelLabel(modelId){
return _last || 'Unknown';
}
function _gatewayProviderName(provider){
const text=String(provider||'').trim();
if(!text)return'';
return text.replace(/^custom:/,'').replace(/[-_]/g,' ').replace(/\b\w/g,c=>c.toUpperCase());
}
function _gatewayRoutingLabel(routing){
if(!routing)return'';
const provider=_gatewayProviderName(routing.used_provider||routing.provider);
return provider?`via ${provider}`:'';
}
function _formatGatewayModelLabel(modelId,labelText,routing){
if(!routing)return'';
const usedModel=String(routing.used_model||'').trim();
const base=usedModel?getModelLabel(usedModel):(labelText||getModelLabel(modelId));
const via=_gatewayRoutingLabel(routing);
return via?`${base} ${via}`:base;
}
function _gatewayRoutingFailoverText(routing){
if(!routing||!routing.has_failover)return'';
const attempts=Array.isArray(routing.routing)?routing.routing:[];
const providers=attempts.map(a=>_gatewayProviderName(a&&a.provider)).filter(Boolean);
const unique=[];providers.forEach(p=>{if(!unique.includes(p))unique.push(p);});
if(unique.length>=2)return`Failover: ${unique[0]} → ${unique[unique.length-1]}`;
const from=_gatewayProviderName(routing.requested_provider);
const to=_gatewayProviderName(routing.used_provider);
if(from&&to&&from!==to)return`Failover: ${from} → ${to}`;
return'Gateway failover detected';
}
function _gatewayModelWarningText(routing){
if(!routing||!routing.model_changed)return'';
const requested=getModelLabel(routing.requested_model||'requested model');
const used=getModelLabel(routing.used_model||'served model');
return`Model switched: ${requested} → ${used}`;
}
function _latestGatewayRoutingForSession(session){
if(!session)return null;
if(session.gateway_routing)return session.gateway_routing;
const history=Array.isArray(session.gateway_routing_history)?session.gateway_routing_history:[];
return history.length?history[history.length-1]:null;
}
function _stripXmlToolCallsDisplay(s){
// Strip <function_calls>...</function_calls> blocks emitted by DeepSeek and
// similar models in their raw response text. These are processed separately
@@ -4345,19 +4388,41 @@ function renderMessages(){
for(const mi of renderedAssistantIdxs){
const msg=S.messages[mi]||{};
if(msg.role!=='assistant') continue;
const routing=msg._gatewayRouting||null;
const gatewayText=_formatGatewayModelLabel(S.session&&S.session.model||'', '', routing);
const failoverText=_gatewayRoutingFailoverText(routing);
const modelWarningText=_gatewayModelWarningText(routing);
const hasTurnUsage=!!msg._turnUsage;
const compactActivityForMessage=isSimplifiedToolCalling()&&(
assistantThinking.has(mi)||
(S.toolCalls||[]).some(tc=>tc&&(tc.assistant_msg_idx!==undefined?tc.assistant_msg_idx:-1)===mi)
);
const durationText=compactActivityForMessage?'':_formatTurnDuration(msg._turnDuration);
if(!hasTurnUsage&&!durationText) continue;
if(!hasTurnUsage&&!durationText&&!gatewayText&&!failoverText&&!modelWarningText) continue;
const seg=assistantSegments.get(mi);
const row=seg?seg.closest('.assistant-turn'):null;
const footerRows=row?row.querySelectorAll('.msg-foot'):[];
const targetFoot=footerRows.length?footerRows[footerRows.length-1]:null;
if(!targetFoot||targetFoot.querySelector('.msg-usage-inline,.msg-duration-inline')) continue;
if(!targetFoot||targetFoot.querySelector('.msg-usage-inline,.msg-duration-inline,.msg-gateway-inline,.gateway-failover-inline,.msg-model-warning-inline')) continue;
const fragments=[];
if(modelWarningText){
const warning=document.createElement('span');
warning.className='msg-model-warning-inline';
warning.textContent=modelWarningText;
fragments.push(warning);
}
if(failoverText){
const failover=document.createElement('span');
failover.className='gateway-failover-inline';
failover.textContent=failoverText;
fragments.push(failover);
}
if(gatewayText){
const gateway=document.createElement('span');
gateway.className='msg-gateway-inline';
gateway.textContent=gatewayText;
fragments.push(gateway);
}
if(durationText){
const duration=document.createElement('span');
duration.className='msg-duration-inline';
+105
View File
@@ -0,0 +1,105 @@
"""Regression coverage for #732 LLM Gateway routing metadata display."""
from pathlib import Path
from api.models import Session
from api.streaming import _normalize_gateway_routing_metadata
REPO = Path(__file__).resolve().parents[1]
STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8")
MESSAGES_JS = (REPO / "static" / "messages.js").read_text(encoding="utf-8")
UI_JS = (REPO / "static" / "ui.js").read_text(encoding="utf-8")
SESSIONS_JS = (REPO / "static" / "sessions.js").read_text(encoding="utf-8")
STYLE_CSS = (REPO / "static" / "style.css").read_text(encoding="utf-8")
def test_gateway_routing_metadata_is_safely_normalized_from_response_metadata():
metadata = {
"used_provider": "Alibaba Cloud",
"used_model": "deepseek-v3.2",
"requested_provider": "CanopyWave",
"requested_model": "deepseek-v3.2",
"api_key": "fake_credential",
"routing": [
{
"provider": "CanopyWave",
"status": "failed",
"reason": "timeout",
"score": 0.12,
"api_key": "fake_credential",
},
{"provider": "Alibaba Cloud", "status": "selected", "score": 0.91},
],
}
normalized = _normalize_gateway_routing_metadata(metadata, requested_model="deepseek-v3.2", requested_provider="CanopyWave")
assert normalized == {
"used_provider": "Alibaba Cloud",
"used_model": "deepseek-v3.2",
"requested_provider": "CanopyWave",
"requested_model": "deepseek-v3.2",
"provider_changed": True,
"model_changed": False,
"has_failover": True,
"routing": [
{"provider": "CanopyWave", "status": "failed", "reason": "timeout", "score": 0.12},
{"provider": "Alibaba Cloud", "status": "selected", "score": 0.91},
],
}
assert "fake_credential" not in repr(normalized)
def test_gateway_routing_metadata_absent_returns_none_without_placeholder_noise():
assert _normalize_gateway_routing_metadata({}, requested_model="gpt-5.5", requested_provider="openai-codex") is None
assert _normalize_gateway_routing_metadata(None, requested_model="gpt-5.5", requested_provider="openai-codex") is None
def test_session_persists_latest_gateway_routing_and_history_across_reload():
routing = _normalize_gateway_routing_metadata(
{
"used_provider": "provider-b",
"used_model": "model-b",
"requested_provider": "provider-a",
"requested_model": "model-a",
"routing": [
{"provider": "provider-a", "status": "failed"},
{"provider": "provider-b", "status": "selected"},
],
},
requested_model="model-a",
requested_provider="provider-a",
)
session = Session(session_id="732gateway", title="Gateway", gateway_routing=routing, gateway_routing_history=[routing])
session.messages = [{"role": "assistant", "content": "done", "_gatewayRouting": routing}]
session.save()
reloaded = Session.load("732gateway")
assert reloaded.gateway_routing == routing
assert reloaded.gateway_routing_history == [routing]
assert reloaded.messages[-1]["_gatewayRouting"] == routing
compact = reloaded.compact()
assert compact["gateway_routing"] == routing
assert compact["gateway_routing_history"] == [routing]
def test_streaming_captures_gateway_metadata_into_usage_payload_and_assistant_turn():
assert "_extract_gateway_routing_metadata" in STREAMING_PY
assert "usage['gateway_routing']" in STREAMING_PY
assert "_dm['_gatewayRouting']" in STREAMING_PY
assert "s.gateway_routing_history" in STREAMING_PY
def test_frontend_copies_and_formats_gateway_metadata_without_absent_noise():
assert "d.usage.gateway_routing" in MESSAGES_JS
assert "lastAsst._gatewayRouting" in MESSAGES_JS
assert "_formatGatewayModelLabel" in UI_JS
assert "_gatewayRoutingLabel" in UI_JS
assert "msg-gateway-inline" in UI_JS
assert "msg-model-warning-inline" in UI_JS
assert "gateway-failover-inline" in UI_JS
assert "if(!routing)return''" in UI_JS.replace(" ", "")
assert "_formatSessionModelWithGateway" in SESSIONS_JS
assert ".msg-model-warning-inline" in STYLE_CSS
+2 -1
View File
@@ -108,7 +108,8 @@ class TestSidebarDensitySessionRendering(unittest.TestCase):
def test_detailed_mode_uses_message_count_and_model(self):
self.assertIn("typeof s.message_count==='number'?s.message_count:0", SESSIONS_JS)
self.assertIn("if(s.model) metaBits.push(s.model);", SESSIONS_JS)
self.assertIn("const modelMeta=_formatSessionModelWithGateway(s);", SESSIONS_JS)
self.assertIn("if(modelMeta) metaBits.push(modelMeta);", SESSIONS_JS)
self.assertIn("t('session_meta_messages', msgCount)", SESSIONS_JS)
def test_profile_only_when_show_all_profiles(self):
@@ -38,7 +38,7 @@ def test_streaming_persists_context_fields_on_session_before_save():
# Save call follows shortly after
save_call = src.find("\n s.save()", block_start)
assert save_call != -1, "s.save() not found after the post-merge marker"
assert save_call - block_start < 3400, (
assert save_call - block_start < 4200, (
"s.save() should be close to the post-merge marker — block expanded unexpectedly. "
"If you've added a new pre-save mutation block here, bump this limit."
)