diff --git a/ROADMAP.md b/ROADMAP.md index 0d79f036..6b799225 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -238,6 +238,7 @@ Remaining gaps and forward work live in [Forward Work](#forward-work) below. - [x] Transcript-summary card at 10+ rounds - [x] Sidebar dedup keying on per-conversation identity (distinct chats from same platform stay separate) - [x] Gateway session sync skips dup / delete options for external sessions +- [x] LLM Gateway routing metadata display — assistant turns and session metadata show the served model/provider, failover path, and model-switch warnings when response metadata includes `used_provider`, `used_model`, or `routing` (#732) ### MCP integration - [x] MCP server management UI (System Settings → MCP Servers) @@ -284,7 +285,6 @@ Remaining gaps and forward work live in [Forward Work](#forward-work) below. - **Insights / monitoring suite** — agent heartbeat + alerts (#716), quota / rate-limit display (#706), data tabs (#722), monitor dashboard concepts (#766, #721) - **Native MCP server expose** — Hermes WebUI as an MCP server for direct agent integration (#733) -- **Provider failover status** — show active provider per model + LLM Gateway failover state (#732) - **Teams / agents management panel** — editable names, roles, assignments (#719) - **Web UI profile model alignment with Hermes runtime** — design parity (#749) - **DOM windowing / message virtualization** — for sessions with hundreds of messages (#734) diff --git a/api/models.py b/api/models.py index 6a939b4f..85e8bd82 100644 --- a/api/models.py +++ b/api/models.py @@ -331,6 +331,7 @@ class Session: compression_anchor_message_key=None, context_length=None, threshold_tokens=None, last_prompt_tokens=None, + gateway_routing=None, gateway_routing_history=None, parent_session_id: str=None, enabled_toolsets=None, **kwargs): @@ -361,6 +362,8 @@ class Session: self.context_length = context_length self.threshold_tokens = threshold_tokens self.last_prompt_tokens = last_prompt_tokens + self.gateway_routing = gateway_routing if isinstance(gateway_routing, dict) else None + self.gateway_routing_history = gateway_routing_history if isinstance(gateway_routing_history, list) else [] self.parent_session_id = parent_session_id self.is_cli_session = bool(kwargs.get('is_cli_session', False)) self.source_tag = kwargs.get('source_tag') @@ -405,6 +408,7 @@ class Session: 'pending_user_message', 'pending_attachments', 'pending_started_at', 'compression_anchor_visible_idx', 'compression_anchor_message_key', 'context_length', 'threshold_tokens', 'last_prompt_tokens', + 'gateway_routing', 'gateway_routing_history', 'parent_session_id', 'is_cli_session', 'source_tag', 'raw_source', 'session_source', 'source_label', 'enabled_toolsets', @@ -567,6 +571,8 @@ class Session: 'context_length': self.context_length, 'threshold_tokens': self.threshold_tokens, 'last_prompt_tokens': self.last_prompt_tokens, + 'gateway_routing': self.gateway_routing, + 'gateway_routing_history': self.gateway_routing_history, # Only emit 'parent_session_id' when set (the /branch fork link, #1342). # Sessions without a fork must not leak None — see test_session_lineage_metadata_api. **({'parent_session_id': self.parent_session_id} if self.parent_session_id else {}), diff --git a/api/streaming.py b/api/streaming.py index 97d79afc..679f03af 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -69,6 +69,152 @@ _API_SAFE_MSG_KEYS = {'role', 'content', 'tool_calls', 'tool_call_id', 'name', ' _NATIVE_IMAGE_MAX_BYTES = 20 * 1024 * 1024 +_GATEWAY_ROUTING_TOP_LEVEL_KEYS = { + 'used_provider', + 'used_model', + 'requested_provider', + 'requested_model', +} +_GATEWAY_ROUTING_CONTAINER_KEYS = ( + 'llm_gateway', + 'gateway', + 'metadata', + 'response_metadata', + 'routing_metadata', + 'usage', +) +_GATEWAY_ROUTING_ATTEMPT_KEYS = { + 'provider', 'model', 'status', 'reason', 'selection_reason', 'score', + 'latency_ms', 'error', 'timestamp', 'selected', 'attempt', 'attempt_index', +} + + +def _clean_gateway_routing_scalar(value): + if value is None: + return None + if isinstance(value, (str, int, float, bool)): + text = str(value).strip() + if not text: + return None + return value if isinstance(value, (int, float, bool)) else text[:240] + return None + + +def _find_gateway_metadata_payload(payload): + if not isinstance(payload, dict): + return None + if any(k in payload for k in _GATEWAY_ROUTING_TOP_LEVEL_KEYS) or isinstance(payload.get('routing'), list): + return payload + for key in _GATEWAY_ROUTING_CONTAINER_KEYS: + nested = payload.get(key) + found = _find_gateway_metadata_payload(nested) + if found: + return found + return None + + +def _normalize_gateway_routing_metadata(payload, requested_model=None, requested_provider=None): + """Return safe LLM Gateway routing metadata, or None when absent. + + LLM Gateway response metadata can contain provider/model routing details, + but WebUI must only persist display-safe scalars and a bounded routing list. + Secrets or provider-specific request objects are deliberately ignored. + """ + src = _find_gateway_metadata_payload(payload) + if not src: + return None + + normalized = {} + for key in _GATEWAY_ROUTING_TOP_LEVEL_KEYS: + value = _clean_gateway_routing_scalar(src.get(key)) + if value is not None: + normalized[key] = value + + if 'requested_model' not in normalized: + fallback_model = _clean_gateway_routing_scalar(requested_model) + if fallback_model is not None: + normalized['requested_model'] = fallback_model + if 'requested_provider' not in normalized: + fallback_provider = _clean_gateway_routing_scalar(requested_provider) + if fallback_provider is not None: + normalized['requested_provider'] = fallback_provider + + routing = [] + raw_routing = src.get('routing') + if isinstance(raw_routing, list): + for attempt in raw_routing[:12]: + if not isinstance(attempt, dict): + continue + clean_attempt = {} + for key in _GATEWAY_ROUTING_ATTEMPT_KEYS: + value = _clean_gateway_routing_scalar(attempt.get(key)) + if value is not None: + clean_attempt[key] = value + if clean_attempt: + routing.append(clean_attempt) + if routing: + normalized['routing'] = routing + + used_provider = str(normalized.get('used_provider') or '').strip().lower() + requested_provider_norm = str(normalized.get('requested_provider') or '').strip().lower() + used_model = str(normalized.get('used_model') or '').strip().lower() + requested_model_norm = str(normalized.get('requested_model') or '').strip().lower() + provider_changed = bool(used_provider and requested_provider_norm and used_provider != requested_provider_norm) + model_changed = bool(used_model and requested_model_norm and used_model != requested_model_norm) + attempted_providers = [ + str(a.get('provider') or '').strip().lower() + for a in routing + if a.get('provider') + ] + distinct_attempted_providers = {p for p in attempted_providers if p} + failed_before_selection = any( + str(a.get('status') or '').strip().lower() in {'failed', 'error', 'timeout', 'rejected'} + for a in routing + ) + has_failover = bool(provider_changed or len(distinct_attempted_providers) > 1 or failed_before_selection) + + if not ( + normalized.get('used_provider') or normalized.get('used_model') or routing or provider_changed or model_changed + ): + return None + normalized['provider_changed'] = provider_changed + normalized['model_changed'] = model_changed + normalized['has_failover'] = has_failover + return normalized + + +def _extract_gateway_routing_metadata(agent, result, requested_model=None, requested_provider=None): + candidates = [] + if isinstance(result, dict): + candidates.extend([ + result.get('llm_gateway'), + result.get('gateway'), + result.get('metadata'), + result.get('response_metadata'), + result.get('routing_metadata'), + result.get('usage'), + result, + ]) + for attr in ( + 'llm_gateway_metadata', + 'gateway_metadata', + 'last_response_metadata', + 'response_metadata', + 'routing_metadata', + 'last_usage', + ): + if agent is not None: + candidates.append(getattr(agent, attr, None)) + for candidate in candidates: + normalized = _normalize_gateway_routing_metadata( + candidate, + requested_model=requested_model, + requested_provider=requested_provider, + ) + if normalized: + return normalized + return None + def _build_agent_thread_env(profile_runtime_env: dict | None, workspace: str, session_id: str, profile_home: str) -> dict: """Build thread-local agent env with per-run values overriding profile defaults. @@ -2497,12 +2643,25 @@ def _run_agent_streaming( _turn_tps = None if output_tokens and _turn_duration_seconds > 0: _turn_tps = round(float(output_tokens) / _turn_duration_seconds, 1) + _gateway_routing = _extract_gateway_routing_metadata( + agent, + result, + requested_model=resolved_model or model, + requested_provider=resolved_provider, + ) + if _gateway_routing: + s.gateway_routing = _gateway_routing + _history = list(getattr(s, 'gateway_routing_history', None) or []) + _history.append(_gateway_routing) + s.gateway_routing_history = _history[-50:] if s.messages: for _dm in reversed(s.messages): if isinstance(_dm, dict) and _dm.get('role') == 'assistant': _dm['_turnDuration'] = round(_turn_duration_seconds, 3) if _turn_tps is not None: _dm['_turnTps'] = _turn_tps + if _gateway_routing: + _dm['_gatewayRouting'] = _gateway_routing break # Persist context window data on the session so the context-ring # indicator survives a page reload (#1318). Must run BEFORE @@ -2559,6 +2718,8 @@ def _run_agent_streaming( } if _turn_tps is not None: usage['tps'] = _turn_tps + if _gateway_routing: + usage['gateway_routing'] = _gateway_routing # Include context window data from the agent's compressor for the UI indicator. # The session-level persistence happens above (before s.save()) so the values # survive a page reload; this block only populates the live SSE usage payload. diff --git a/docs/pr-media/732/gateway-routing-before.png b/docs/pr-media/732/gateway-routing-before.png new file mode 100644 index 00000000..a431e972 Binary files /dev/null and b/docs/pr-media/732/gateway-routing-before.png differ diff --git a/docs/pr-media/732/gateway-routing-metadata.png b/docs/pr-media/732/gateway-routing-metadata.png new file mode 100644 index 00000000..155cf489 Binary files /dev/null and b/docs/pr-media/732/gateway-routing-metadata.png differ diff --git a/static/messages.js b/static/messages.js index 7360e73b..43ebda9d 100644 --- a/static/messages.js +++ b/static/messages.js @@ -910,6 +910,12 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ if(typeof d.usage.tps==='number'&&d.usage.tps>0){ lastAsst._turnTps=d.usage.tps; } + if(d.usage.gateway_routing){ + lastAsst._gatewayRouting=d.usage.gateway_routing; + if(S.session)S.session.gateway_routing=d.usage.gateway_routing; + if(S.session&&Array.isArray(S.session.gateway_routing_history))S.session.gateway_routing_history.push(d.usage.gateway_routing); + else if(S.session)S.session.gateway_routing_history=[d.usage.gateway_routing]; + } } } if(d.session.tool_calls&&d.session.tool_calls.length){ diff --git a/static/sessions.js b/static/sessions.js index 1fdf6ff2..713098c1 100644 --- a/static/sessions.js +++ b/static/sessions.js @@ -25,6 +25,15 @@ let _sessionObservedStreaming = null; const _sessionStreamingById = new Map(); const _sessionListSnapshotById = new Map(); +function _formatSessionModelWithGateway(s){ + if(!s||!s.model)return''; + const routing=(typeof _latestGatewayRoutingForSession==='function')?_latestGatewayRoutingForSession(s):(s.gateway_routing||null); + if(typeof _formatGatewayModelLabel==='function'){ + return _formatGatewayModelLabel(s.model,s.model,routing)||s.model; + } + return s.model; +} + function _getSessionViewedCounts() { if (_sessionViewedCounts !== null) return _sessionViewedCounts; try { @@ -2338,7 +2347,8 @@ function renderSessionListFromCache(){ : `${msgCount} msg${msgCount===1?'':'s'}`; metaBits.push(msgLabel); if(childCount>0) metaBits.push(t('session_meta_children', childCount)); - if(s.model) metaBits.push(s.model); + const modelMeta=_formatSessionModelWithGateway(s); + if(modelMeta) metaBits.push(modelMeta); const sourceLabel=_getChannelLabel(s); if(s.is_cli_session&&sourceLabel) metaBits.push(sourceLabel); if(readOnly) metaBits.push('read-only'); diff --git a/static/style.css b/static/style.css index 3022b147..4fc9caf8 100644 --- a/static/style.css +++ b/static/style.css @@ -2794,13 +2794,25 @@ main.main.showing-logs > #mainLogs{display:flex;} gap: 8px; } .msg-usage-inline, -.msg-duration-inline { +.msg-duration-inline, +.msg-gateway-inline, +.gateway-failover-inline, +.msg-model-warning-inline { font-size: 11px; color: var(--muted); opacity: .7; flex: 0 0 auto; font-variant-numeric: tabular-nums; } +.gateway-failover-inline { + color: var(--accent); + opacity: .9; +} +.msg-model-warning-inline { + color: var(--error); + opacity: .95; + font-weight: 600; +} .msg-foot-with-usage .msg-time, .msg-foot-with-usage .msg-actions { opacity: 0; diff --git a/static/ui.js b/static/ui.js index 3d7a7fcd..199e5eba 100644 --- a/static/ui.js +++ b/static/ui.js @@ -739,9 +739,11 @@ function syncModelChip(){ } const opt=_selectedModelOption(); const text=opt?opt.textContent:getModelLabel(sel.value||''); - label.textContent=text; - if(mobileLabel) mobileLabel.textContent=text; - chip.title=sel.value||'Conversation model'; + const gatewayRouting=_latestGatewayRoutingForSession(S.session); + const displayText=_formatGatewayModelLabel(sel.value||'',text,gatewayRouting)||text; + label.textContent=displayText; + if(mobileLabel) mobileLabel.textContent=displayText; + chip.title=gatewayRouting?`${sel.value||'Conversation model'} ${_gatewayRoutingLabel(gatewayRouting)}`:(sel.value||'Conversation model'); chip.classList.toggle('active',!!(dd&&dd.classList.contains('open'))); if(mobileAction) mobileAction.classList.toggle('active',!!(dd&&dd.classList.contains('open'))); } @@ -1647,6 +1649,47 @@ function getModelLabel(modelId){ return _last || 'Unknown'; } +function _gatewayProviderName(provider){ + const text=String(provider||'').trim(); + if(!text)return''; + return text.replace(/^custom:/,'').replace(/[-_]/g,' ').replace(/\b\w/g,c=>c.toUpperCase()); +} +function _gatewayRoutingLabel(routing){ + if(!routing)return''; + const provider=_gatewayProviderName(routing.used_provider||routing.provider); + return provider?`via ${provider}`:''; +} +function _formatGatewayModelLabel(modelId,labelText,routing){ + if(!routing)return''; + const usedModel=String(routing.used_model||'').trim(); + const base=usedModel?getModelLabel(usedModel):(labelText||getModelLabel(modelId)); + const via=_gatewayRoutingLabel(routing); + return via?`${base} ${via}`:base; +} +function _gatewayRoutingFailoverText(routing){ + if(!routing||!routing.has_failover)return''; + const attempts=Array.isArray(routing.routing)?routing.routing:[]; + const providers=attempts.map(a=>_gatewayProviderName(a&&a.provider)).filter(Boolean); + const unique=[];providers.forEach(p=>{if(!unique.includes(p))unique.push(p);}); + if(unique.length>=2)return`Failover: ${unique[0]} → ${unique[unique.length-1]}`; + const from=_gatewayProviderName(routing.requested_provider); + const to=_gatewayProviderName(routing.used_provider); + if(from&&to&&from!==to)return`Failover: ${from} → ${to}`; + return'Gateway failover detected'; +} +function _gatewayModelWarningText(routing){ + if(!routing||!routing.model_changed)return''; + const requested=getModelLabel(routing.requested_model||'requested model'); + const used=getModelLabel(routing.used_model||'served model'); + return`Model switched: ${requested} → ${used}`; +} +function _latestGatewayRoutingForSession(session){ + if(!session)return null; + if(session.gateway_routing)return session.gateway_routing; + const history=Array.isArray(session.gateway_routing_history)?session.gateway_routing_history:[]; + return history.length?history[history.length-1]:null; +} + function _stripXmlToolCallsDisplay(s){ // Strip ... blocks emitted by DeepSeek and // similar models in their raw response text. These are processed separately @@ -4345,19 +4388,41 @@ function renderMessages(){ for(const mi of renderedAssistantIdxs){ const msg=S.messages[mi]||{}; if(msg.role!=='assistant') continue; + const routing=msg._gatewayRouting||null; + const gatewayText=_formatGatewayModelLabel(S.session&&S.session.model||'', '', routing); + const failoverText=_gatewayRoutingFailoverText(routing); + const modelWarningText=_gatewayModelWarningText(routing); const hasTurnUsage=!!msg._turnUsage; const compactActivityForMessage=isSimplifiedToolCalling()&&( assistantThinking.has(mi)|| (S.toolCalls||[]).some(tc=>tc&&(tc.assistant_msg_idx!==undefined?tc.assistant_msg_idx:-1)===mi) ); const durationText=compactActivityForMessage?'':_formatTurnDuration(msg._turnDuration); - if(!hasTurnUsage&&!durationText) continue; + if(!hasTurnUsage&&!durationText&&!gatewayText&&!failoverText&&!modelWarningText) continue; const seg=assistantSegments.get(mi); const row=seg?seg.closest('.assistant-turn'):null; const footerRows=row?row.querySelectorAll('.msg-foot'):[]; const targetFoot=footerRows.length?footerRows[footerRows.length-1]:null; - if(!targetFoot||targetFoot.querySelector('.msg-usage-inline,.msg-duration-inline')) continue; + if(!targetFoot||targetFoot.querySelector('.msg-usage-inline,.msg-duration-inline,.msg-gateway-inline,.gateway-failover-inline,.msg-model-warning-inline')) continue; const fragments=[]; + if(modelWarningText){ + const warning=document.createElement('span'); + warning.className='msg-model-warning-inline'; + warning.textContent=modelWarningText; + fragments.push(warning); + } + if(failoverText){ + const failover=document.createElement('span'); + failover.className='gateway-failover-inline'; + failover.textContent=failoverText; + fragments.push(failover); + } + if(gatewayText){ + const gateway=document.createElement('span'); + gateway.className='msg-gateway-inline'; + gateway.textContent=gatewayText; + fragments.push(gateway); + } if(durationText){ const duration=document.createElement('span'); duration.className='msg-duration-inline'; diff --git a/tests/test_732_gateway_routing_metadata.py b/tests/test_732_gateway_routing_metadata.py new file mode 100644 index 00000000..3604f27d --- /dev/null +++ b/tests/test_732_gateway_routing_metadata.py @@ -0,0 +1,105 @@ +"""Regression coverage for #732 LLM Gateway routing metadata display.""" + +from pathlib import Path + +from api.models import Session +from api.streaming import _normalize_gateway_routing_metadata + + +REPO = Path(__file__).resolve().parents[1] +STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8") +MESSAGES_JS = (REPO / "static" / "messages.js").read_text(encoding="utf-8") +UI_JS = (REPO / "static" / "ui.js").read_text(encoding="utf-8") +SESSIONS_JS = (REPO / "static" / "sessions.js").read_text(encoding="utf-8") +STYLE_CSS = (REPO / "static" / "style.css").read_text(encoding="utf-8") + + +def test_gateway_routing_metadata_is_safely_normalized_from_response_metadata(): + metadata = { + "used_provider": "Alibaba Cloud", + "used_model": "deepseek-v3.2", + "requested_provider": "CanopyWave", + "requested_model": "deepseek-v3.2", + "api_key": "fake_credential", + "routing": [ + { + "provider": "CanopyWave", + "status": "failed", + "reason": "timeout", + "score": 0.12, + "api_key": "fake_credential", + }, + {"provider": "Alibaba Cloud", "status": "selected", "score": 0.91}, + ], + } + + normalized = _normalize_gateway_routing_metadata(metadata, requested_model="deepseek-v3.2", requested_provider="CanopyWave") + + assert normalized == { + "used_provider": "Alibaba Cloud", + "used_model": "deepseek-v3.2", + "requested_provider": "CanopyWave", + "requested_model": "deepseek-v3.2", + "provider_changed": True, + "model_changed": False, + "has_failover": True, + "routing": [ + {"provider": "CanopyWave", "status": "failed", "reason": "timeout", "score": 0.12}, + {"provider": "Alibaba Cloud", "status": "selected", "score": 0.91}, + ], + } + assert "fake_credential" not in repr(normalized) + + +def test_gateway_routing_metadata_absent_returns_none_without_placeholder_noise(): + assert _normalize_gateway_routing_metadata({}, requested_model="gpt-5.5", requested_provider="openai-codex") is None + assert _normalize_gateway_routing_metadata(None, requested_model="gpt-5.5", requested_provider="openai-codex") is None + + +def test_session_persists_latest_gateway_routing_and_history_across_reload(): + routing = _normalize_gateway_routing_metadata( + { + "used_provider": "provider-b", + "used_model": "model-b", + "requested_provider": "provider-a", + "requested_model": "model-a", + "routing": [ + {"provider": "provider-a", "status": "failed"}, + {"provider": "provider-b", "status": "selected"}, + ], + }, + requested_model="model-a", + requested_provider="provider-a", + ) + session = Session(session_id="732gateway", title="Gateway", gateway_routing=routing, gateway_routing_history=[routing]) + session.messages = [{"role": "assistant", "content": "done", "_gatewayRouting": routing}] + session.save() + + reloaded = Session.load("732gateway") + + assert reloaded.gateway_routing == routing + assert reloaded.gateway_routing_history == [routing] + assert reloaded.messages[-1]["_gatewayRouting"] == routing + compact = reloaded.compact() + assert compact["gateway_routing"] == routing + assert compact["gateway_routing_history"] == [routing] + + +def test_streaming_captures_gateway_metadata_into_usage_payload_and_assistant_turn(): + assert "_extract_gateway_routing_metadata" in STREAMING_PY + assert "usage['gateway_routing']" in STREAMING_PY + assert "_dm['_gatewayRouting']" in STREAMING_PY + assert "s.gateway_routing_history" in STREAMING_PY + + +def test_frontend_copies_and_formats_gateway_metadata_without_absent_noise(): + assert "d.usage.gateway_routing" in MESSAGES_JS + assert "lastAsst._gatewayRouting" in MESSAGES_JS + assert "_formatGatewayModelLabel" in UI_JS + assert "_gatewayRoutingLabel" in UI_JS + assert "msg-gateway-inline" in UI_JS + assert "msg-model-warning-inline" in UI_JS + assert "gateway-failover-inline" in UI_JS + assert "if(!routing)return''" in UI_JS.replace(" ", "") + assert "_formatSessionModelWithGateway" in SESSIONS_JS + assert ".msg-model-warning-inline" in STYLE_CSS diff --git a/tests/test_issue673.py b/tests/test_issue673.py index c3b815bd..b60cfa46 100644 --- a/tests/test_issue673.py +++ b/tests/test_issue673.py @@ -108,7 +108,8 @@ class TestSidebarDensitySessionRendering(unittest.TestCase): def test_detailed_mode_uses_message_count_and_model(self): self.assertIn("typeof s.message_count==='number'?s.message_count:0", SESSIONS_JS) - self.assertIn("if(s.model) metaBits.push(s.model);", SESSIONS_JS) + self.assertIn("const modelMeta=_formatSessionModelWithGateway(s);", SESSIONS_JS) + self.assertIn("if(modelMeta) metaBits.push(modelMeta);", SESSIONS_JS) self.assertIn("t('session_meta_messages', msgCount)", SESSIONS_JS) def test_profile_only_when_show_all_profiles(self): diff --git a/tests/test_pr1341_context_window_persistence.py b/tests/test_pr1341_context_window_persistence.py index 42687c75..42268207 100644 --- a/tests/test_pr1341_context_window_persistence.py +++ b/tests/test_pr1341_context_window_persistence.py @@ -38,7 +38,7 @@ def test_streaming_persists_context_fields_on_session_before_save(): # Save call follows shortly after save_call = src.find("\n s.save()", block_start) assert save_call != -1, "s.save() not found after the post-merge marker" - assert save_call - block_start < 3400, ( + assert save_call - block_start < 4200, ( "s.save() should be close to the post-merge marker — block expanded unexpectedly. " "If you've added a new pre-save mutation block here, bump this limit." )