From 69d0fc73c9ce664ad8679ad7ac8a83fe864f7723 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 6 Jul 2026 22:11:33 -0600 Subject: [PATCH] fix: derive service tier support from agent fast-mode metadata --- api/config.py | 155 +++++++++++++++--- api/routes.py | 16 +- static/panels.js | 19 +-- static/ui.js | 10 ++ tests/test_auxiliary_models_settings.py | 13 +- tests/test_credential_pool_providers.py | 7 +- tests/test_issue4536_service_tier.py | 76 ++++++++- ...t_issue4756_session_visit_model_refresh.py | 2 +- tests/test_live_models_ttl_cache.py | 4 +- tests/test_model_cache_metadata.py | 2 +- 10 files changed, 250 insertions(+), 54 deletions(-) diff --git a/api/config.py b/api/config.py index 3d14e7bb..8fd72c23 100644 --- a/api/config.py +++ b/api/config.py @@ -2226,10 +2226,12 @@ def _apply_provider_prefix( result = [] for m in raw_models: mid = m["id"] + entry = dict(m) if mid.startswith("@") or "/" in mid: - result.append({"id": mid, "label": m["label"]}) + result.append(entry) else: - result.append({"id": f"@{provider_id}:{mid}", "label": m["label"]}) + entry["id"] = f"@{provider_id}:{mid}" + result.append(entry) return result @@ -3853,6 +3855,74 @@ def _is_openai_family_provider(provider: str | None) -> bool: return resolved in ("openai", "openai-api", "openai-codex") +def _normalize_openai_family_model_id(model_id: str | None) -> str: + """Return a model id in the form expected by hermes_cli fast-mode resolution.""" + model = str(model_id or "").strip() + if not model: + return "" + + if model.startswith("@") and ":" in model: + model = model.split(":", 1)[1].strip() + + if "://" in model: + return model + + if "/" in model: + provider_hint, candidate = model.split("/", 1) + if provider_hint.strip().lower() in {"openai", "openai-api", "openai-codex"}: + model = candidate.strip() + else: + return "" + + return model + + +def _legacy_openai_service_tier_overrides(model_id: str | None, provider: str | None) -> dict: + """Compatibility fallback for standalone WebUI installs without hermes_cli. + + Normal operation delegates to Hermes Agent model metadata. This fallback + preserves the old WebUI behavior when the agent package is unavailable, + while still failing closed for codex model slugs and foreign provider IDs. + """ + if not _is_openai_family_provider(provider): + return {} + resolved_provider = str(_resolve_provider_alias(str(provider or "").strip().lower())) + raw_model = str(model_id or "").strip() + if "://" not in raw_model and "/" in raw_model: + provider_hint = raw_model.split("/", 1)[0].strip().lower() + if provider_hint not in {"openai", "openai-api", "openai-codex"}: + return {} + normalized_model = _normalize_openai_family_model_id(model_id) + if not normalized_model: + if resolved_provider == "openai-codex": + return {} + return {"service_tier": "priority"} + lowered = normalized_model.lower() + if "codex" in lowered: + return {} + if lowered.startswith(("gpt-", "o1", "o3", "o4")): + return {"service_tier": "priority"} + return {} + + +def _resolve_main_model_fast_mode_overrides(model_id: str | None, provider: str | None = None) -> dict: + """Return provider request overrides for the main model fast-mode setting.""" + normalized_model = _normalize_openai_family_model_id(model_id) + if not normalized_model: + return _legacy_openai_service_tier_overrides(model_id, provider) + try: + from hermes_cli.models import resolve_fast_mode_overrides + except Exception: + logger.debug("Failed to import hermes_cli.models.resolve_fast_mode_overrides; using WebUI compatibility fallback.") + return _legacy_openai_service_tier_overrides(model_id, provider) + try: + resolved = resolve_fast_mode_overrides(normalized_model) + except Exception: + logger.debug("Failed to resolve fast-mode overrides for %r; using WebUI compatibility fallback.", normalized_model) + return _legacy_openai_service_tier_overrides(model_id, provider) + return resolved if isinstance(resolved, dict) else {} + + def _main_model_supports_service_tier( model_id: str | None, provider: str | None, @@ -3860,26 +3930,43 @@ def _main_model_supports_service_tier( """Return True when the current main-model selection can use OpenAI service tier.""" if not _is_openai_family_provider(provider): return False - resolved = str(provider or "").strip().lower() - if resolved == "openai-codex": - return False - raw_model = str(model_id or "").strip().lower() - if not raw_model: - return True - if "/" in raw_model: - prefix, bare_model = raw_model.split("/", 1) - if prefix != "openai": - return False - else: - bare_model = raw_model - if "codex" in bare_model: - return False return ( - _is_first_party_model("openai", bare_model) - or bare_model.startswith(("gpt-", "o1", "o3", "o4")) + str(_resolve_main_model_fast_mode_overrides(model_id, provider).get("service_tier", "")).strip().lower() + == "priority" ) +def _model_supports_fast_tier_for_provider(model_id: str | None, provider: str | None) -> bool: + """Return whether a provider/model entry supports WebUI's service-tier toggle.""" + return _main_model_supports_service_tier(model_id, provider) + + +def _annotate_fast_tier_model_groups(payload: dict | None) -> dict | None: + """Add service-tier capability metadata to OpenAI-family model groups.""" + if not isinstance(payload, dict): + return payload + groups = payload.get("groups") + if not isinstance(groups, list): + return payload + for group in groups: + if not isinstance(group, dict): + continue + provider_id = str(group.get("provider_id") or "").strip() + if not _is_openai_family_provider(provider_id): + continue + for bucket in ("models", "extra_models"): + models = group.get(bucket) + if not isinstance(models, list): + continue + for model in models: + if not isinstance(model, dict): + continue + model_id = str(model.get("id") or "").strip() + if model_id: + model["supports_fast_tier"] = _model_supports_fast_tier_for_provider(model_id, provider_id) + return payload + + def _public_main_service_tier(model_cfg: dict) -> str: """Return the saved main-model service tier only for OpenAI-family providers.""" if not isinstance(model_cfg, dict): @@ -4120,6 +4207,7 @@ def get_auxiliary_models() -> dict: "main": { "provider": main_provider, "model": main_model, + "supports_fast_tier": _main_model_supports_service_tier(main_model, main_provider), "service_tier": _public_main_service_tier(model_cfg), **_public_advanced_model_options(model_cfg), }, @@ -4447,13 +4535,13 @@ def _minimal_static_models_catalog() -> dict: "models": [{"id": default_model, "label": label}], } ) - return { + return _annotate_fast_tier_model_groups({ "active_provider": active_provider, "default_model": default_model, "configured_model_badges": {}, "groups": groups, "aliases": {}, - } + }) except Exception: logger.debug("minimal static models catalog build failed", exc_info=True) return { @@ -4812,7 +4900,7 @@ def _static_models_catalog_without_live_probes() -> dict: if not groups and default_model: return copy.deepcopy(_minimal_static_models_catalog()) - return { + return _annotate_fast_tier_model_groups({ "active_provider": active_provider, "default_model": default_model, "configured_model_badges": _configured_model_badges_from_static_catalog( @@ -4822,7 +4910,7 @@ def _static_models_catalog_without_live_probes() -> dict: ), "groups": groups, "aliases": model_aliases, - } + }) except Exception: logger.debug("static models catalog build failed", exc_info=True) return copy.deepcopy(_minimal_static_models_catalog()) @@ -5339,7 +5427,7 @@ def _load_models_cache_from_disk() -> dict | None: # disk save path does not persist `aliases`, so reconstruct them from # current config to keep the /api/models.aliases contract intact (a # disk-cache hit must not silently drop `/model ` resolution). - return { + return _annotate_fast_tier_model_groups({ "active_provider": cache["active_provider"], "default_model": cache["default_model"], "configured_model_badges": cache["configured_model_badges"], @@ -5349,7 +5437,7 @@ def _load_models_cache_from_disk() -> dict | None: if isinstance(cache.get("aliases"), dict) else _model_aliases_from_config() ), - } + }) except Exception: return None @@ -5407,13 +5495,13 @@ def _load_stale_models_cache_from_disk() -> dict | None: # duration of the over-budget stale fallback. Reconstruct from # current config, mirroring the live/static catalog alias build. aliases = _model_aliases_from_config() - return { + return _annotate_fast_tier_model_groups({ "active_provider": cache["active_provider"], "default_model": cache["default_model"], "configured_model_badges": cache["configured_model_badges"], "groups": cache["groups"], "aliases": aliases, - } + }) except Exception: return None @@ -5478,7 +5566,7 @@ def _get_fresh_memory_models_cache(now: float) -> dict | None: _available_models_cache_source_fingerprint = None return None if _is_valid_models_cache(_available_models_cache): - return copy.deepcopy(_available_models_cache) + return _annotate_fast_tier_model_groups(copy.deepcopy(_available_models_cache)) _available_models_cache = None _available_models_cache_ts = 0.0 _available_models_live_rebuild_ts = 0.0 @@ -6661,6 +6749,19 @@ def get_available_models(*, prefer_cache: bool = False, force_refresh: bool = Fa allow_empty: bool = False, ) -> None: picker_models = copy.deepcopy(raw_models or []) + if _is_openai_family_provider(provider_id): + for _model in picker_models: + if not isinstance(_model, dict): + continue + _model_id = str(_model.get("id") or "").strip() + if not _model_id: + continue + _model["supports_fast_tier"] = ( + str( + _resolve_main_model_fast_mode_overrides(_model_id, provider_id).get("service_tier", "") + ).strip().lower() + == "priority" + ) if apply_prefix: picker_models = _apply_provider_prefix(picker_models, provider_id, active_provider) visible_models, extra_models = _split_picker_overflow_models( diff --git a/api/routes.py b/api/routes.py index 9561a604..ec750f47 100644 --- a/api/routes.py +++ b/api/routes.py @@ -18552,7 +18552,11 @@ def _handle_live_models(handler, parsed): # Normalise to {id, label} — provider_model_ids() returns plain string IDs. # For ollama-cloud use the shared Ollama formatter (handles `:variant` suffix). # For all other providers use a simpler hyphen-split capitaliser. - from api.config import _format_ollama_label as _fmt_ollama + from api.config import ( + _format_ollama_label as _fmt_ollama, + _is_openai_family_provider as _is_fast_tier_provider, + _model_supports_fast_tier_for_provider, + ) def _make_label(mid): """Best-effort human label from a model ID string.""" @@ -18579,7 +18583,15 @@ def _handle_live_models(handler, parsed): label = label.replace(orig.title(), orig) return label - models_out = [{"id": mid, "label": _make_label(mid)} for mid in ids if mid] + annotate_fast_tier = _is_fast_tier_provider(provider) + models_out = [] + for mid in ids: + if not mid: + continue + entry = {"id": mid, "label": _make_label(mid)} + if annotate_fast_tier: + entry["supports_fast_tier"] = _model_supports_fast_tier_for_provider(mid, provider) + models_out.append(entry) return _finish({"provider": provider, "models": models_out, "count": len(models_out)}) diff --git a/static/panels.js b/static/panels.js index b0828882..43b062f0 100644 --- a/static/panels.js +++ b/static/panels.js @@ -9020,6 +9020,11 @@ async function loadSettingsPanel(){ for(const m of [...(g.models||[]),...(g.extra_models||[])]){ const opt=document.createElement('option'); opt.value=m.id;opt.textContent=m.label; + if(m && (m.supports_fast_tier === true || String(m.supports_fast_tier).toLowerCase()==='true')){ + opt.dataset.fast='1'; + }else if(m && (m.supports_fast_tier === false || String(m.supports_fast_tier).toLowerCase()==='false')){ + opt.dataset.fast='0'; + } og.appendChild(opt); } modelSel.appendChild(og); @@ -12045,17 +12050,9 @@ function _mainModelSupportsServiceTier(cfg){ const optgroup=selectedOpt&&selectedOpt.parentElement&&selectedOpt.parentElement.tagName==='OPTGROUP'?selectedOpt.parentElement:null; const provider=((selectedOpt&&selectedOpt.dataset&&selectedOpt.dataset.provider)||(optgroup&&optgroup.dataset&&optgroup.dataset.provider)||(cfg&&cfg.provider)||'').trim().toLowerCase(); if(provider!=='openai'&&provider!=='openai-api'&&provider!=='openai-codex') return false; - if(provider==='openai-codex') return false; - const rawModel=String((selectedOpt&&selectedOpt.value)||(selected&&selected.value)||(cfg&&cfg.model)||'').trim().toLowerCase(); - if(!rawModel) return true; - let bareModel=rawModel; - if(rawModel.includes('/')){ - const slash=rawModel.indexOf('/'); - if(rawModel.slice(0,slash)!=='openai') return false; - bareModel=rawModel.slice(slash+1); - } - if(bareModel.includes('codex')) return false; - return bareModel.startsWith('gpt-')||bareModel.startsWith('o1')||bareModel.startsWith('o3')||bareModel.startsWith('o4'); + const fastSupport = selectedOpt&&selectedOpt.dataset?selectedOpt.dataset.fast:''; + if(fastSupport) return fastSupport==='1'||fastSupport==='true'; + return cfg&&cfg.supports_fast_tier===true; } function _openAuxAdvancedOptions(taskKey,cfg){ diff --git a/static/ui.js b/static/ui.js index 313eba88..b804fc3e 100644 --- a/static/ui.js +++ b/static/ui.js @@ -3123,6 +3123,11 @@ async function populateModelDropdown(opts={}){ const opt=document.createElement('option'); opt.value=m.id; opt.textContent=m.label; + if(m && (m.supports_fast_tier === true || String(m.supports_fast_tier).toLowerCase()==='true')){ + opt.dataset.fast='1'; + }else if(m && (m.supports_fast_tier === false || String(m.supports_fast_tier).toLowerCase()==='false')){ + opt.dataset.fast='0'; + } og.appendChild(opt); _dynamicModelLabels[m.id]=m.label||m.id; } @@ -3221,6 +3226,11 @@ function _addLiveModelsToSelect(provider, models, sel){ opt.textContent=m.label||m.id; opt.title='Live model — fetched from provider'; opt.dataset.provider=provider; + if(m && (m.supports_fast_tier === true || String(m.supports_fast_tier).toLowerCase()==='true')){ + opt.dataset.fast='1'; + }else if(m && (m.supports_fast_tier === false || String(m.supports_fast_tier).toLowerCase()==='false')){ + opt.dataset.fast='0'; + } providerGroup.appendChild(opt); _dynamicModelLabels[mid]=m.label||m.id; added++; diff --git a/tests/test_auxiliary_models_settings.py b/tests/test_auxiliary_models_settings.py index 0e85a20b..0e3fff0e 100644 --- a/tests/test_auxiliary_models_settings.py +++ b/tests/test_auxiliary_models_settings.py @@ -122,16 +122,19 @@ class TestAuxiliaryModelsJS: assert helper_idx >= 0 helper_body = PANELS_JS[helper_idx:helper_idx + 1300] assert "_mainModelSupportsServiceTier" in PANELS_JS - assert "rawModel.includes('/')" in helper_body - assert "rawModel.slice(0,slash)!=='openai'" in helper_body - assert "bareModel.includes('codex')" in helper_body - assert "bareModel.startsWith('gpt-')" in helper_body - assert "bareModel.startsWith('o4')" in helper_body + assert "selectedOpt.dataset.fast" in helper_body + assert "return cfg&&cfg.supports_fast_tier===true" in helper_body + assert "return fastSupport==='1'||fastSupport==='true'" in helper_body + assert "provider!=='openai'&&provider!=='openai-api'&&provider!=='openai-codex')" in helper_body + assert "provider==='openai-codex') return false" not in helper_body assert "auxAdvancedServiceTier" in modal_body assert "isMain&&_mainModelSupportsServiceTier(cfg)" in modal_body assert "settings_main_advanced_service_tier" in modal_body assert "settings_main_advanced_service_tier_default" in modal_body assert "settings_main_advanced_service_tier_priority" in modal_body + assert "m.supports_fast_tier" in PANELS_JS + assert "opt.dataset.fast='1'" in PANELS_JS + assert "opt.dataset.fast='0'" in PANELS_JS def test_main_advanced_save_omits_unsupported_timing_keys(self): """Saving main-model options must not send blank timing keys that backend treats as clears.""" diff --git a/tests/test_credential_pool_providers.py b/tests/test_credential_pool_providers.py index 9b34c77b..7f111865 100644 --- a/tests/test_credential_pool_providers.py +++ b/tests/test_credential_pool_providers.py @@ -380,13 +380,14 @@ def test_apply_provider_prefix_no_double_prefix(): from api.config import _apply_provider_prefix raw = [ - {"id": "@copilot:gpt-5.4", "label": "already prefixed"}, - {"id": "openai/gpt-5.4", "label": "slash form"}, - {"id": "bare-model", "label": "bare"}, + {"id": "@copilot:gpt-5.4", "label": "already prefixed", "supports_fast_tier": True}, + {"id": "openai/gpt-5.4", "label": "slash form", "supports_fast_tier": True}, + {"id": "bare-model", "label": "bare", "supports_fast_tier": False}, ] result = _apply_provider_prefix(raw, "copilot", "openai-codex") ids = [m["id"] for m in result] assert ids == ["@copilot:gpt-5.4", "openai/gpt-5.4", "@copilot:bare-model"], ids + assert [m["supports_fast_tier"] for m in result] == [True, True, False] def test_apply_provider_prefix_active_provider_no_prefix(): diff --git a/tests/test_issue4536_service_tier.py b/tests/test_issue4536_service_tier.py index 9af29454..3908ed90 100644 --- a/tests/test_issue4536_service_tier.py +++ b/tests/test_issue4536_service_tier.py @@ -1,5 +1,7 @@ """Tests for issue #4536 — main-model service_tier persistence and guarded forwarding.""" +import sys + import pytest @@ -61,6 +63,7 @@ class TestIssue4536ServiceTier: payload = config.get_auxiliary_models() assert payload["main"]["service_tier"] == "priority" + assert payload["main"]["supports_fast_tier"] is True def test_main_service_tier_default_clears_persisted_value(self, monkeypatch, tmp_path): """Choosing Default/off should clear service_tier from persisted main-model options.""" @@ -92,6 +95,11 @@ class TestIssue4536ServiceTier: ) assert openai_payload == {"service_tier": "priority"} + codex_payload = config._main_model_request_overrides( + {"model": {"provider": "openai-codex", "default": "gpt-5.5", "service_tier": "priority"}}, + ) + assert codex_payload == {"service_tier": "priority"} + openrouter_payload = config._main_model_request_overrides( {"model": {"provider": "openrouter", "default": "meta-llama/llama-3.1", "service_tier": "priority"}}, ) @@ -107,10 +115,10 @@ class TestIssue4536ServiceTier: ) assert openai_alias_payload == {"service_tier": "priority"} - codex_payload = config._main_model_request_overrides( + codex_nonfast_payload = config._main_model_request_overrides( {"model": {"provider": "openai-codex", "default": "gpt-5.3-codex", "service_tier": "priority"}}, ) - assert codex_payload == {} + assert codex_nonfast_payload == {} codex_empty_model_payload = config._main_model_request_overrides( {"model": {"provider": "openai-codex", "service_tier": "priority"}}, @@ -122,6 +130,16 @@ class TestIssue4536ServiceTier: ) assert stale_openai_payload == {} + foreign_prefixed_openai_payload = config._main_model_request_overrides( + {"model": {"provider": "openai", "default": "openrouter/gpt-5.5", "service_tier": "priority"}}, + ) + assert foreign_prefixed_openai_payload == {} + + codex_unknown_model_payload = config._main_model_request_overrides( + {"model": {"provider": "openai-codex", "default": "gpt-7.0-codex", "service_tier": "priority"}}, + ) + assert codex_unknown_model_payload == {} + def test_auxiliary_payload_hides_service_tier_for_non_openai_main_models(self, monkeypatch, tmp_path): """Saved service_tier should not be re-exposed once the main model switches away from OpenAI.""" from api import config @@ -194,3 +212,57 @@ class TestIssue4536ServiceTier: assert result["ok"] is True text = config_path.read_text(encoding="utf-8") assert "service_tier" not in text + + def test_standalone_agent_import_failure_preserves_supported_service_tier(self, monkeypatch, tmp_path): + """If hermes_cli is unavailable, Settings saves should not delete supported service_tier state.""" + from api import config + + config_path = tmp_path / "config.yaml" + config_path.write_text( + "model:\n provider: openai-codex\n default: gpt-5.5\n service_tier: priority\n", + encoding="utf-8", + ) + monkeypatch.setattr(config, "_get_config_path", lambda: config_path) + monkeypatch.setattr(config, "invalidate_models_cache", lambda: None) + monkeypatch.setattr( + config, + "cfg", + {"model": {"provider": "openai-codex", "default": "gpt-5.5", "service_tier": "priority"}}, + ) + monkeypatch.setitem(sys.modules, "hermes_cli.models", None) + + assert config._main_model_request_overrides( + {"model": {"provider": "openai-codex", "default": "gpt-5.5", "service_tier": "priority"}}, + ) == {"service_tier": "priority"} + + result = config.set_hermes_default_model("gpt-5.5", provider="openai-codex") + + assert result["ok"] is True + text = config_path.read_text(encoding="utf-8") + assert "service_tier: priority" in text + + def test_standalone_agent_import_failure_still_blocks_codex_slug(self, monkeypatch): + """The compatibility fallback must not broadly enable Codex-specific model slugs.""" + from api import config + + monkeypatch.setitem(sys.modules, "hermes_cli.models", None) + + assert config._main_model_request_overrides( + {"model": {"provider": "openai-codex", "default": "gpt-5.3-codex", "service_tier": "priority"}}, + ) == {} + + def test_auxiliary_payload_marks_false_fast_tier_for_unsupported_main_models(self, monkeypatch, tmp_path): + """The frontend needs explicit false metadata, not an absent field that falls back stale.""" + from api import config + + config_path = tmp_path / "config.yaml" + config_path.write_text( + "model:\n provider: openai-codex\n default: gpt-5.3-codex\n service_tier: priority\n", + encoding="utf-8", + ) + monkeypatch.setattr(config, "_get_config_path", lambda: config_path) + + payload = config.get_auxiliary_models() + + assert payload["main"]["service_tier"] == "" + assert payload["main"]["supports_fast_tier"] is False diff --git a/tests/test_issue4756_session_visit_model_refresh.py b/tests/test_issue4756_session_visit_model_refresh.py index 8e9863f4..cd84b8fd 100644 --- a/tests/test_issue4756_session_visit_model_refresh.py +++ b/tests/test_issue4756_session_visit_model_refresh.py @@ -21,7 +21,7 @@ def _catalog(label: str) -> dict: { "provider": "OpenAI", "provider_id": "openai", - "models": [{"id": label, "label": label}], + "models": [{"id": label, "label": label, "supports_fast_tier": False}], } ], "aliases": {}, diff --git a/tests/test_live_models_ttl_cache.py b/tests/test_live_models_ttl_cache.py index adcddfcc..7e81a1a6 100644 --- a/tests/test_live_models_ttl_cache.py +++ b/tests/test_live_models_ttl_cache.py @@ -43,7 +43,7 @@ def test_live_models_cache_hits_within_ttl(monkeypatch): assert calls == ["openai"] assert first == second - assert first["models"] == [{"id": "openai/gpt-test", "label": "GPT Test"}] + assert first["models"] == [{"id": "openai/gpt-test", "label": "GPT Test", "supports_fast_tier": True}] def test_live_models_cache_expires(monkeypatch): @@ -111,7 +111,7 @@ def test_live_models_cache_returns_deep_copies(monkeypatch): second = routes._handle_live_models(object(), parsed) assert second["provider"] == "openai" - assert second["models"] == [{"id": "openai/gpt-test", "label": "GPT Test"}] + assert second["models"] == [{"id": "openai/gpt-test", "label": "GPT Test", "supports_fast_tier": True}] def test_live_models_endpoint_respects_picker_visibility_budget(monkeypatch): diff --git a/tests/test_model_cache_metadata.py b/tests/test_model_cache_metadata.py index 8881d51c..d624ce36 100644 --- a/tests/test_model_cache_metadata.py +++ b/tests/test_model_cache_metadata.py @@ -26,7 +26,7 @@ def test_save_models_cache_to_disk_preserves_response_metadata(tmp_path, monkeyp { "provider": "OpenAI", "provider_id": "openai", - "models": [{"id": "gpt-5.4-mini", "label": "GPT 5.4 Mini"}], + "models": [{"id": "gpt-5.4-mini", "label": "GPT 5.4 Mini", "supports_fast_tier": True}], } ], }