diff --git a/api/config.py b/api/config.py index 972ee61f..7ca16e46 100644 --- a/api/config.py +++ b/api/config.py @@ -1963,6 +1963,65 @@ def _get_label_for_model(model_id: str, existing_groups: list) -> str: ) +def _read_live_provider_model_ids(provider_id: str) -> list[str]: + """Return live model IDs from Hermes CLI for a provider, or [] on failure. + + WebUI's static ``_PROVIDER_MODELS`` table is only a fallback. The agent CLI + owns the provider registry and catalog-discovery logic, so ordinary picker + groups should ask ``hermes_cli.models.provider_model_ids()`` first (#1240). + Provider aliases are tried as a secondary lookup because WebUI keeps a few + display-facing IDs (for example ``google`` / ``x-ai``) that Hermes CLI may + normalize internally. + """ + pid = str(provider_id or "").strip() + if not pid: + return [] + try: + from hermes_cli.models import provider_model_ids as _provider_model_ids + except Exception: + return [] + + candidates = [pid] + try: + alias = _resolve_provider_alias(pid) + except Exception: + alias = "" + if alias and alias not in candidates: + candidates.append(alias) + + seen: set[str] = set() + for candidate in candidates: + try: + live_ids = _provider_model_ids(candidate) or [] + except Exception: + logger.debug("Failed to load %s models from hermes_cli", candidate) + continue + result: list[str] = [] + for mid in live_ids: + mid_s = str(mid or "").strip() + if mid_s and mid_s not in seen: + seen.add(mid_s) + result.append(mid_s) + if result: + return result + return [] + + +def _models_from_live_provider_ids(provider_id: str, live_ids: list[str]) -> list[dict]: + """Convert Hermes CLI model ids into WebUI picker model entries.""" + formatter = _format_ollama_label if provider_id in ("ollama", "ollama-cloud") else None + models: list[dict] = [] + seen: set[str] = set() + for mid in live_ids: + mid_s = str(mid or "").strip() + if not mid_s or mid_s in seen: + continue + seen.add(mid_s) + label = formatter(mid_s) if formatter else _get_label_for_model(mid_s, []) + models.append({"id": mid_s, "label": label}) + return models + + def _read_visible_codex_cache_model_ids() -> list[str]: """Return visible model slugs from Codex's local models_cache.json. @@ -2912,18 +2971,33 @@ def get_available_models() -> dict: group_entry["extra_models"] = extras groups.append(group_entry) elif pid in _PROVIDER_MODELS or pid in cfg.get("providers", {}): - raw_models = copy.deepcopy(_PROVIDER_MODELS.get(pid, [])) - detected_models = auto_detected_models_by_provider.get(pid, []) - if detected_models and not raw_models: - raw_models = copy.deepcopy(detected_models) - provider_cfg = cfg.get("providers", {}).get(pid, {}) + raw_models = [] + + # User-configured model allowlists are explicit local + # source-of-truth and should still beat auto-discovery. + # Otherwise, ask Hermes CLI first so WebUI tracks the same + # live catalog as the agent/CLI picker; WebUI's static + # _PROVIDER_MODELS table is now a fallback only (#1240). if isinstance(provider_cfg, dict) and "models" in provider_cfg: cfg_models = provider_cfg["models"] if isinstance(cfg_models, dict): raw_models = [{"id": k, "label": k} for k in cfg_models.keys()] elif isinstance(cfg_models, list): raw_models = [{"id": k, "label": k} for k in cfg_models] + + if not raw_models: + raw_models = _models_from_live_provider_ids( + pid, + _read_live_provider_model_ids(pid), + ) + + if not raw_models: + raw_models = copy.deepcopy(_PROVIDER_MODELS.get(pid, [])) + + detected_models = auto_detected_models_by_provider.get(pid, []) + if detected_models and not raw_models: + raw_models = copy.deepcopy(detected_models) models = _apply_provider_prefix(raw_models, pid, active_provider) groups.append( { diff --git a/tests/test_issue1240_generic_cli_catalog_sync.py b/tests/test_issue1240_generic_cli_catalog_sync.py new file mode 100644 index 00000000..d0ba5dd0 --- /dev/null +++ b/tests/test_issue1240_generic_cli_catalog_sync.py @@ -0,0 +1,162 @@ +"""Regression tests for #1240 — WebUI model catalog should delegate to Hermes CLI. + +The WebUI picker should not freeze ordinary providers to its static +``_PROVIDER_MODELS`` snapshot when Hermes CLI can return a fresher provider +catalog. Static lists remain a fallback only. +""" + +from __future__ import annotations + +import sys +import types + +import api.config as config + + +_PROVIDER_ENV_VARS = ( + "ANTHROPIC_API_KEY", + "OPENAI_API_KEY", + "OPENROUTER_API_KEY", + "GOOGLE_API_KEY", + "GEMINI_API_KEY", + "GLM_API_KEY", + "KIMI_API_KEY", + "DEEPSEEK_API_KEY", + "OPENCODE_ZEN_API_KEY", + "OPENCODE_GO_API_KEY", + "MINIMAX_API_KEY", + "MINIMAX_CN_API_KEY", + "XAI_API_KEY", + "MISTRAL_API_KEY", + "OLLAMA_CLOUD_API_KEY", + "OLLAMA_API_KEY", + "NOUS_API_KEY", + "NVIDIA_API_KEY", +) + + +def _scrub_provider_env(monkeypatch): + for name in _PROVIDER_ENV_VARS: + monkeypatch.delenv(name, raising=False) + + +def _install_fake_hermes_cli(monkeypatch, *, provider_id: str, live_ids, raise_on_lookup: bool = False): + """Install a hermes_cli stub that reports one authenticated provider.""" + fake_pkg = types.ModuleType("hermes_cli") + fake_pkg.__path__ = [] + + fake_models = types.ModuleType("hermes_cli.models") + fake_models.list_available_providers = lambda: [ + {"id": provider_id, "authenticated": True} + ] + + calls: list[str] = [] + + def provider_model_ids(pid): + calls.append(pid) + if raise_on_lookup: + raise RuntimeError("simulated provider_model_ids failure") + return list(live_ids) if pid == provider_id else [] + + fake_models.provider_model_ids = provider_model_ids + + fake_auth = types.ModuleType("hermes_cli.auth") + + def get_auth_status(pid): + if pid == provider_id: + return {"logged_in": True, "key_source": ""} + return {"logged_in": False, "key_source": ""} + + fake_auth.get_auth_status = get_auth_status + + monkeypatch.setitem(sys.modules, "hermes_cli", fake_pkg) + monkeypatch.setitem(sys.modules, "hermes_cli.models", fake_models) + monkeypatch.setitem(sys.modules, "hermes_cli.auth", fake_auth) + monkeypatch.delitem(sys.modules, "agent.credential_pool", raising=False) + monkeypatch.delitem(sys.modules, "agent", raising=False) + config.invalidate_models_cache() + return calls + + +def _configure(monkeypatch, tmp_path, *, provider: str, default: str = ""): + monkeypatch.setattr(config, "_get_config_path", lambda: tmp_path / "missing-config.yaml") + monkeypatch.setattr(config, "_models_cache_path", tmp_path / "models_cache.json") + monkeypatch.setattr( + config, + "cfg", + { + "model": {"provider": provider, "default": default}, + "providers": {}, + "fallback_providers": [], + }, + ) + monkeypatch.setattr(config, "_cfg_mtime", 0.0) + config.invalidate_models_cache() + + +def _provider_group(result: dict, provider_id: str) -> dict: + return next(g for g in result["groups"] if g.get("provider_id") == provider_id) + + +def _ids(group: dict) -> list[str]: + return [m.get("id") for m in group.get("models", [])] + + +def test_generic_provider_uses_hermes_cli_catalog_before_static_snapshot(monkeypatch, tmp_path): + """A normal provider should show fresh CLI-discovered models. + + ``claude-sonnet-5.0`` is intentionally absent from WebUI's static Anthropic + list. Before this fix the group came entirely from ``_PROVIDER_MODELS`` and + this model was invisible even though Hermes CLI knew about it. + """ + _scrub_provider_env(monkeypatch) + calls = _install_fake_hermes_cli( + monkeypatch, + provider_id="anthropic", + live_ids=["claude-opus-4.7", "claude-sonnet-5.0"], + ) + _configure(monkeypatch, tmp_path, provider="anthropic", default="claude-opus-4.7") + + result = config.get_available_models() + group = _provider_group(result, "anthropic") + + assert calls == ["anthropic"] + assert _ids(group) == ["claude-opus-4.7", "claude-sonnet-5.0"] + assert group["models"][1]["label"] == "Claude Sonnet 5.0" + + +def test_generic_provider_keeps_static_catalog_as_cli_failure_fallback(monkeypatch, tmp_path): + _scrub_provider_env(monkeypatch) + calls = _install_fake_hermes_cli( + monkeypatch, + provider_id="anthropic", + live_ids=[], + raise_on_lookup=True, + ) + _configure(monkeypatch, tmp_path, provider="anthropic", default="claude-opus-4.7") + + result = config.get_available_models() + group = _provider_group(result, "anthropic") + + assert calls == ["anthropic"] + assert "claude-opus-4.7" in _ids(group) + assert "claude-sonnet-4.6" in _ids(group) + + +def test_generic_provider_prefixes_live_ids_when_not_active_provider(monkeypatch, tmp_path): + """Provider-qualified live IDs must route through the selected provider.""" + _scrub_provider_env(monkeypatch) + calls = _install_fake_hermes_cli( + monkeypatch, + provider_id="anthropic", + live_ids=["claude-sonnet-5.0"], + ) + # Anthropic is authenticated via Hermes CLI, but OpenAI is the active + # default. The Anthropic row still has to be pickable/routable. + _configure(monkeypatch, tmp_path, provider="openai", default="gpt-5.5") + + result = config.get_available_models() + group = _provider_group(result, "anthropic") + + assert "anthropic" in calls + assert _ids(group) == ["@anthropic:claude-sonnet-5.0"] diff --git a/tests/test_issue644.py b/tests/test_issue644.py index 0b3c5967..5c6b4939 100644 --- a/tests/test_issue644.py +++ b/tests/test_issue644.py @@ -116,8 +116,9 @@ class TestConfigYamlModelsLoading: ) break - def test_provider_in_provider_models_but_no_cfg_override_unchanged(self): - """When no models key in cfg.providers, hardcoded _PROVIDER_MODELS still used.""" + def test_provider_in_provider_models_but_no_cfg_override_uses_static_fallback(self, monkeypatch): + """When Hermes CLI has no live catalog, _PROVIDER_MODELS remains fallback.""" + monkeypatch.setattr(_cfg, "_read_live_provider_model_ids", lambda _pid: []) cfg = { "model": {"provider": "anthropic"}, "providers": { @@ -132,10 +133,9 @@ class TestConfigYamlModelsLoading: for g in result["groups"]: if g["provider"] == "Anthropic": returned_ids = {m["id"] for m in g["models"]} - # Should still have the hardcoded models overlap = raw_ids & returned_ids assert overlap, ( - f"No _PROVIDER_MODELS models found in Anthropic group. " + f"No _PROVIDER_MODELS fallback models found in Anthropic group. " f"Expected subset of {raw_ids}, got {returned_ids}" ) break diff --git a/tests/test_model_resolver.py b/tests/test_model_resolver.py index 56c591dd..a492936f 100644 --- a/tests/test_model_resolver.py +++ b/tests/test_model_resolver.py @@ -234,18 +234,16 @@ def test_no_duplicate_when_default_model_is_prefixed(): _cfg.cfg.update(old_cfg) -def test_default_provider_models_not_prefixed(): +def test_default_provider_models_not_prefixed(monkeypatch): """The active provider's models remain bare (no @prefix added).""" import api.config as _cfg - raw_anthropic_ids = {m['id'] for m in _cfg._PROVIDER_MODELS.get('anthropic', [])} + monkeypatch.setattr(_cfg, "_read_live_provider_model_ids", lambda pid: ["claude-sonnet-5.0"] if pid == "anthropic" else []) result = _available_models_with_provider('anthropic') groups = {g['provider']: g['models'] for g in result['groups']} if 'Anthropic' in groups: returned_ids = {m['id'] for m in groups['Anthropic']} - for bare_id in raw_anthropic_ids: - assert bare_id in returned_ids, ( - f"_PROVIDER_MODELS entry '{bare_id}' is missing from the Anthropic group" - ) + assert "claude-sonnet-5.0" in returned_ids + assert not any(mid.startswith('@anthropic:') for mid in returned_ids), returned_ids # ── get_available_models(): phantom "Custom" group regression ───────────── @@ -437,8 +435,10 @@ def test_custom_endpoint_uses_model_config_api_key_for_model_discovery(monkeypat return False def _fake_urlopen(req, timeout=10): - captured['auth'] = req.get_header('Authorization') - captured['ua'] = req.get_header('User-agent') + url = getattr(req, 'full_url', '') + if 'example.test' in url: + captured['auth'] = req.get_header('Authorization') + captured['ua'] = req.get_header('User-agent') return _Resp() monkeypatch.setattr('urllib.request.urlopen', _fake_urlopen)