"""Regression for PR #1970 LM Studio provider × cfg.model.base_url shape. PR #1970 added `_get_provider_base_url()` + a dedicated lmstudio branch in `get_available_models()` for fetching live loaded models via the OpenAI-compatible /v1/models endpoint. The initial implementation only looked at `cfg["providers"]["lmstudio"]["base_url"]`, missing the historical shape where users put `base_url` under `cfg["model"]` (when `cfg["model"]["provider"] == "lmstudio"`). That shape is what `tests/test_issue1527_lmstudio_base_url_classification.py` covers and what real users have in their config.yaml — 3 pre-existing tests started failing on stage-337 because of this gap. This regression test pins the helper's two-location lookup so a future change can't accidentally drop the model.base_url fallback again. """ from __future__ import annotations import sys import types import api.config as config import pytest class _RestoreCfg: """Context manager: snapshot cfg, restore on exit (test isolation).""" def __enter__(self): import copy self._snapshot = copy.deepcopy(config.cfg) self._cfg_mtime = config._cfg_mtime self._cfg_path = getattr(config, "_cfg_path", None) return self def __exit__(self, *exc): config.cfg.clear() config.cfg.update(self._snapshot) config._cfg_mtime = self._cfg_mtime config._cfg_path = self._cfg_path def _set_cfg(cfg_dict: dict): config.cfg.clear() config.cfg.update(cfg_dict) try: config._cfg_mtime = config.Path(config._get_config_path()).stat().st_mtime except Exception: config._cfg_mtime = 0.0 config._cfg_path = config._get_config_path() def _groups_by_provider_id(result: dict) -> dict: return {group["provider_id"]: group for group in result["groups"]} def _stub_hermes_cli(monkeypatch): fake_models = types.ModuleType("hermes_cli.models") fake_models.list_available_providers = lambda: [] fake_models.provider_model_ids = lambda _pid: [] fake_auth = types.ModuleType("hermes_cli.auth") fake_auth.get_auth_status = lambda _pid: {"key_source": "none"} monkeypatch.setitem(sys.modules, "hermes_cli.models", fake_models) monkeypatch.setitem(sys.modules, "hermes_cli.auth", fake_auth) monkeypatch.setattr( config, "_get_auth_store_path", lambda: config.Path("/tmp/does-not-exist-auth.json"), ) @pytest.fixture(autouse=True) def _isolate_models_cache(tmp_path, monkeypatch): monkeypatch.setattr(config, "_models_cache_path", tmp_path / "models_cache.json") config.invalidate_models_cache() yield config.invalidate_models_cache() def test_get_provider_base_url_finds_explicit_providers_entry(): """When providers..base_url is set, return that value.""" with _RestoreCfg(): config.cfg.clear() config.cfg.update({ "providers": { "lmstudio": {"base_url": "http://10.0.0.5:1234/v1", "api_key": "x"}, }, }) assert config._get_provider_base_url("lmstudio") == "http://10.0.0.5:1234/v1" def test_get_provider_base_url_strips_trailing_slash(): with _RestoreCfg(): config.cfg.clear() config.cfg.update({ "providers": { "lmstudio": {"base_url": "http://10.0.0.5:1234/v1/", "api_key": "x"}, }, }) assert config._get_provider_base_url("lmstudio") == "http://10.0.0.5:1234/v1" def test_get_provider_base_url_falls_back_to_model_base_url(): """When providers..base_url is unset but cfg.model.base_url is set AND cfg.model.provider matches, the helper returns model.base_url.""" with _RestoreCfg(): config.cfg.clear() config.cfg.update({ "model": { "provider": "lmstudio", "base_url": "http://192.168.1.22:1234/v1", "default": "qwen3.6-35b-a3b@q6_k", }, "providers": { "lmstudio": {"api_key": "local-key"}, # no base_url here }, }) # Was returning None before the fix — the regression that broke # test_issue1527_lmstudio_base_url_classification. assert config._get_provider_base_url("lmstudio") == "http://192.168.1.22:1234/v1" def test_get_provider_base_url_returns_none_when_unconfigured(): """Unconfigured provider returns None (sentinel for 'use SDK default').""" with _RestoreCfg(): config.cfg.clear() config.cfg.update({"providers": {}}) assert config._get_provider_base_url("openai") is None assert config._get_provider_base_url("anthropic") is None assert config._get_provider_base_url("lmstudio") is None def test_get_provider_base_url_model_block_only_matches_active_provider(): """cfg.model.base_url must NOT leak to providers other than cfg.model.provider. If model.provider is anthropic but providers.openai exists without base_url, _get_provider_base_url("openai") must still return None — otherwise we'd silently rewrite the OpenAI SDK target to an Anthropic endpoint URL. """ with _RestoreCfg(): config.cfg.clear() config.cfg.update({ "model": { "provider": "anthropic", "base_url": "https://my-anthropic-proxy.example.com/v1", }, "providers": { "openai": {"api_key": "ok"}, # no base_url "anthropic": {"api_key": "ak"}, # no base_url }, }) # Active provider gets the model.base_url fallback. assert config._get_provider_base_url("anthropic") == "https://my-anthropic-proxy.example.com/v1" # OpenAI must NOT inherit it. assert config._get_provider_base_url("openai") is None def test_get_provider_base_url_explicit_wins_over_model_fallback(): """If both providers..base_url AND cfg.model.base_url are set with matching provider, the explicit providers entry wins.""" with _RestoreCfg(): config.cfg.clear() config.cfg.update({ "model": { "provider": "lmstudio", "base_url": "http://wrong:1234/v1", }, "providers": { "lmstudio": {"base_url": "http://correct:1234/v1", "api_key": "x"}, }, }) assert config._get_provider_base_url("lmstudio") == "http://correct:1234/v1" def test_get_provider_base_url_treats_list_providers_as_unconfigured(): """List-shaped providers must behave like a missing providers map.""" with _RestoreCfg(): for providers_cfg in ([], ["lmstudio"]): _set_cfg({ "model": { "provider": "lmstudio", "base_url": "http://192.168.1.22:1234/v1", }, "providers": providers_cfg, }) assert config._get_provider_base_url("lmstudio") == "http://192.168.1.22:1234/v1" assert config._get_provider_base_url("openai") is None def test_get_provider_base_url_treats_malformed_provider_entry_as_unconfigured(): """Truth-y non-dict provider entries must not reach ``.get("base_url")``.""" with _RestoreCfg(): _set_cfg({ "model": { "provider": "lmstudio", "base_url": "http://192.168.1.22:1234/v1", }, "providers": {"lmstudio": "enabled"}, }) assert config._get_provider_base_url("lmstudio") == "http://192.168.1.22:1234/v1" def test_lmstudio_fallback_works_when_hermes_cli_unavailable(tmp_path, monkeypatch): """The lmstudio branch must populate models from the urlopen fallback even when `from hermes_cli.models import provider_model_ids` raises ImportError. Pre-fix, the outer try/except in the lmstudio branch caught the ImportError and silently aborted the whole branch, never running the urlopen fallback — a CI-vs-local divergence where local environments with hermes_cli installed worked, and CI (clean editable install) failed with empty model groups. Caught in CI on stage-337; fix splits the hermes_cli try from the urlopen fallback so each runs independently. """ import json as _json import socket as _socket import urllib.request as _urlreq import api.config as config # Block hermes_cli import the way a CI runner without the package would. blocked_modules = [name for name in list(sys.modules) if name == "hermes_cli" or name.startswith("hermes_cli.")] for name in blocked_modules: monkeypatch.delitem(sys.modules, name, raising=False) class _Blocker: def find_module(self, name, path=None): if name == "hermes_cli" or name.startswith("hermes_cli."): return self return None def load_module(self, name): raise ImportError(f"hermes_cli blocked for test: {name}") blocker = _Blocker() sys.meta_path.insert(0, blocker) try: # Set up a config that points lmstudio at a fake base_url under cfg.model. cfgfile = tmp_path / "config.yaml" cfgfile.write_text( """ model: provider: lmstudio default: qwen3.6-35b-a3b@q6_k base_url: http://10.0.0.5:1234/v1 providers: lmstudio: api_key: local-key """, encoding="utf-8", ) monkeypatch.setattr(config, "_get_config_path", lambda: cfgfile) config.reload_config() config.invalidate_models_cache() class _ModelsResponse: def __enter__(self): return self def __exit__(self, *args): pass def read(self): return _json.dumps( {"data": [{"id": "qwen3.6-35b-a3b@q6_k"}, {"id": "another-model"}]} ).encode() monkeypatch.setattr(_urlreq, "urlopen", lambda *_a, **_kw: _ModelsResponse()) monkeypatch.setattr( _socket, "getaddrinfo", lambda *_a, **_kw: [ (_socket.AF_INET, _socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0)) ], ) result = config.get_available_models() groups = {g["provider_id"]: g for g in result["groups"]} # Fallback must succeed despite hermes_cli being unimportable. assert "lmstudio" in groups, ( f"lmstudio group missing when hermes_cli unavailable; groups={list(groups)}" ) model_ids = {m["id"] for m in groups["lmstudio"]["models"]} assert "qwen3.6-35b-a3b@q6_k" in model_ids assert "another-model" in model_ids finally: try: sys.meta_path.remove(blocker) except ValueError: pass def test_lmstudio_fallback_handles_malformed_providers_list(tmp_path, monkeypatch): """LM Studio fallback must ignore list-shaped providers instead of crashing.""" import json as _json import socket as _socket import urllib.request as _urlreq blocked_modules = [name for name in list(sys.modules) if name == "hermes_cli" or name.startswith("hermes_cli.")] for name in blocked_modules: monkeypatch.delitem(sys.modules, name, raising=False) class _Blocker: def find_module(self, name, path=None): if name == "hermes_cli" or name.startswith("hermes_cli."): return self return None def load_module(self, name): raise ImportError(f"hermes_cli blocked for test: {name}") blocker = _Blocker() sys.meta_path.insert(0, blocker) try: cfgfile = tmp_path / "config.yaml" cfgfile.write_text( """ model: provider: lmstudio default: qwen3.6-35b-a3b@q6_k base_url: http://10.0.0.5:1234/v1 providers: - lmstudio """, encoding="utf-8", ) monkeypatch.setattr(config, "_get_config_path", lambda: cfgfile) config.reload_config() config.invalidate_models_cache() class _ModelsResponse: def __enter__(self): return self def __exit__(self, *args): pass def read(self): return _json.dumps( {"data": [{"id": "qwen3.6-35b-a3b@q6_k"}, {"id": "another-model"}]} ).encode() monkeypatch.setattr(_urlreq, "urlopen", lambda *_a, **_kw: _ModelsResponse()) monkeypatch.setattr( _socket, "getaddrinfo", lambda *_a, **_kw: [ (_socket.AF_INET, _socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0)) ], ) result = config.get_available_models() groups = _groups_by_provider_id(result) assert "lmstudio" in groups, ( f"lmstudio group missing for malformed providers list; groups={list(groups)}" ) model_ids = {model["id"] for model in groups["lmstudio"]["models"]} assert "qwen3.6-35b-a3b@q6_k" in model_ids assert "another-model" in model_ids finally: try: sys.meta_path.remove(blocker) except ValueError: pass def test_provider_catalog_treats_malformed_providers_as_unconfigured(monkeypatch): """Catalog lookup must not crash when providers is a non-dict value.""" _stub_hermes_cli(monkeypatch) monkeypatch.setattr("socket.getaddrinfo", lambda *a, **k: []) with _RestoreCfg(): _set_cfg({ "model": { "default": "claude-sonnet-4-5", "provider": "anthropic", }, "providers": ["anthropic"], }) config.invalidate_models_cache() result = config.get_available_models() groups = _groups_by_provider_id(result) assert "anthropic" in groups, f"Expected anthropic group, got: {list(groups)}" assert groups["anthropic"]["models"], "anthropic group should still fall back to models" def test_provider_catalog_preserves_dict_shaped_raw_key_lookup(monkeypatch): """Dict-shaped providers must still resolve configured models through the raw key.""" _stub_hermes_cli(monkeypatch) monkeypatch.setattr("socket.getaddrinfo", lambda *a, **k: []) with _RestoreCfg(): _set_cfg({ "model": { "default": "my-model", "provider": "CLIPpoxy", }, "providers": { "CLIPpoxy": { "models": ["my-model", "another-model"], }, }, }) config.invalidate_models_cache() result = config.get_available_models() groups = _groups_by_provider_id(result) assert "clippoxy" in groups, f"Expected clippoxy group, got: {list(groups)}" model_ids = [] for model in groups["clippoxy"]["models"]: mid = model["id"] model_ids.append(mid.split(":", 1)[-1] if mid.startswith("@") and ":" in mid else mid) assert "my-model" in model_ids assert "another-model" in model_ids def test_provider_catalog_rejects_non_active_models_only_custom_provider(monkeypatch): """A models-only custom provider config must NOT render when it is not the active/configured provider (#5301 review finding: admitting any models-only config re-opens the copilot-2 duplicate-alias regression).""" _stub_hermes_cli(monkeypatch) monkeypatch.setattr("socket.getaddrinfo", lambda *a, **k: []) with _RestoreCfg(): _set_cfg({ "model": { "default": "my-model", "provider": "CLIPpoxy", }, "providers": { "CLIPpoxy": { "models": ["my-model"], }, "OtherCustom": { "models": ["unrelated-model"], }, }, }) config.invalidate_models_cache() result = config.get_available_models() groups = _groups_by_provider_id(result) assert "clippoxy" in groups, f"Active models-only provider should render, got: {list(groups)}" assert "othercustom" not in groups, ( f"Non-active models-only provider must not render as a phantom group, got: {list(groups)}" ) def test_provider_catalog_treats_malformed_provider_entry_as_unconfigured(monkeypatch): """Provider catalog raw-key lookup must ignore truth-y non-dict entries.""" _stub_hermes_cli(monkeypatch) monkeypatch.setattr("socket.getaddrinfo", lambda *a, **k: []) with _RestoreCfg(): _set_cfg({ "model": { "default": "claude-sonnet-4-5", "provider": "anthropic", }, "providers": {"anthropic": "enabled"}, }) config.invalidate_models_cache() result = config.get_available_models() groups = _groups_by_provider_id(result) assert "anthropic" in groups, f"Expected anthropic group, got: {list(groups)}" assert groups["anthropic"]["models"], "anthropic group should still fall back to models"