""" Tests for resolve_model_provider() model routing logic. Verifies that model IDs are correctly resolved to (model, provider, base_url) tuples for different provider configurations. """ import pytest import api.config as config def _resolve_with_config(model_id, provider=None, base_url=None, default=None, custom_providers=None): """Helper: temporarily set config.cfg model/custom provider sections, call resolve, restore.""" old_cfg = dict(config.cfg) model_cfg = {} if provider: model_cfg['provider'] = provider if base_url: model_cfg['base_url'] = base_url if default: model_cfg['default'] = default config.cfg['model'] = model_cfg if model_cfg else {} if custom_providers is not None: config.cfg['custom_providers'] = custom_providers try: return config.resolve_model_provider(model_id) finally: config.cfg.clear() config.cfg.update(old_cfg) def _resolve_with_catalog(model_id, advertised_ids, *, provider=None, base_url=None, provider_id='custom', default=None): """Resolve with a seeded models-catalog snapshot (#5979 provenance). ``advertised_ids`` is the list of model ids the endpoint's own group advertised (what the user could have picked from the dropdown). Pass ``None`` to simulate a COLD/unbuilt catalog. Seeds ``config._available_models_cache`` (the snapshot ``_endpoint_advertised_model_ids`` reads) for the duration of the call, resets the derivation memo, and restores both afterwards so tests don't leak catalog state into each other. """ old_cache = config._available_models_cache old_memo = config._advertised_model_ids_memo if advertised_ids is None: config._available_models_cache = None else: config._available_models_cache = { 'groups': [{ 'provider_id': provider_id, 'models': [{'id': mid, 'label': mid} for mid in advertised_ids], }] } config._advertised_model_ids_memo = None # force recompute against the seeded snapshot try: return _resolve_with_config(model_id, provider=provider, base_url=base_url, default=default) finally: config._available_models_cache = old_cache config._advertised_model_ids_memo = old_memo # ── OpenRouter prefix handling ──────────────────────────────────────────── def test_openrouter_free_keeps_full_path(): """openrouter/free must NOT be stripped to 'free' when provider is openrouter.""" model, provider, base_url = _resolve_with_config( 'openrouter/free', provider='openrouter', base_url='https://openrouter.ai/api/v1', ) assert model == 'openrouter/free', f"Expected 'openrouter/free', got '{model}'" assert provider == 'openrouter' def test_openrouter_model_with_provider_prefix(): """anthropic/claude-sonnet-4.6 via openrouter keeps full path.""" model, provider, base_url = _resolve_with_config( 'anthropic/claude-sonnet-4.6', provider='openrouter', base_url='https://openrouter.ai/api/v1', ) assert model == 'anthropic/claude-sonnet-4.6' assert provider == 'openrouter' # ── Direct provider prefix stripping ───────────────────────────────────── def test_anthropic_prefix_stripped_for_direct_api(): """anthropic/claude-sonnet-4.6 strips prefix when provider is anthropic.""" model, provider, base_url = _resolve_with_config( 'anthropic/claude-sonnet-4.6', provider='anthropic', ) assert model == 'claude-sonnet-4.6' assert provider == 'anthropic' def test_openai_prefix_stripped_for_direct_api(): """openai/gpt-5.4-mini strips prefix when provider is openai.""" model, provider, base_url = _resolve_with_config( 'openai/gpt-5.4-mini', provider='openai', ) assert model == 'gpt-5.4-mini' assert provider == 'openai' # ── Cross-provider routing ─────────────────────────────────────────────── def test_cross_provider_routes_through_openrouter(): """Picking openai model when config is anthropic routes via openrouter.""" model, provider, base_url = _resolve_with_config( 'openai/gpt-5.4-mini', provider='anthropic', ) assert model == 'openai/gpt-5.4-mini' assert provider == 'openrouter' assert base_url is None # openrouter uses its own endpoint # ── Bare model names ───────────────────────────────────────────────────── def test_bare_model_uses_config_provider(): """A model name without / uses the config provider and base_url.""" model, provider, base_url = _resolve_with_config( 'gemma-4-26B', provider='custom', base_url='http://192.168.1.160:4000', ) assert model == 'gemma-4-26B' assert provider == 'custom' assert base_url == 'http://192.168.1.160:4000' def test_empty_model_returns_config_defaults(): """Empty model string returns config provider and base_url.""" model, provider, base_url = _resolve_with_config( '', provider='anthropic', ) assert model == '' assert provider == 'anthropic' # ── @provider:model hint routing (Issue #138 v2) ──────────────────────── def test_provider_hint_routes_to_specific_provider(): """@minimax:MiniMax-M2.7 routes to minimax provider directly.""" model, provider, base_url = _resolve_with_config( '@minimax:MiniMax-M2.7', provider='anthropic', ) assert model == 'MiniMax-M2.7' assert provider == 'minimax' assert base_url is None # resolve_runtime_provider will fill this def test_provider_hint_zai(): """@zai:GLM-5 routes to zai provider directly.""" model, provider, base_url = _resolve_with_config( '@zai:GLM-5', provider='openai', ) assert model == 'GLM-5' assert provider == 'zai' def test_provider_hint_deepseek(): """@deepseek:deepseek-chat routes to deepseek provider.""" model, provider, base_url = _resolve_with_config( '@deepseek:deepseek-chat', provider='anthropic', ) assert model == 'deepseek-chat' assert provider == 'deepseek' def test_slash_prefix_non_default_still_routes_openrouter(): """minimax/MiniMax-M2.7 (old format) still routes through openrouter.""" model, provider, base_url = _resolve_with_config( 'minimax/MiniMax-M2.7', provider='anthropic', ) assert model == 'minimax/MiniMax-M2.7' assert provider == 'openrouter' def test_custom_provider_model_with_slash_routes_to_named_custom_provider(): """Slash-containing custom endpoint model IDs must not be mistaken for OpenRouter models.""" model, provider, base_url = _resolve_with_config( 'google/gemma-4-26b-a4b', provider='openrouter', base_url='https://openrouter.ai/api/v1', custom_providers=[{ 'name': 'Local LM Studio', 'base_url': 'http://lmstudio.local:1234/v1', 'model': 'google/gemma-4-26b-a4b', }], ) assert model == 'google/gemma-4-26b-a4b' assert provider == 'custom:local-lm-studio' assert base_url == 'http://lmstudio.local:1234/v1' # ── #3872: bare ``custom`` provider is a vendor-routing proxy — preserve the # full model id (the prefix is intrinsic). #433's redundant-prefix strip is # scoped to real first-party providers (provider=openai + proxy base_url), # which is covered by test_custom_endpoint_slash_model_routes_to_custom_not_openrouter. def test_custom_remote_preserves_intrinsic_vendor_prefix_3872(): """#3872: bedrock/opus-4-6 on a bare-custom remote proxy keeps its full id. A bare ``custom`` provider with a remote base_url is a vendor-routing proxy (LiteLLM, Bedrock gateway). ``bedrock/`` is an intrinsic routing segment the proxy needs whole; stripping it to ``opus-4-6`` makes the proxy return 403 "model not allowed for your group". The proxy advertised the full id (the user picked it from the dropdown), so provenance preserves it. """ model, provider, base_url = _resolve_with_catalog( 'bedrock/opus-4-6', advertised_ids=['bedrock/opus-4-6'], provider='custom', base_url='https://router.example.com/v1', ) assert model == 'bedrock/opus-4-6', f"intrinsic prefix must be preserved, got {model!r}" assert provider == 'custom' assert base_url == 'https://router.example.com/v1' def test_custom_remote_strips_redundant_first_party_prefix_433(): """#433: bare-custom remote proxy strips a prefix ONLY when the endpoint advertised just the BARE id (not the full ``vendor/model``). #433 is a verified real relay whose ``/v1/models`` returned bare ``gpt-5.4`` and rejected ``openai/gpt-5.4`` (a stale cross-provider leftover). The strip is now justified by PROVENANCE — the catalog advertises exactly ``gpt-5.4`` and NOT ``openai/gpt-5.4`` — instead of the old catalog-family guess that couldn't tell this stale-leftover case apart from #5979's advertised-full-id case. Behaviour is also pinned by test_sprint40_ui_polish.py::test_prefixed_model_stripped_for_custom_endpoint. """ model, provider, base_url = _resolve_with_catalog( 'openai/gpt-5.4', advertised_ids=['gpt-5.4'], # relay advertises ONLY the bare id provider='custom', base_url='https://router.example.com/v1', ) assert model == 'gpt-5.4', f"redundant first-party prefix must be stripped, got {model!r}" assert provider == 'custom' def test_custom_remote_preserves_advertised_full_id_5979(): """#5979 (the P0 regression): a custom proxy that advertises the FULL ``x-ai/grok-4.5`` must receive it whole — never the bare ``grok-4.5``. The old code stripped it because ``grok-4.5`` had graduated into the x-ai first-party catalog (agent commit 62ada5175), flipping ``_is_first_party_model`` to True for a model the proxy routes on by its full ``x-ai/`` namespace. This is the exact HTTP 400 b3nw hit ("Invalid model format ... grok-4.5"). The provenance rule preserves it because the endpoint advertised the full id. """ # Reproduce the data-driven trigger: grok-4.5 present in the x-ai catalog. xai_catalog = config._PROVIDER_MODELS.get('x-ai') assert isinstance(xai_catalog, list) assert any(isinstance(m, dict) and m.get('id') == 'grok-4.5' for m in xai_catalog), ( "guard: grok-4.5 must be first-party for this regression test to be meaningful" ) model, provider, base_url = _resolve_with_catalog( 'x-ai/grok-4.5', advertised_ids=['x-ai/grok-4.5'], # proxy advertises the FULL namespaced id provider='custom', base_url='https://proxy.example.com/v1', ) assert model == 'x-ai/grok-4.5', ( f"advertised full id must be preserved for routing, got {model!r}" ) assert provider == 'custom' assert base_url == 'https://proxy.example.com/v1' def test_named_custom_slug_preserves_advertised_full_id_5979(): """#5979 (named-custom variant): provider=custom: proxy advertising the full ``x-ai/grok-4.5`` also preserves it.""" model, provider, base_url = _resolve_with_catalog( 'x-ai/grok-4.5', advertised_ids=['x-ai/grok-4.5'], provider='custom:my-gateway', provider_id='custom:my-gateway', base_url='https://proxy.example.com/v1', ) assert model == 'x-ai/grok-4.5', f"full id must be preserved for custom:slug, got {model!r}" def test_custom_remote_cold_catalog_preserves_verbatim_5979(): """#5979 fail-safe: with a COLD/unbuilt catalog (no provenance signal), a custom proxy preserves the model id verbatim — for BOTH the advertised-full case and the stale-leftover case. Preserve is the correct cold default: a wrong strip destroys a namespace the user actively selected this session (recurs every turn, unrepairable), while a wrong preserve only mis-sends a stale leftover that fails loudly and heals the instant the catalog builds. The active-user-selection case (#5979) is cold-safe by construction — you can't select from a dropdown that was never built. """ model_a, _, _ = _resolve_with_catalog( 'x-ai/grok-4.5', advertised_ids=None, provider='custom', base_url='https://proxy.example.com/v1', ) assert model_a == 'x-ai/grok-4.5', f"cold catalog must preserve verbatim, got {model_a!r}" model_b, _, _ = _resolve_with_catalog( 'openai/gpt-5.4', advertised_ids=None, provider='custom', base_url='https://router.example.com/v1', ) assert model_b == 'openai/gpt-5.4', f"cold catalog must preserve verbatim, got {model_b!r}" def test_custom_remote_prefers_full_id_when_both_advertised_5979(): """When a proxy advertises BOTH the full ``x-ai/grok-4.5`` and a bare ``grok-4.5``, the exact full selection wins (preserve).""" model, _, _ = _resolve_with_catalog( 'x-ai/grok-4.5', advertised_ids=['x-ai/grok-4.5', 'grok-4.5'], provider='custom', base_url='https://proxy.example.com/v1', ) assert model == 'x-ai/grok-4.5', f"exact full selection must win, got {model!r}" def test_custom_remote_preserves_unknown_prefix_548(): """#548: an unknown vendor prefix (zai-org/GLM-5.1) is always preserved. The proxy advertised the full id; ``zai-org`` isn't in _PROVIDER_MODELS so even the bare-advertised belt could never strip it. """ model, provider, base_url = _resolve_with_catalog( 'zai-org/GLM-5.1', advertised_ids=['zai-org/GLM-5.1'], provider='custom', base_url='https://api.deepinfra.com/v1/openai', ) assert model == 'zai-org/GLM-5.1', f"unknown prefix must be preserved, got {model!r}" assert provider == 'custom' def test_named_custom_slug_preserves_intrinsic_vendor_prefix_3872(): """#3872 (named-custom variant): provider=custom: + remote base_url also preserves an intrinsic vendor prefix the endpoint advertised whole. """ model, provider, base_url = _resolve_with_catalog( 'bedrock/opus-4-6', advertised_ids=['bedrock/opus-4-6'], provider='custom:my-gateway', provider_id='custom:my-gateway', base_url='https://router.example.com/v1', ) assert model == 'bedrock/opus-4-6', f"intrinsic prefix must be preserved for custom:slug, got {model!r}" def test_first_party_provider_proxy_still_strips_prefix_433(): """#433/dc2334c5: provider=openai + remote proxy still strips the prefix. This is the deliberate behaviour the #3872 fix must NOT regress: a real first-party provider pointed at an OpenAI-compatible proxy expects the bare id. (Mirrors the public-host branch of test_custom_endpoint_slash_model_routes_to_custom_not_openrouter.) """ model, provider, base_url = _resolve_with_config( 'openai/gpt-5.4', provider='openai', base_url='https://litellm.example.com/v1', ) assert model == 'gpt-5.4', f"redundant first-party prefix must be stripped, got {model!r}" def test_custom_provider_models_dict_routes_to_named_custom_provider(): """Models listed only under custom_providers[].models still route to that endpoint.""" model, provider, base_url = _resolve_with_config( 'sensenova-6.7-flash-lite', provider='xiaomi', custom_providers=[{ 'name': 'LiteLLM Proxy', 'base_url': 'http://127.0.0.1:8080/v1', 'model': 'deepseek-v4-flash', 'models': { 'deepseek-v4-flash': {}, 'sensenova-6.7-flash-lite': {}, }, }], ) assert model == 'sensenova-6.7-flash-lite' assert provider == 'custom:litellm-proxy' assert base_url == 'http://127.0.0.1:8080/v1' # ── Issue #2047: parenthesized local provider names with ports ──────────── def test_custom_provider_name_with_parenthesized_port_uses_safe_slug(): """Setup-generated names like 'Local (host:port)' must not leak ':' into slugs.""" model, provider, base_url = _resolve_with_config( 'deepseek-v4-flash', provider='custom', custom_providers=[{ 'name': 'Local (127.0.0.1:15721)', 'base_url': 'http://127.0.0.1:15721/v1', 'model': 'deepseek-v4-flash', }], ) assert model == 'deepseek-v4-flash' assert provider == 'custom:local-127.0.0.1-15721' assert base_url == 'http://127.0.0.1:15721/v1' def test_safe_custom_provider_hint_keeps_model_after_port_slug(): """The safe slug emitted by the picker must parse back without corrupting the model.""" model, provider, base_url = _resolve_with_config( '@custom:local-127.0.0.1-15721:deepseek-v4-flash', provider='custom', ) assert model == 'deepseek-v4-flash' assert provider == 'custom:local-127.0.0.1-15721' assert base_url is None # ── Issue #1922: default model shadowed by overlapping custom_providers[] ── def test_default_model_not_shadowed_by_overlapping_custom_provider(): r'''Regression test for #1922. When the active provider is an explicit non-custom provider (e.g. ai-gateway, openrouter, xiaomi) AND the requested model_id matches the configured default model, the active provider's base_url must take precedence over an overlapping custom_providers[] entry. Otherwise the WebUI routes to 'custom:' with the wrong endpoint, causing 401 errors. This test mirrors the reported scenario: - provider: ai-gateway - base_url: https://api.ai-gateway.example/v1 - default: gpt-5.4 - An overlapping custom_providers[] entry with the same default model Expected: active provider (ai-gateway) wins over custom provider. ''' model, provider, base_url = _resolve_with_config( 'gpt-5.4', provider='ai-gateway', base_url='https://api.ai-gateway.example/v1', default='gpt-5.4', custom_providers=[{ 'name': 'My Custom Endpoint', 'base_url': 'http://localhost:8080/v1', 'model': 'gpt-5.4', }], ) assert model == 'gpt-5.4', f'Expected model=gpt-5.4, got {model!r}' assert provider == 'ai-gateway', f'Expected provider=ai-gateway, got {provider!r}' assert base_url == 'https://api.ai-gateway.example/v1', f'Expected base_url from active provider, got {base_url!r}' def test_default_model_shadowed_with_xiaomi_provider(): r'''Same regression test with provider=xiaomi instead of ai-gateway.''' model, provider, base_url = _resolve_with_config( 'deepseek-v4-flash', provider='xiaomi', default='deepseek-v4-flash', custom_providers=[{ 'name': 'LiteLLM Proxy', 'base_url': 'http://127.0.0.1:8080/v1', 'model': 'deepseek-v4-flash', }], ) assert model == 'deepseek-v4-flash' assert provider == 'xiaomi' assert base_url is None # xiaomi has no config base_url in this test # ── get_available_models() @provider: hint behaviour ────────────────────── @pytest.fixture(autouse=True) def _isolate_models_cache(): """Invalidate the models TTL cache before and after every test in this file. Several helpers here mutate ``config.cfg`` in-memory and call ``get_available_models()``. Without this guard, a prior test that called ``get_available_models()`` leaves a 60-second TTL cache entry; the next test that mutates cfg and calls the function gets a cache hit instead of running the function body, causing silently wrong results (e.g. the ``test_custom_endpoint_uses_model_config_api_key_for_model_discovery`` ``KeyError: 'auth'`` on CI where ``urlopen`` is never reached). """ try: config.invalidate_models_cache() except Exception: pass yield try: config.invalidate_models_cache() except Exception: pass def _available_models_with_provider(provider): """Helper: temporarily set active_provider in config.""" old_cfg = dict(config.cfg) config.cfg['model'] = {'provider': provider} try: return config.get_available_models() finally: config.cfg.clear() config.cfg.update(old_cfg) def test_non_default_provider_models_use_hint_prefix(): """With anthropic as default, minimax model IDs should use @minimax: prefix.""" result = _available_models_with_provider('anthropic') groups = {g['provider']: g['models'] for g in result['groups']} if 'MiniMax' in groups: for m in groups['MiniMax']: assert m['id'].startswith('@minimax:'), ( f"Expected @minimax: prefix, got: {m['id']!r}" ) def test_no_duplicate_when_default_model_is_prefixed(): """Issue #147 Bug 2: 'anthropic/claude-opus-4.6' as default_model must not inject a duplicate alongside the existing bare 'claude-opus-4.6' entry in the same provider group.""" import api.config as _cfg old_cfg = dict(_cfg.cfg) _cfg.cfg['model'] = { 'provider': 'anthropic', 'default': 'anthropic/claude-opus-4.6', } try: result = _cfg.get_available_models() norm = lambda mid: mid.split('/', 1)[-1] if '/' in mid else mid # Check each group individually: no group should have two entries that # normalize to the same bare model name for g in result['groups']: bare_ids = [norm(m['id']) for m in g['models']] duplicates = [mid for mid in set(bare_ids) if bare_ids.count(mid) > 1] assert not duplicates, ( f"Provider group '{g['provider']}' has duplicate models after normalization: " f"{duplicates}\nFull group: {[m['id'] for m in g['models']]}" ) finally: _cfg.cfg.clear() _cfg.cfg.update(old_cfg) def test_default_provider_models_not_prefixed(monkeypatch): """The active provider's models remain bare (no @prefix added).""" import api.config as _cfg monkeypatch.setattr(_cfg, "_read_live_provider_model_ids", lambda pid: ["claude-sonnet-5.0"] if pid == "anthropic" else []) result = _available_models_with_provider('anthropic') groups = {g['provider']: g['models'] for g in result['groups']} if 'Anthropic' in groups: returned_ids = {m['id'] for m in groups['Anthropic']} assert "claude-sonnet-5.0" in returned_ids assert not any(mid.startswith('@anthropic:') for mid in returned_ids), returned_ids # ── get_available_models(): phantom "Custom" group regression ───────────── # # When the user has model.provider set to a real provider (e.g. openai-codex) # AND a model.base_url set, hermes_cli reports the 'custom' pseudo-provider as # authenticated. The WebUI picker must NOT build a separate "Custom" group in # that case — the base_url belongs to the active provider. def _available_models_with_full_cfg(provider, default, base_url): """Helper: set model.provider, model.default, model.base_url at once. Clears model-override env vars (HERMES_MODEL, OPENAI_MODEL, LLM_MODEL) during the call so the real hermes profile environment doesn't leak into the test and override the fixture's default model. """ import os import api.config as _cfg old_cfg = dict(_cfg.cfg) _cfg.cfg['model'] = { 'provider': provider, 'default': default, 'base_url': base_url, } try: _cfg._cfg_mtime = _cfg.Path(_cfg._get_config_path()).stat().st_mtime except Exception: # No config.yaml on this machine (e.g. CI); pin to 0.0 so the mtime check # inside get_available_models() sees 0.0 == 0.0 and doesn't call reload_config(), # which would overwrite the in-memory cfg we just set up. _cfg._cfg_mtime = 0.0 # Clear model-override env vars to prevent the real profile from leaking in _model_env_keys = ('HERMES_MODEL', 'OPENAI_MODEL', 'LLM_MODEL') _saved_env = {k: os.environ.pop(k, None) for k in _model_env_keys} try: return _cfg.get_available_models() finally: _cfg.cfg.clear() _cfg.cfg.update(old_cfg) for k, v in _saved_env.items(): if v is not None: os.environ[k] = v def test_no_phantom_custom_group_when_active_provider_is_set(monkeypatch): """Issue: with provider=openai-codex + base_url set, gpt-5.4 was landing under a phantom "Custom" group instead of the "OpenAI Codex" group.""" import sys, types # Force hermes_cli to report both the real provider and the phantom # 'custom' as authenticated, simulating what list_available_providers() # returns when base_url is configured. fake_mod = types.ModuleType('hermes_cli.models') fake_mod.list_available_providers = lambda: [ {'id': 'openai-codex', 'authenticated': True}, {'id': 'custom', 'authenticated': True}, ] fake_auth = types.ModuleType('hermes_cli.auth') fake_auth.get_auth_status = lambda pid: {'key_source': 'env'} monkeypatch.setitem(sys.modules, 'hermes_cli.models', fake_mod) monkeypatch.setitem(sys.modules, 'hermes_cli.auth', fake_auth) result = _available_models_with_full_cfg( provider='openai-codex', default='gpt-5.4', base_url='https://chatgpt.com/backend-api/codex', ) group_names = [g['provider'] for g in result['groups']] assert 'Custom' not in group_names, ( f"Phantom 'Custom' group present; full groups: {group_names}" ) def test_default_model_lands_under_active_provider_group(monkeypatch): """The configured default_model must appear under the active provider's display group, even when the model isn't in _PROVIDER_MODELS[provider] AND the active provider isn't the alphabetical first detected provider. Regression guard for a hyphen-vs-space bug in the "ensure default_model appears" post-pass: the substring check `active_provider.lower() in g.get('provider', '').lower()` was failing for 'openai-codex' vs display name 'OpenAI Codex' (hyphen vs. space), silently falling back to groups[0] — which, when another provider sorted earlier alphabetically (e.g. 'anthropic'), placed gpt-5.4 in the WRONG group. """ import sys, types fake_mod = types.ModuleType('hermes_cli.models') fake_mod.list_available_providers = lambda: [ {'id': 'anthropic', 'authenticated': True}, # sorts before openai-codex {'id': 'openai-codex', 'authenticated': True}, {'id': 'custom', 'authenticated': True}, ] fake_auth = types.ModuleType('hermes_cli.auth') fake_auth.get_auth_status = lambda pid: {'key_source': 'env'} monkeypatch.setitem(sys.modules, 'hermes_cli.models', fake_mod) monkeypatch.setitem(sys.modules, 'hermes_cli.auth', fake_auth) result = _available_models_with_full_cfg( provider='openai-codex', default='gpt-5.4', base_url='https://chatgpt.com/backend-api/codex', ) groups = {g['provider']: [m['id'] for m in g['models']] for g in result['groups']} assert 'OpenAI Codex' in groups, f"OpenAI Codex group missing: {list(groups)}" norm = lambda mid: mid.split('/', 1)[-1].split(':', 1)[-1] assert 'gpt-5.4' in {norm(mid) for mid in groups['OpenAI Codex']}, ( f"gpt-5.4 not in OpenAI Codex group; contents: {groups['OpenAI Codex']}" ) # And crucially, it must NOT have landed in the alphabetically-first # group (Anthropic) via the fallback path. assert 'gpt-5.4' not in {norm(mid) for mid in groups.get('Anthropic', [])}, ( f"gpt-5.4 leaked into Anthropic group via fallback: {groups.get('Anthropic')}" ) def test_unknown_providers_do_not_inherit_default_model(monkeypatch): """Detected providers without their own model catalog must not be filled with the global default_model placeholder. Regression guard for the bug where unknown providers ended up showing gpt-5.4-mini even though those providers do not serve it. Minimax-Cn is now known and should show its own catalog instead. """ import sys, types fake_mod = types.ModuleType('hermes_cli.models') fake_mod.list_available_providers = lambda: [ {'id': 'openai-codex', 'authenticated': True}, {'id': 'alibaba', 'authenticated': True}, {'id': 'minimax-cn', 'authenticated': True}, ] fake_auth = types.ModuleType('hermes_cli.auth') fake_auth.get_auth_status = lambda pid: {'key_source': 'env'} monkeypatch.setitem(sys.modules, 'hermes_cli.models', fake_mod) monkeypatch.setitem(sys.modules, 'hermes_cli.auth', fake_auth) result = _available_models_with_full_cfg( provider='openai-codex', default='gpt-5.4-mini', base_url='', ) groups = {g['provider']: [m['id'] for m in g['models']] for g in result['groups']} norm = lambda mid: mid.split('/', 1)[-1].split(':', 1)[-1] assert 'Alibaba' not in groups, ( f"Alibaba should not inherit the default model placeholder: {groups}" ) assert 'MiniMax (China)' in groups, ( f"Minimax-Cn should render its own static catalog: {groups}" ) assert not any( norm(mid) == 'gpt-5.4-mini' for mid in groups.get('Alibaba', []) + groups.get('MiniMax (China)', []) ), ( f"Unknown provider groups still inherited the default model: {groups}" ) def test_custom_endpoint_uses_model_config_api_key_for_model_discovery(monkeypatch): """Custom endpoint model discovery must use model.api_key from config.yaml, not only environment variables, otherwise the dropdown collapses to the default model when /v1/models requires auth.""" import json as _json import api.config as _cfg old_cfg = dict(_cfg.cfg) _cfg.cfg['model'] = { 'provider': 'custom', 'default': 'gpt-5.4', 'base_url': 'https://example.test/v1', 'api_key': 'sk-test-model-key', } try: _cfg._cfg_mtime = _cfg.Path(_cfg._get_config_path()).stat().st_mtime except Exception: # No config.yaml on this machine (e.g. CI); pin to 0.0 so the mtime check # inside get_available_models() sees 0.0 == 0.0 and skips reload_config(). _cfg._cfg_mtime = 0.0 _cfg.cfg.pop('providers', None) captured = {} class _Resp: def read(self): return _json.dumps({'data': [{'id': 'gpt-5.2', 'name': 'GPT-5.2'}]}).encode('utf-8') def __enter__(self): return self def __exit__(self, exc_type, exc, tb): return False def _fake_urlopen(req, timeout=10): url = getattr(req, 'full_url', '') if 'example.test' in url: captured['auth'] = req.get_header('Authorization') captured['ua'] = req.get_header('User-agent') return _Resp() monkeypatch.setattr('urllib.request.urlopen', _fake_urlopen) monkeypatch.setattr('socket.getaddrinfo', lambda *a, **k: []) monkeypatch.delenv('OPENAI_API_KEY', raising=False) monkeypatch.delenv('HERMES_API_KEY', raising=False) monkeypatch.delenv('HERMES_OPENAI_API_KEY', raising=False) monkeypatch.delenv('LOCAL_API_KEY', raising=False) monkeypatch.delenv('OPENROUTER_API_KEY', raising=False) monkeypatch.delenv('API_KEY', raising=False) try: result = _cfg.get_available_models() finally: _cfg.cfg.clear() _cfg.cfg.update(old_cfg) assert captured['auth'] == 'Bearer sk-test-model-key' assert captured['ua'] == 'OpenAI/Python 1.0' groups = {g['provider']: [m['id'] for m in g['models']] for g in result['groups']} assert 'Custom' in groups # Model ID may be prefixed with @provider: due to cross-provider dedup (#1228) assert any('gpt-5.2' in m for m in groups['Custom']), f'gpt-5.2 not found in Custom: {groups}' # -- Issue #230: custom provider with slash model name ----------------------- def test_custom_endpoint_slash_model_routes_to_custom_not_openrouter(): """Regression test for #230, updated for #1625. When provider=custom (or any non-openrouter provider) and base_url is set, a model name containing a slash (e.g. google/gemma-4-26b-a4b) must NOT be rerouted to OpenRouter -- it should stay on the configured custom endpoint. #1625 layered an additional rule on top: a base_url pointing at a loopback or private-IP host is treated as a local model server (LM Studio, Ollama, llama.cpp, vLLM, TabbyAPI), which register models under their full HuggingFace path. On such hosts the prefix is now PRESERVED. The original #433 strip behaviour still applies on public hosts (real OpenAI-compatible proxies like LiteLLM at https://litellm.example.com/v1). """ # --- custom provider with slash model name should NOT go to openrouter --- model, provider, base_url = _resolve_with_config( 'google/gemma-4-26b-a4b', provider='custom', base_url='http://127.0.0.1:1234/v1', default='google/gemma-4-26b-a4b', ) assert provider.startswith('custom'), ( "Expected provider starting with 'custom', got '{}'. " "Slash in model name should NOT trigger OpenRouter rerouting when base_url is set.".format(provider) ) assert base_url == 'http://127.0.0.1:1234/v1', ( "Expected base_url 'http://127.0.0.1:1234/v1', got '{}'.".format(base_url) ) # #1625 (supersedes the v0.50 #433 strip-on-custom rule for loopback hosts): # 127.0.0.1 base_url is almost certainly a local LM Studio / Ollama / etc., # which keys models on the full HuggingFace path. Preserve the prefix. assert model == 'google/gemma-4-26b-a4b', ( "Model name prefix must be PRESERVED on loopback base_url (#1625), got '{}'.".format(model) ) # --- public-host openai-compatible proxy STILL strips per #433 ---------- model2, provider2, base_url2 = _resolve_with_config( 'google/gemma-4-26b-a4b', provider='openai', base_url='https://litellm.example.com/v1', default='google/gemma-4-26b-a4b', ) assert model2 == 'gemma-4-26b-a4b', ( "Public-host OpenAI-compat proxy must still strip prefix per #433, got '{}'.".format(model2) ) # --- openrouter with slash model name MUST still route to openrouter ----- model_or, provider_or, _ = _resolve_with_config( 'google/gemma-4-26b-a4b', provider='openrouter', base_url='https://openrouter.ai/api/v1', default='google/gemma-4-26b-a4b', ) assert provider_or == 'openrouter', ( "Expected provider 'openrouter', got '{}'. " "Slash model via openrouter provider must still resolve to openrouter.".format(provider_or) ) assert model_or == 'google/gemma-4-26b-a4b', ( "Model name should be preserved for openrouter, got '{}'.".format(model_or) ) # ── #4210: custom provider (no base_url) must not be hijacked to openrouter # when the model id has a known-provider prefix (sibling of #3872, which # only covered the base_url-set variant). Bug-report case 1. def test_custom_provider_no_base_url_with_known_prefix_keeps_custom_and_full_id_4210(): """#4210: provider=custom:llm-proxy (no base_url) + 'x-ai/grok-2' must NOT be redirected to openrouter. The prefix is intrinsic to the custom proxy's routing; the user did not pick anything from the OpenRouter dropdown.""" model, provider, base_url = _resolve_with_config( 'x-ai/grok-2', provider='custom:llm-proxy', default='x-ai/grok-2', ) assert provider == 'custom:llm-proxy', ( "Custom provider must not be hijacked to openrouter when no base_url is " "set; got provider={!r} model={!r}".format(provider, model) ) assert model == 'x-ai/grok-2', ( "Custom provider must preserve the full model id; got model={!r}".format(model) ) assert base_url is None def test_bare_custom_provider_no_base_url_with_known_prefix_keeps_custom_and_full_id_4210(): """#4210 sibling: bare 'custom' (no 'custom:') with no base_url must also not be hijacked to openrouter for a known-prefix model id.""" model, provider, base_url = _resolve_with_config( 'google/gemma-2-9b', provider='custom', default='google/gemma-2-9b', ) assert provider == 'custom', ( "Bare 'custom' provider must not be hijacked to openrouter when no " "base_url is set; got provider={!r}".format(provider) ) assert model == 'google/gemma-2-9b' assert base_url is None # ── providers: (config.yaml user-defined provider) scan (#5511) ───────────── def _resolve_with_providers(model_id, providers_cfg, *, provider=None, default=None): """Helper: temporarily set config.cfg['providers'] + model, call resolve, restore.""" old_cfg = dict(config.cfg) model_cfg = {} if provider: model_cfg['provider'] = provider if default: model_cfg['default'] = default config.cfg['model'] = model_cfg config.cfg['providers'] = providers_cfg try: return config.resolve_model_provider(model_id) finally: config.cfg.clear() config.cfg.update(old_cfg) def test_providers_scan_routes_user_defined_allowlist_5511(): """A user-defined providers..models allowlist routes a bare model id to that provider (the feature #5511 adds).""" model, provider, base_url = _resolve_with_providers( 'my-model-1', {'myprov': {'base_url': 'https://my.example/v1', 'models': ['my-model-1', 'my-model-2']}}, provider='openai', default='gpt-5', ) assert provider == 'myprov', f"user-defined provider allowlist must route; got {provider!r}" assert model == 'my-model-1' assert base_url == 'https://my.example/v1' def test_providers_scan_skips_copilot_settings_map_5511(): """providers.copilot.models is a per-model SETTINGS map, NOT a routable allowlist — a Copilot per-model settings entry must NOT hijack routing away from the model's real provider (#5511 gate-cert CORE finding).""" model, provider, base_url = _resolve_with_providers( 'gpt-5', {'copilot': {'models': {'gpt-5': {'reasoning_effort': 'high'}}}}, provider='openai', default='gpt-5', ) assert provider == 'openai', ( "Copilot settings-map entry must NOT hijack routing; " f"gpt-5 must stay on openai, got {provider!r}" ) assert model == 'gpt-5' def test_providers_scan_copilot_list_shape_also_skipped_5511(): """Defense in depth: even if providers.copilot.models is a list shape, the Copilot exclusion still prevents a routing hijack.""" model, provider, base_url = _resolve_with_providers( 'gpt-5', {'copilot': {'models': ['gpt-5', 'gpt-5-mini']}}, provider='openai', default='gpt-5', ) assert provider == 'openai', ( f"Copilot (list shape) must not hijack routing; got {provider!r}" ) assert model == 'gpt-5' def test_providers_scan_honors_active_provider_ownership_5511(): """When the active provider owns the model (it's the configured default), another provider's overlapping providers..models entry must NOT hijack routing away from the active provider (#5511 gate finding — active ai-gateway + default gpt-5 was being pulled to providers.openai.models.gpt-5).""" model, provider, base_url = _resolve_with_providers( 'gpt-5', {'openai': {'models': ['gpt-5']}}, provider='ai-gateway', default='gpt-5', ) assert provider == 'ai-gateway', ( "active provider that owns the default model must keep routing; " f"gpt-5 must stay on ai-gateway, got {provider!r}" ) assert model == 'gpt-5' def test_providers_scan_active_provider_own_entry_still_matches_5511(): """The ownership guard still lets the ACTIVE provider's own providers: entry match (e.g. active myprov + a model in providers.myprov.models resolves to myprov with its base_url).""" model, provider, base_url = _resolve_with_providers( 'gpt-5', {'myprov': {'base_url': 'https://my.example/v1', 'models': ['gpt-5']}}, provider='myprov', default='gpt-5', ) assert provider == 'myprov', f"active provider's own entry must match; got {provider!r}" assert base_url == 'https://my.example/v1' def test_providers_scan_ownership_guard_canonicalises_aliased_active_provider_5511(): """An ALIASED active provider (e.g. 'z-ai' → canonical 'zai') must still be recognized as owning its catalog models, so another providers..models entry can't hijack an active-owned model (#5511 latent-bug gate finding — _provider_models_set was built with the raw alias, missing the canonical _PROVIDER_MODELS key, so the ownership guard silently failed).""" import api.config as config # Pick a real catalog model id owned by the canonical 'zai' provider. zai_models = config._PROVIDER_MODELS.get('zai') or [] zai_ids = [m.get('id') for m in zai_models if isinstance(m, dict) and m.get('id')] if not zai_ids: import pytest pytest.skip("no zai catalog models to exercise the alias ownership guard") owned = zai_ids[0] model, provider, base_url = _resolve_with_providers( owned, {'openai': {'models': [owned]}}, provider='z-ai', # aliased form the user may write in config ) assert config._canonicalise_provider_id('z-ai') == 'zai' assert provider == 'z-ai', ( "aliased active provider (z-ai→zai) that owns the model must not be " f"hijacked by providers.openai.models; got {provider!r}" ) def test_providers_scan_ownership_guard_canonicalises_gemini_alias_5511(): """A Gemini-family alias (google-gemini → gemini) must canonicalise so the active provider is recognized as owning its catalog models (#5511 latent bug: `gemini` is in _PROVIDER_MODELS but not _PROVIDER_DISPLAY, so the alias was rejected and the ownership guard silently failed).""" import api.config as config assert config._canonicalise_provider_id('google-gemini') == 'gemini', ( "google-gemini must canonicalise to gemini" ) gem_models = config._PROVIDER_MODELS.get('gemini') or [] gem_ids = [m.get('id') for m in gem_models if isinstance(m, dict) and m.get('id')] if not gem_ids: import pytest pytest.skip("no gemini catalog models to exercise the alias ownership guard") owned = gem_ids[0] model, provider, base_url = _resolve_with_providers( owned, {'openai': {'models': [owned]}}, provider='google-gemini', ) assert provider == 'google-gemini', ( "aliased active Gemini provider that owns the model must not be hijacked " f"by providers.openai.models; got {provider!r}" ) def test_providers_scan_active_own_providers_entry_owns_over_other_slug_5511(): """An active provider defined purely via config.yaml `providers:` (no static catalog entry) owns the models in its OWN providers..models allowlist, so another provider's entry listing the same bare id (even earlier in config order) must NOT hijack it (#5511 gate finding 5).""" # 'openai' entry lists gpt-x first, but the ACTIVE provider (myprov) also # declares gpt-x in its own providers.myprov.models — active must win. model, provider, base_url = _resolve_with_providers( 'gpt-x', { 'openai': {'models': ['gpt-x']}, 'myprov': {'base_url': 'https://my.example/v1', 'models': ['gpt-x']}, }, provider='myprov', ) assert provider == 'myprov', ( "active provider's own providers: allowlist must own its model over " f"another slug's overlapping entry; got {provider!r}" ) assert base_url == 'https://my.example/v1'