hermes-webui/tests/test_issue6335_catalog_admission_regression.py

"""Regression test for #6335 / #6338 catalog admission gate.

Issue #6335: after removing a provider's API key, a metadata-only
``providers.<name>: {name: ...}`` entry in config.yaml still rendered the
provider in the chat model selector.  The fix gates known-provider admission
on credential evidence: a provider listed in config.yaml is only admitted
when it was *already* detected from a credential source (env vars, hermes
auth, credential pool) — or carries an explicit route (``api`` /
``base_url`` / ``api_key`` / ``key_env``), or is the active provider with a
configured models allowlist.

Follow-up #6338: Hermes reports detected credentials under canonical alias
names (``xai``, ``gemini``) while WebUI preserves config.yaml keys as
``x-ai``, ``google``.  The credential check must compare in an
alias-normalised identity space or authenticated providers disappear from
the selector.

This test drives ``api/config.py::get_available_models()`` end-to-end with
a stubbed ``hermes_cli`` (deterministic, offline) and asserts the
catalog-builder admission behaviour:

1. Mocked Hermes provider detection reports authenticated ``xai`` and
   ``gemini`` (the agent's canonical alias names).
2. ``providers.x-ai`` and ``providers.google`` are metadata-only entries
   (name only — no route, no models allowlist).
3. The active provider is a *different* provider (``openai-codex``), so the
   test cannot pass through the unconditional active-provider path.
4. The xAI and Google groups remain admitted (alias-normalised credential
   match).
5. A metadata-only known provider with no matching credential, route field,
   or configured model list (``openai-api``) stays absent.
"""

from __future__ import annotations

import json
import sys
import types

import api.config as config
import api.profiles as profiles


def _install_fake_hermes_cli(monkeypatch):
    """Stub hermes_cli so detection is deterministic and offline.

    ``list_available_providers()`` reports authenticated ``xai`` and
    ``gemini`` — the agent's canonical alias names — so the ONLY way the
    metadata-only ``providers.x-ai`` / ``providers.google`` entries can be
    admitted is via the alias-normalised ``_already_credentialed`` gate
    under test.
    """
    fake_pkg = types.ModuleType("hermes_cli")
    fake_pkg.__path__ = []

    fake_models = types.ModuleType("hermes_cli.models")
    fake_models.list_available_providers = lambda: [
        {"id": "xai", "authenticated": True},
        {"id": "gemini", "authenticated": True},
    ]
    fake_models.provider_model_ids = lambda pid: []
    fake_models._PROVIDER_ALIASES = {
        "x-ai": "xai",
        "x.ai": "xai",
        "grok": "xai",
        "google": "gemini",
        "google-gemini": "gemini",
        "google-ai-studio": "gemini",
    }

    fake_auth = types.ModuleType("hermes_cli.auth")
    fake_auth.get_auth_status = lambda _pid: {}

    monkeypatch.setitem(sys.modules, "hermes_cli", fake_pkg)
    monkeypatch.setitem(sys.modules, "hermes_cli.models", fake_models)
    monkeypatch.setitem(sys.modules, "hermes_cli.auth", fake_auth)

    # Remove the real agent.credential_pool so no pool evidence leaks in
    # from the host environment.
    monkeypatch.delitem(sys.modules, "agent.credential_pool", raising=False)
    monkeypatch.delitem(sys.modules, "agent", raising=False)


def _call_get_available_models(monkeypatch, tmp_path):
    _install_fake_hermes_cli(monkeypatch)

    # Active provider is a DIFFERENT provider (openai-codex), so the
    # x-ai/google entries can only be admitted through the credential gate,
    # never through the active-provider path.
    (tmp_path / "auth.json").write_text(
        json.dumps(
            {
                "version": 1,
                "providers": {},
                "active_provider": "openai-codex",
                "credential_pool": {},
            }
        ),
        encoding="utf-8",
    )
    # config.yaml must exist on disk: get_available_models() reloads the
    # config from disk when the path/mtime changed, wiping in-memory cfg
    # overrides.  Metadata-only entries only (name) — no api/base_url/
    # api_key/key_env route fields, no models allowlist.
    (tmp_path / "config.yaml").write_text(
        "model:\n  provider: openai-codex\nproviders:\n"
        "  x-ai:\n    name: xAI\n"
        "  google:\n    name: Google\n"
        "  openai-api:\n    name: OpenAI API\n",
        encoding="utf-8",
    )
    monkeypatch.setattr(profiles, "get_active_hermes_home", lambda: tmp_path)

    for var in (
        "OPENAI_API_KEY",
        "HERMES_API_KEY",
        "HERMES_OPENAI_API_KEY",
        "LOCAL_API_KEY",
        "OPENROUTER_API_KEY",
        "API_KEY",
        "XAI_API_KEY",
        "GOOGLE_API_KEY",
        "GEMINI_API_KEY",
    ):
        monkeypatch.delenv(var, raising=False)

    old_cfg = dict(config.cfg)
    old_mtime = config._cfg_mtime
    old_path = config._cfg_path
    old_fingerprint = config._cfg_fingerprint
    config.cfg.clear()
    config.cfg["model"] = {"provider": "openai-codex"}
    config.cfg["providers"] = {
        "x-ai": {"name": "xAI"},
        "google": {"name": "Google"},
        "openai-api": {"name": "OpenAI API"},
    }
    cfg_path = config.Path(config._get_config_path())
    try:
        config._cfg_mtime = cfg_path.stat().st_mtime
    except Exception:
        config._cfg_mtime = 0.0
    # Pin the path/fingerprint globals to THIS fixture so the path_changed /
    # fingerprint staleness checks cannot force a disk reload that wipes the
    # in-memory overrides above.  A preceding config test (e.g.
    # test_byok_model_dropdown.py) can leave _cfg_path pointing at its own
    # deleted temp config; without the pin, get_available_models() sees
    # path_changed=True and reloads from disk, dropping the fixture's x-ai /
    # google entries (gate certification 2026-08-07: byok -> 6335 = 1 failed).
    config._cfg_path = cfg_path
    config._cfg_fingerprint = config._fingerprint_config(config.cfg)

    config.invalidate_models_cache()
    try:
        return config.get_available_models()
    finally:
        config.cfg.clear()
        config.cfg.update(old_cfg)
        config._cfg_mtime = old_mtime
        config._cfg_path = old_path
        config._cfg_fingerprint = old_fingerprint
        config.invalidate_models_cache()


def _group_ids(result):
    return [g.get("provider_id") for g in result.get("groups", [])]


def _group_names(result):
    return [g.get("provider") for g in result.get("groups", [])]


def test_predecessor_leaked_cfg_path_fingerprint_does_not_break_admission(
    monkeypatch, tmp_path
):
    """Predecessor-order regression: leaked ``_cfg_path``/``_cfg_fingerprint``
    must not trigger a disk reload that wipes the fixture providers.

    A preceding config test (``tests/test_byok_model_dropdown.py`` sorts
    before this file) repoints ``_get_config_path`` at its own temporary
    config and leaves ``config._cfg_path`` / ``config._cfg_fingerprint``
    pointing at that now-deleted file.  The next
    ``get_available_models()`` call then sees ``path_changed=True`` and
    reloads from disk, dropping the fixture's ``x-ai`` / ``google`` entries
    (reproduced by the 2026-08-07 gate certification: byok -> 6335 = 1
    failed, ``got ['openai-codex', 'gemini']``).

    This test runs FIRST in this file, deliberately simulates that exact
    leaked state, and asserts the helper pins over it — proving the two
    admission tests below also pass after a leaking predecessor.
    """
    import api.config as config

    leaked_path = tmp_path / "deleted-predecessor-config.yaml"
    old_path, old_fingerprint = config._cfg_path, config._cfg_fingerprint
    try:
        config._cfg_path = leaked_path
        config._cfg_fingerprint = "stale-predecessor-fingerprint"
        result = _call_get_available_models(monkeypatch, tmp_path)
        ids = _group_ids(result)
        assert "openai-codex" in ids, f"active provider group missing; got {ids}"
        assert "x-ai" in ids, f"leaked _cfg_path must not wipe x-ai; got {ids}"
        assert "google" in ids, f"leaked _cfg_path must not wipe google; got {ids}"
        assert "xAI" in _group_names(result)
        assert "Google" in _group_names(result)
    finally:
        config._cfg_path = old_path
        config._cfg_fingerprint = old_fingerprint


def test_credentialed_alias_config_providers_stay_admitted(monkeypatch, tmp_path):
    """Authenticated xai/gemini keep metadata-only x-ai/google entries in the picker.

    Covers review points 1-4: detection reports authenticated ``xai`` /
    ``gemini`` (alias names), config has metadata-only ``x-ai`` / ``google``
    entries, the active provider is ``openai-codex`` (a different provider),
    and the xAI / Google groups remain admitted through the alias-normalised
    credential check.
    """
    result = _call_get_available_models(monkeypatch, tmp_path)
    ids = _group_ids(result)

    # Point 3 sanity: the active provider is a different provider, so these
    # groups cannot be explained by the unconditional active-provider path.
    assert "openai-codex" in ids, f"active provider group missing; got {ids}"

    # Point 4: the alias-matched credential evidence keeps both groups.
    assert "x-ai" in ids, f"authenticated xai must admit configured x-ai group; got {ids}"
    assert "google" in ids, f"authenticated gemini must admit configured google group; got {ids}"
    assert "xAI" in _group_names(result)
    assert "Google" in _group_names(result)


def test_keyless_metadata_only_known_provider_stays_absent(monkeypatch, tmp_path):
    """A metadata-only known provider with no credential/route/models is dropped.

    Review point 5: ``providers.openai-api: {name: ...}`` is a *known*
    provider (in ``_PROVIDER_MODELS`` / ``_PROVIDER_DISPLAY``) but carries no
    credential evidence, no route field, and no configured model list — the
    #6335 bug.  It must not render in the model selector.
    """
    result = _call_get_available_models(monkeypatch, tmp_path)
    ids = _group_ids(result)
    assert "openai-api" not in ids, (
        f"keyless metadata-only openai-api must stay absent; got {ids}"
    )