hermes-webui/tests/test_4413_seed_provider_models.py

"""Regression tests for #4413 — provider-model seeder from hermes_cli.

Covers three defects identified in PR review:

1.  **NameError at import time** — the seeder called ``_get_label_for_model``
    before it was defined, so the bare ``except Exception: pass`` swallowed the
    error exactly when the seeder had real work to do.

2.  **Provider alias mismatch** — the core uses canonical IDs (``xai``) while
    the WebUI indexes by display IDs (``x-ai``).  Without alias resolution the
    seeder created a duplicate provider entry instead of merging.

3.  **Live models endpoint** — ``_OPENAI_COMPAT_ENDPOINTS`` had the wrong ZAI
    URL (``/v1`` instead of ``/api/paas/v4``), and the live probe didn't prefer
    the provider's configured ``base_url``.
"""
import pathlib
import sys
import unittest.mock as mock

import pytest

REPO = pathlib.Path(__file__).parent.parent
sys.path.insert(0, str(REPO))
sys.path.insert(0, str(REPO.parent / ".hermes" / "hermes-agent"))

from api.config import (
    _PROVIDER_MODELS,
    _seed_provider_models_from_core,
    _resolve_provider_alias,
)


# ── helpers ──────────────────────────────────────────────────────────────────

def _deepcopy_providers():
    """Snapshot _PROVIDER_MODELS so tests can mutate freely and restore after."""
    return {k: list(v) for k, v in _PROVIDER_MODELS.items()}


@pytest.fixture
def restore_providers():
    """Restore _PROVIDER_MODELS to its pre-test state."""
    snapshot = _deepcopy_providers()
    yield
    _PROVIDER_MODELS.clear()
    _PROVIDER_MODELS.update(snapshot)


def _patch_core_pm(fake_pm: dict):
    """Patch ``hermes_cli.models._PROVIDER_MODELS`` with *fake_pm*."""
    # Build a fake hermes_cli.models module with just _PROVIDER_MODELS.
    fake_module = mock.MagicMock()
    fake_module._PROVIDER_MODELS = fake_pm
    return mock.patch.dict(sys.modules, {"hermes_cli.models": fake_module})


# ── Test 1: seeder actually adds missing models (catches NameError) ──────────

class TestSeederAddsMissingModels:
    """The seeder must add models that exist in core but not in WebUI.

    Before the fix, ``_get_label_for_model`` wasn't bound at call time, so the
    seeder raised ``NameError`` and the bare ``except Exception: pass`` swallowed
    it.  This test would have caught that bug.
    """

    def test_missing_model_is_added(self, restore_providers):
        """A model in core but not in WebUI should appear after seeding."""
        fake_pm = {"zai": ["glm-9.99-experimental"]}
        with _patch_core_pm(fake_pm):
            _seed_provider_models_from_core()

        zai_ids = [m["id"] for m in _PROVIDER_MODELS.get("zai", [])]
        assert "glm-9.99-experimental" in zai_ids, (
            "Seeder did not add glm-9.99-experimental — likely a NameError "
            "on _get_label_for_model at call time"
        )

    def test_added_model_has_label(self, restore_providers):
        """Seeded entries must have a human-readable label, not just an ID."""
        fake_pm = {"zai": ["glm-9.99-experimental"]}
        with _patch_core_pm(fake_pm):
            _seed_provider_models_from_core()

        zai_entries = {m["id"]: m for m in _PROVIDER_MODELS.get("zai", [])}
        assert "glm-9.99-experimental" in zai_entries
        label = zai_entries["glm-9.99-experimental"]["label"]
        assert isinstance(label, str) and label.strip(), (
            f"Label for seeded model is empty/missing: {label!r}"
        )

    def test_existing_model_not_duplicated(self, restore_providers):
        """A model that already exists should not be added a second time."""
        fake_pm = {"zai": ["glm-5.2"]}
        with _patch_core_pm(fake_pm):
            _seed_provider_models_from_core()

        zai_ids = [m["id"] for m in _PROVIDER_MODELS.get("zai", [])]
        assert zai_ids.count("glm-5.2") == 1, (
            f"glm-5.2 appears {zai_ids.count('glm-5.2')} times — should be 1"
        )

    def test_no_op_without_hermes_cli(self, restore_providers):
        """Seeder must be a no-op (not raise) when hermes_cli is unavailable."""
        with mock.patch.dict(sys.modules, {"hermes_cli.models": None}):
            # Should silently return without error
            _seed_provider_models_from_core()


# ── Test 2: aliased provider merges instead of duplicating ───────────────────

class TestAliasedProviderMerge:
    """Providers whose canonical ID differs between core and WebUI must merge.

    The core uses ``"xai"`` while the WebUI indexes by ``"x-ai"``.  Without
    alias resolution the seeder creates a duplicate ``"xai"`` entry.
    """

    def test_xai_merges_into_x_hyphen_ai(self, restore_providers):
        """Core's 'xai' should merge into WebUI's 'x-ai', not create 'xai'."""
        # Verify the alias exists in the first place
        assert _resolve_provider_alias("x-ai") == "xai", (
            "Expected _resolve_provider_alias('x-ai') == 'xai'"
        )

        fake_pm = {"xai": ["grok-99-test"]}
        with _patch_core_pm(fake_pm):
            _seed_provider_models_from_core()

        # The new model should be under "x-ai" (WebUI's key)
        x_hyphen_ai_ids = [m["id"] for m in _PROVIDER_MODELS.get("x-ai", [])]
        assert "grok-99-test" in x_hyphen_ai_ids, (
            "grok-99-test was not added to 'x-ai' — alias resolution failed"
        )

        # No duplicate "xai" key should have been created
        assert "xai" not in _PROVIDER_MODELS or all(
            m["id"] != "grok-99-test" for m in _PROVIDER_MODELS.get("xai", [])
        ), (
            "A duplicate 'xai' key was created with grok-99-test — "
            "provider alias was not resolved before .get()"
        )

    def test_unknown_provider_not_seeded(self, restore_providers):
        """A provider in core but not in WebUI must NOT be seeded.

        Adding new vendors is a maintainer curation decision, not something
        the seeder should do implicitly (#4413 review)."""
        fake_pm = {"totally-new-provider": ["model-alpha"]}
        with _patch_core_pm(fake_pm):
            _seed_provider_models_from_core()

        assert "totally-new-provider" not in _PROVIDER_MODELS, (
            "Seeder must not inject brand-new providers — only enrich existing ones. "
            "Adding vendors is a maintainer curation decision (#4413)."
        )


# ── Test 3: exception narrowing ─────────────────────────────────────────────

class TestExceptionNarrowing:
    """The seeder's hermes_cli import should only catch ImportError, not all
    exceptions (which would hide real bugs like the NameError)."""

    def test_does_not_swallow_runtime_errors(self, restore_providers):
        """If _get_label_for_model raises, the error should propagate (not be
        silently swallowed by a bare except)."""
        fake_pm = {"zai": ["boom-model"]}
        with _patch_core_pm(fake_pm):
            with mock.patch(
                "api.config._get_label_for_model",
                side_effect=RuntimeError("deliberate failure"),
            ):
                # The seeder itself should NOT catch RuntimeError — it only
                # catches ImportError for the optional hermes_cli import.
                with pytest.raises(RuntimeError, match="deliberate failure"):
                    _seed_provider_models_from_core()