501 lines
17 KiB
Python
501 lines
17 KiB
Python
"""Regression coverage for the /api/models rebuild-budget fallback."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
import time
|
|
|
|
import pytest
|
|
|
|
import api.config as cfg
|
|
import api.profiles as profiles
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def isolate_models_catalog_state(monkeypatch, tmp_path):
|
|
config_path = tmp_path / "config.yaml"
|
|
config_path.write_text("model: {}\n", encoding="utf-8")
|
|
auth_store_path = tmp_path / "auth.json"
|
|
auth_store_path.write_text("{}", encoding="utf-8")
|
|
hermes_home = tmp_path / "hermes-home"
|
|
hermes_home.mkdir()
|
|
(hermes_home / ".env").write_text("", encoding="utf-8")
|
|
|
|
monkeypatch.setattr(cfg, "_get_config_path", lambda: config_path)
|
|
monkeypatch.setattr(cfg, "_cfg_path", config_path, raising=False)
|
|
monkeypatch.setattr(cfg, "_cfg_mtime", config_path.stat().st_mtime, raising=False)
|
|
monkeypatch.setattr(cfg, "_cfg_has_in_memory_overrides", lambda: True)
|
|
monkeypatch.setattr(cfg, "_get_auth_store_path", lambda: auth_store_path)
|
|
monkeypatch.setattr(cfg, "_load_models_cache_from_disk", lambda: None)
|
|
monkeypatch.setattr(cfg, "_save_models_cache_to_disk", lambda *_a, **_k: None)
|
|
# Point the stale-cache loader's path at an isolated (by default nonexistent)
|
|
# temp file so the over-budget stale fallback (#3928 follow-up) stays
|
|
# hermetic — otherwise it would read the real ~/.../models_cache.json and
|
|
# tests asserting the static fallback become order/environment-dependent
|
|
# flakes. Tests exercising the stale path override _get_models_cache_path
|
|
# to write their own payload.
|
|
monkeypatch.setattr(cfg, "_get_models_cache_path", lambda: tmp_path / "models_cache.json")
|
|
monkeypatch.setattr(cfg, "_delete_models_cache_on_disk", lambda: None)
|
|
monkeypatch.setattr(
|
|
cfg,
|
|
"_models_cache_source_fingerprint",
|
|
lambda: "unit-test-fingerprint",
|
|
)
|
|
monkeypatch.setattr(cfg, "_available_models_cache", None, raising=False)
|
|
monkeypatch.setattr(cfg, "_available_models_cache_ts", 0.0, raising=False)
|
|
monkeypatch.setattr(
|
|
cfg,
|
|
"_available_models_cache_source_fingerprint",
|
|
None,
|
|
raising=False,
|
|
)
|
|
monkeypatch.setattr(cfg, "_cache_build_in_progress", False, raising=False)
|
|
monkeypatch.setattr(cfg, "cfg", {}, raising=False)
|
|
monkeypatch.setattr(profiles, "get_active_hermes_home", lambda: hermes_home)
|
|
monkeypatch.setattr(cfg.os, "getenv", lambda key, default=None: default or "")
|
|
|
|
return {
|
|
"auth_store_path": auth_store_path,
|
|
"models_cache_path": tmp_path / "models_cache.json",
|
|
}
|
|
|
|
|
|
def _configure_local_sources(
|
|
monkeypatch,
|
|
auth_store_path,
|
|
*,
|
|
env_vars: dict[str, str] | None = None,
|
|
):
|
|
cfg.cfg = {
|
|
"model": {
|
|
"provider": "openai-api",
|
|
"default": "gpt-5.5",
|
|
},
|
|
"fallback_providers": [
|
|
{"provider": "anthropic", "model": "claude-sonnet-4.6"},
|
|
],
|
|
"providers": {
|
|
"openai_api": {"api_key": "sk-openai"},
|
|
"anthropic": {"api_key": "sk-anthropic"},
|
|
},
|
|
"custom_providers": [
|
|
{
|
|
"name": "ZenMux",
|
|
"models": ["vendor/model-a", {"id": "vendor/model-b"}],
|
|
}
|
|
],
|
|
}
|
|
auth_store_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"credential_pool": {
|
|
"openrouter": [
|
|
{
|
|
"source": "env:openrouter_api_key",
|
|
"label": "manual key",
|
|
"key_source": "env",
|
|
}
|
|
],
|
|
"copilot": [
|
|
{
|
|
"source": "gh_cli",
|
|
"label": "gh auth token",
|
|
"key_source": "gh auth token",
|
|
}
|
|
],
|
|
}
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
env_lookup = dict(env_vars or {})
|
|
monkeypatch.setattr(cfg.os, "getenv", lambda key, default=None: env_lookup.get(key, default or ""))
|
|
|
|
|
|
def _build_stale_disk_cache_payload() -> dict:
|
|
return {
|
|
"active_provider": "ollama-cloud",
|
|
"default_model": "ollama-cloud/chat-1",
|
|
"configured_model_badges": {
|
|
"ollama-cloud/chat-1": {
|
|
"role": "primary",
|
|
"provider": "ollama-cloud",
|
|
"label": "Primary",
|
|
}
|
|
},
|
|
"groups": [
|
|
{
|
|
"provider": "Ollama Cloud",
|
|
"provider_id": "ollama-cloud",
|
|
"models": [
|
|
{"id": "ollama-cloud/chat-1", "label": "Chat 1"},
|
|
],
|
|
}
|
|
],
|
|
"aliases": {
|
|
"chat": "ollama-cloud/chat-1",
|
|
},
|
|
# Current schema and sources, but a stale _webui_version so the STRICT loader
|
|
# rejects it: the "recoverable stale cache" case. A source-fingerprint
|
|
# mismatch is a wrong catalog and is covered by the rejection test below.
|
|
"_schema_version": cfg._MODELS_CACHE_SCHEMA_VERSION,
|
|
"_webui_version": "v999",
|
|
"_source_fingerprint": cfg._models_cache_source_fingerprint(),
|
|
}
|
|
|
|
|
|
def test_static_catalog_budget_fallback_lists_local_providers(
|
|
monkeypatch,
|
|
isolate_models_catalog_state,
|
|
):
|
|
_configure_local_sources(
|
|
monkeypatch,
|
|
isolate_models_catalog_state["auth_store_path"],
|
|
)
|
|
monkeypatch.setattr(
|
|
cfg,
|
|
"_read_live_provider_model_ids",
|
|
lambda _provider_id: (_ for _ in ()).throw(
|
|
AssertionError("static timeout fallback must stay network-free")
|
|
),
|
|
)
|
|
|
|
catalog = cfg._static_models_catalog_without_live_probes()
|
|
provider_ids = [group["provider_id"] for group in catalog["groups"]]
|
|
|
|
assert provider_ids[0] == "openai-api"
|
|
assert "anthropic" in provider_ids
|
|
assert "custom:zenmux" in provider_ids
|
|
assert "copilot" not in provider_ids
|
|
assert not (
|
|
len(catalog["groups"]) == 1
|
|
and catalog["groups"][0]["provider"] == "Default"
|
|
)
|
|
assert any(
|
|
badge["role"] == "fallback" and badge["provider"] == "anthropic"
|
|
for badge in catalog["configured_model_badges"].values()
|
|
)
|
|
|
|
|
|
def test_budget_exceeded_foreground_uses_richer_static_catalog_and_refreshes_out_of_band(
|
|
monkeypatch,
|
|
isolate_models_catalog_state,
|
|
):
|
|
_configure_local_sources(
|
|
monkeypatch,
|
|
isolate_models_catalog_state["auth_store_path"],
|
|
)
|
|
monkeypatch.setattr(cfg, "_LIVE_REBUILD_BUDGET_SECONDS", 0.05, raising=False)
|
|
|
|
live_result = {
|
|
"active_provider": "openrouter",
|
|
"default_model": "openrouter/google/gemini-2.5-pro",
|
|
"configured_model_badges": {},
|
|
"groups": [
|
|
{
|
|
"provider": "OpenRouter",
|
|
"provider_id": "openrouter",
|
|
"models": [
|
|
{
|
|
"id": "openrouter/google/gemini-2.5-pro",
|
|
"label": "Gemini 2.5 Pro",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
"aliases": {},
|
|
}
|
|
rebuild_calls = {"count": 0}
|
|
|
|
def _slow_rebuild(_builder):
|
|
rebuild_calls["count"] += 1
|
|
time.sleep(0.2)
|
|
return copy.deepcopy(live_result)
|
|
|
|
monkeypatch.setattr(cfg, "_invoke_models_rebuild", _slow_rebuild)
|
|
|
|
expected_fallback = cfg._static_models_catalog_without_live_probes()
|
|
result = cfg.get_available_models()
|
|
|
|
assert result == expected_fallback
|
|
assert not (
|
|
len(result["groups"]) == 1 and result["groups"][0]["provider"] == "Default"
|
|
)
|
|
|
|
deadline = time.monotonic() + 0.5
|
|
while time.monotonic() < deadline:
|
|
if cfg._available_models_cache == live_result:
|
|
break
|
|
time.sleep(0.01)
|
|
|
|
assert rebuild_calls["count"] == 1
|
|
assert cfg._available_models_cache == live_result
|
|
assert cfg._cache_build_in_progress is False
|
|
|
|
|
|
def test_budget_exceeded_uses_shape_only_stale_cache_before_static_fallback(
|
|
monkeypatch,
|
|
isolate_models_catalog_state,
|
|
):
|
|
_configure_local_sources(
|
|
monkeypatch,
|
|
isolate_models_catalog_state["auth_store_path"],
|
|
)
|
|
monkeypatch.setattr(cfg, "_LIVE_REBUILD_BUDGET_SECONDS", 0.05, raising=False)
|
|
models_cache_path = isolate_models_catalog_state["models_cache_path"]
|
|
monkeypatch.setattr(cfg, "_get_models_cache_path", lambda: models_cache_path)
|
|
models_cache_path.write_text(
|
|
json.dumps(_build_stale_disk_cache_payload()),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
live_result = {
|
|
"active_provider": "openrouter",
|
|
"default_model": "openrouter/google/gemini-2.5-pro",
|
|
"configured_model_badges": {},
|
|
"groups": [
|
|
{
|
|
"provider": "OpenRouter",
|
|
"provider_id": "openrouter",
|
|
"models": [
|
|
{
|
|
"id": "openrouter/google/gemini-2.5-pro",
|
|
"label": "Gemini 2.5 Pro",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
"aliases": {},
|
|
}
|
|
rebuild_calls = {"count": 0}
|
|
|
|
def _slow_rebuild(_builder):
|
|
rebuild_calls["count"] += 1
|
|
time.sleep(0.2)
|
|
return copy.deepcopy(live_result)
|
|
|
|
monkeypatch.setattr(cfg, "_invoke_models_rebuild", _slow_rebuild)
|
|
|
|
expected_fallback = _build_stale_disk_cache_payload()
|
|
expected_fallback.pop("_schema_version")
|
|
expected_fallback.pop("_webui_version")
|
|
expected_fallback.pop("_source_fingerprint")
|
|
result = cfg.get_available_models()
|
|
|
|
assert any(
|
|
g["provider_id"] == "ollama-cloud" for g in result["groups"]
|
|
)
|
|
assert result == expected_fallback
|
|
assert "ollama-cloud" not in {
|
|
g["provider_id"] for g in cfg._static_models_catalog_without_live_probes()["groups"]
|
|
}
|
|
assert (
|
|
cfg._available_models_cache is None
|
|
or cfg._available_models_cache != expected_fallback
|
|
)
|
|
|
|
deadline = time.monotonic() + 3.0
|
|
while time.monotonic() < deadline:
|
|
if cfg._available_models_cache == live_result:
|
|
break
|
|
time.sleep(0.01)
|
|
|
|
assert rebuild_calls["count"] == 1
|
|
assert cfg._available_models_cache == live_result
|
|
assert cfg._cache_build_in_progress is False
|
|
|
|
|
|
def test_budget_exceeded_fallback_uses_static_when_stale_disk_cache_invalid(
|
|
monkeypatch,
|
|
isolate_models_catalog_state,
|
|
):
|
|
_configure_local_sources(
|
|
monkeypatch,
|
|
isolate_models_catalog_state["auth_store_path"],
|
|
)
|
|
monkeypatch.setattr(cfg, "_LIVE_REBUILD_BUDGET_SECONDS", 0.05, raising=False)
|
|
models_cache_path = isolate_models_catalog_state["models_cache_path"]
|
|
monkeypatch.setattr(cfg, "_get_models_cache_path", lambda: models_cache_path)
|
|
models_cache_path.write_text(
|
|
json.dumps({"active_provider": "openai-api", "default_model": "gpt-5.5"}),
|
|
encoding="utf-8",
|
|
)
|
|
assert cfg._load_stale_models_cache_from_disk() is None
|
|
|
|
live_result = {
|
|
"active_provider": "openrouter",
|
|
"default_model": "openrouter/google/gemini-2.5-pro",
|
|
"configured_model_badges": {},
|
|
"groups": [
|
|
{
|
|
"provider": "OpenRouter",
|
|
"provider_id": "openrouter",
|
|
"models": [
|
|
{
|
|
"id": "openrouter/google/gemini-2.5-pro",
|
|
"label": "Gemini 2.5 Pro",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
"aliases": {},
|
|
}
|
|
rebuild_calls = {"count": 0}
|
|
|
|
def _slow_rebuild(_builder):
|
|
rebuild_calls["count"] += 1
|
|
time.sleep(0.2)
|
|
return copy.deepcopy(live_result)
|
|
|
|
monkeypatch.setattr(cfg, "_invoke_models_rebuild", _slow_rebuild)
|
|
|
|
expected_static = cfg._static_models_catalog_without_live_probes()
|
|
result = cfg.get_available_models()
|
|
|
|
assert result == expected_static
|
|
assert all(g["provider_id"] != "ollama-cloud" for g in result["groups"])
|
|
|
|
deadline = time.monotonic() + 3.0
|
|
while time.monotonic() < deadline:
|
|
if cfg._available_models_cache == live_result:
|
|
break
|
|
time.sleep(0.01)
|
|
|
|
assert rebuild_calls["count"] == 1
|
|
assert cfg._available_models_cache == live_result
|
|
|
|
|
|
def test_load_stale_models_cache_from_disk_defaults_missing_aliases(
|
|
monkeypatch,
|
|
isolate_models_catalog_state,
|
|
):
|
|
payload = _build_stale_disk_cache_payload()
|
|
payload.pop("aliases")
|
|
models_cache_path = isolate_models_catalog_state["models_cache_path"]
|
|
monkeypatch.setattr(cfg, "_get_models_cache_path", lambda: models_cache_path)
|
|
models_cache_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
|
|
stale = cfg._load_stale_models_cache_from_disk()
|
|
assert stale is not None
|
|
assert stale["aliases"] == {}
|
|
|
|
|
|
def test_load_stale_models_cache_from_disk_rejects_cross_schema(
|
|
monkeypatch,
|
|
isolate_models_catalog_state,
|
|
):
|
|
"""A shape-valid but cross-schema cache must be rejected even on the stale
|
|
fallback path — its groups/badge shape may be incompatible with the current
|
|
picker and serving it could surface a broken catalog."""
|
|
payload = _build_stale_disk_cache_payload()
|
|
payload["_schema_version"] = cfg._MODELS_CACHE_SCHEMA_VERSION + 1
|
|
models_cache_path = isolate_models_catalog_state["models_cache_path"]
|
|
monkeypatch.setattr(cfg, "_get_models_cache_path", lambda: models_cache_path)
|
|
models_cache_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
|
|
assert cfg._load_stale_models_cache_from_disk() is None
|
|
|
|
|
|
def test_load_stale_models_cache_from_disk_rejects_source_fingerprint_mismatch(
|
|
monkeypatch,
|
|
isolate_models_catalog_state,
|
|
):
|
|
payload = _build_stale_disk_cache_payload()
|
|
payload["_source_fingerprint"] = "other-sources"
|
|
models_cache_path = isolate_models_catalog_state["models_cache_path"]
|
|
monkeypatch.setattr(cfg, "_get_models_cache_path", lambda: models_cache_path)
|
|
models_cache_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
|
|
assert cfg._load_stale_models_cache_from_disk() is None
|
|
|
|
|
|
def test_load_stale_models_cache_reconstructs_aliases_from_config(
|
|
monkeypatch,
|
|
isolate_models_catalog_state,
|
|
):
|
|
"""When the disk cache lacks aliases (the save path never persisted them),
|
|
the stale loader must reconstruct them from current config so /model <alias>
|
|
slash-command resolution keeps working during the over-budget fallback."""
|
|
payload = _build_stale_disk_cache_payload()
|
|
payload.pop("aliases", None)
|
|
models_cache_path = isolate_models_catalog_state["models_cache_path"]
|
|
monkeypatch.setattr(cfg, "_get_models_cache_path", lambda: models_cache_path)
|
|
models_cache_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
monkeypatch.setattr(
|
|
cfg, "cfg", {"model": {"aliases": {"fast": "ollama-cloud/chat-1"}}}, raising=False
|
|
)
|
|
|
|
stale = cfg._load_stale_models_cache_from_disk()
|
|
assert stale is not None
|
|
assert stale["aliases"] == {"fast": "ollama-cloud/chat-1"}
|
|
|
|
|
|
def test_default_group_survives_only_as_emergency_last_resort(
|
|
monkeypatch,
|
|
isolate_models_catalog_state,
|
|
):
|
|
cfg.cfg = {
|
|
"model": {
|
|
"provider": "anthropic",
|
|
"default": "claude-sonnet-4.6",
|
|
}
|
|
}
|
|
monkeypatch.setattr(
|
|
cfg,
|
|
"_get_providers_cfg",
|
|
lambda: (_ for _ in ()).throw(RuntimeError("boom")),
|
|
)
|
|
|
|
catalog = cfg._static_models_catalog_without_live_probes()
|
|
|
|
assert catalog["active_provider"] == "anthropic"
|
|
assert len(catalog["groups"]) == 1
|
|
assert catalog["groups"][0]["provider"] == "Default"
|
|
assert catalog["groups"][0]["provider_id"] == "anthropic"
|
|
assert catalog["groups"][0]["models"][0]["id"] == "claude-sonnet-4.6"
|
|
|
|
|
|
def test_provider_models_list_of_dicts_without_id_does_not_collapse_catalog(
|
|
monkeypatch,
|
|
isolate_models_catalog_state,
|
|
):
|
|
"""A legal ``providers.<id>.models`` list of dicts keyed by ``model``/``name``
|
|
(not ``id``) must still build the rich catalog instead of KeyError-ing into
|
|
the minimal one-model fallback."""
|
|
cfg.cfg = {
|
|
"model": {
|
|
"provider": "openai-api",
|
|
"default": "gpt-5.5",
|
|
},
|
|
"providers": {
|
|
"openai_api": {
|
|
"api_key": "***",
|
|
# list-of-dicts keyed by "model"/"name", and a bare string —
|
|
# all legal config shapes that the strict item["id"] path broke.
|
|
"models": [
|
|
{"model": "gpt-5.5", "label": "GPT-5.5"},
|
|
{"name": "gpt-4.1"},
|
|
"gpt-4o",
|
|
{"label": "no-id-no-model"}, # nothing usable → skipped, no crash
|
|
],
|
|
},
|
|
},
|
|
}
|
|
|
|
catalog = cfg._static_models_catalog_without_live_probes()
|
|
provider_ids = [group["provider_id"] for group in catalog["groups"]]
|
|
|
|
# Must NOT have collapsed to the emergency single "Default" group.
|
|
assert not (
|
|
len(catalog["groups"]) == 1
|
|
and catalog["groups"][0]["provider"] == "Default"
|
|
)
|
|
assert "openai-api" in provider_ids
|
|
openai_group = next(g for g in catalog["groups"] if g["provider_id"] == "openai-api")
|
|
group_model_ids = {str(m.get("id") or "") for m in openai_group["models"]}
|
|
# All three identifiable models survive; the unusable entry is dropped.
|
|
assert any(mid.endswith("gpt-5.5") for mid in group_model_ids)
|
|
assert any(mid.endswith("gpt-4.1") for mid in group_model_ids)
|
|
assert any(mid.endswith("gpt-4o") for mid in group_model_ids)
|