nesquena-hermes
d025b37186
Two full-suite iterations pinned the real vector: set_hermes_default_model() calls
resolve_model_provider() which reads the module-global config.cfg (config.py:2488).
A prior lmstudio test leaves cfg={model:{provider:lmstudio,base_url:1234}} in
_cfg_cache; resolve_model_provider then returns lmstudio's provider+base_url, which
set_hermes_default_model persists to the test's OWN config file -> the on-disk
assertion sees 'provider: lmstudio' instead of the written 'provider: openai'.
The autouse fixture now _cfg_cache.CLEAR()s its CONTENTS (not just re-aliases cfg),
so resolve_model_provider reads an empty model cfg and can't inject a stale
provider/base_url. Deterministically proven: reproduced the exact lmstudio leak with
a probe test (fixture on=PASS; _cfg_cache.clear() line removed=the exact full-suite
failure reproduces). Also clears _yaml_file_cache + fingerprint/mtime for defense.
197 lines
8.3 KiB
Python
197 lines
8.3 KiB
Python
"""Tests for issue #4536 — main-model service_tier persistence and guarded forwarding."""
|
|
|
|
import pytest
|
|
|
|
|
|
class TestIssue4536ServiceTier:
|
|
@pytest.fixture(autouse=True)
|
|
def _isolate_config_globals(self):
|
|
"""Make these tests hermetic against leaked module-global config caches.
|
|
|
|
Two shared caches in api.config can serve a PRIOR test's config into this
|
|
one, causing a full-suite-only flake (passes in isolation):
|
|
|
|
1. `_yaml_file_cache` — memoized YAML parse keyed on
|
|
(str(config_path), st_mtime_ns, st_size). pytest reuses tmp_path base
|
|
dirs across the session, so a prior test's config.yaml can share both
|
|
the path string AND the (mtime_ns, size) of this test's tiny file
|
|
(same coarse mtime tick) → set_hermes_default_model() reads the STALE
|
|
cached dict (e.g. an lmstudio config) and writes THAT back, so the
|
|
on-disk assertion sees the wrong provider. This is the actual root
|
|
cause of the #4536 service_tier flake.
|
|
2. `cfg` / `_cfg_fingerprint` — if a prior test rebound `config.cfg`,
|
|
get_auxiliary_models() reads the stale dict after reload_config().
|
|
|
|
Clear both before AND after each test so neither an inherited leak nor
|
|
our own run can poison a neighbor.
|
|
"""
|
|
from api import config
|
|
|
|
def _reset():
|
|
with config._cfg_lock:
|
|
config._cfg_cache.clear()
|
|
config.cfg = config._cfg_cache
|
|
config._cfg_fingerprint = None
|
|
config._cfg_mtime = 0.0
|
|
with config._yaml_file_cache_lock:
|
|
config._yaml_file_cache.clear()
|
|
|
|
_reset()
|
|
yield
|
|
_reset()
|
|
|
|
def test_main_service_tier_roundtrip_via_auxiliary_endpoint(self, monkeypatch, tmp_path):
|
|
"""service_tier set on main model should persist in config and return via /api/model/auxiliary payload."""
|
|
from api import config
|
|
|
|
config_path = tmp_path / "config.yaml"
|
|
config_path.write_text(
|
|
"model:\n provider: openai\n default: gpt-5.5\n",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(config, "_get_config_path", lambda: config_path)
|
|
monkeypatch.setattr(config, "invalidate_models_cache", lambda: None)
|
|
|
|
result = config.set_hermes_default_model(
|
|
"gpt-5.5",
|
|
advanced={"service_tier": "priority"},
|
|
)
|
|
assert result["ok"] is True
|
|
assert "service_tier: priority" in config_path.read_text(encoding="utf-8")
|
|
|
|
payload = config.get_auxiliary_models()
|
|
assert payload["main"]["service_tier"] == "priority"
|
|
|
|
def test_main_service_tier_default_clears_persisted_value(self, monkeypatch, tmp_path):
|
|
"""Choosing Default/off should clear service_tier from persisted main-model options."""
|
|
from api import config
|
|
|
|
config_path = tmp_path / "config.yaml"
|
|
config_path.write_text(
|
|
"model:\n provider: openai\n default: gpt-5.5\n service_tier: priority\n",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(config, "_get_config_path", lambda: config_path)
|
|
monkeypatch.setattr(config, "invalidate_models_cache", lambda: None)
|
|
|
|
result = config.set_hermes_default_model(
|
|
"gpt-5.5",
|
|
advanced={"service_tier": "default"},
|
|
)
|
|
assert result["ok"] is True
|
|
payload = config.get_auxiliary_models()
|
|
assert payload["main"]["service_tier"] == ""
|
|
assert "service_tier" not in config_path.read_text(encoding="utf-8")
|
|
|
|
def test_main_request_overrides_only_for_openai_family(self, monkeypatch):
|
|
"""service_tier forwarding should only happen for OpenAI-family providers."""
|
|
from api import config
|
|
|
|
openai_payload = config._main_model_request_overrides(
|
|
{"model": {"provider": "openai", "default": "gpt-5.5", "service_tier": "priority"}},
|
|
)
|
|
assert openai_payload == {"service_tier": "priority"}
|
|
|
|
openrouter_payload = config._main_model_request_overrides(
|
|
{"model": {"provider": "openrouter", "default": "meta-llama/llama-3.1", "service_tier": "priority"}},
|
|
)
|
|
assert openrouter_payload == {}
|
|
|
|
def resolve_alias(model: str, *_args):
|
|
return model, "openai", ""
|
|
|
|
monkeypatch.setattr(config, "resolve_model_provider", resolve_alias)
|
|
|
|
openai_alias_payload = config._main_model_request_overrides(
|
|
{"model": {"default": "gpt-5.5", "service_tier": "priority"}},
|
|
)
|
|
assert openai_alias_payload == {"service_tier": "priority"}
|
|
|
|
codex_payload = config._main_model_request_overrides(
|
|
{"model": {"provider": "openai-codex", "default": "gpt-5.3-codex", "service_tier": "priority"}},
|
|
)
|
|
assert codex_payload == {}
|
|
|
|
codex_empty_model_payload = config._main_model_request_overrides(
|
|
{"model": {"provider": "openai-codex", "service_tier": "priority"}},
|
|
)
|
|
assert codex_empty_model_payload == {}
|
|
|
|
stale_openai_payload = config._main_model_request_overrides(
|
|
{"model": {"provider": "openai", "default": "meta-llama/llama-3.1", "service_tier": "priority"}},
|
|
)
|
|
assert stale_openai_payload == {}
|
|
|
|
def test_auxiliary_payload_hides_service_tier_for_non_openai_main_models(self, monkeypatch, tmp_path):
|
|
"""Saved service_tier should not be re-exposed once the main model switches away from OpenAI."""
|
|
from api import config
|
|
|
|
config_path = tmp_path / "config.yaml"
|
|
config_path.write_text(
|
|
"model:\n provider: openrouter\n default: meta-llama/llama-3.1\n service_tier: priority\n",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(config, "_get_config_path", lambda: config_path)
|
|
|
|
payload = config.get_auxiliary_models()
|
|
|
|
assert payload["main"]["service_tier"] == ""
|
|
|
|
def test_effective_model_gates_service_tier_forwarding(self):
|
|
"""service_tier must be gated on the effective send model, not the saved default."""
|
|
from api import config
|
|
|
|
cfg_priority = {"model": {"provider": "openai", "default": "gpt-5.5", "service_tier": "priority"}}
|
|
|
|
openai_effective = config._main_model_request_overrides(
|
|
cfg_priority, effective_model="gpt-5.5", effective_provider="openai",
|
|
)
|
|
assert openai_effective.get("service_tier") == "priority"
|
|
|
|
openrouter_effective = config._main_model_request_overrides(
|
|
cfg_priority, effective_model="meta-llama/llama-3.1", effective_provider="openrouter",
|
|
)
|
|
assert openrouter_effective == {}
|
|
|
|
custom_effective = config._main_model_request_overrides(
|
|
cfg_priority, effective_model="my-local-model", effective_provider="custom:local",
|
|
)
|
|
assert custom_effective == {}
|
|
|
|
codex_effective = config._main_model_request_overrides(
|
|
cfg_priority, effective_model="gpt-5.3-codex", effective_provider="openai-codex",
|
|
)
|
|
assert codex_effective == {}
|
|
|
|
def test_no_effective_params_falls_back_to_saved_default(self):
|
|
"""Without effective_model/provider, the function still reads the saved config."""
|
|
from api import config
|
|
|
|
result = config._main_model_request_overrides(
|
|
{"model": {"provider": "openai", "default": "gpt-5.5", "service_tier": "priority"}},
|
|
)
|
|
assert result == {"service_tier": "priority"}
|
|
|
|
def test_switching_main_model_away_from_openai_clears_service_tier(self, monkeypatch, tmp_path):
|
|
"""A non-OpenAI default-model save should remove stale OpenAI service-tier state."""
|
|
from api import config
|
|
|
|
config_path = tmp_path / "config.yaml"
|
|
config_path.write_text(
|
|
"model:\n provider: openai\n default: gpt-5.5\n service_tier: priority\n",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(config, "_get_config_path", lambda: config_path)
|
|
monkeypatch.setattr(config, "invalidate_models_cache", lambda: None)
|
|
monkeypatch.setattr(
|
|
config,
|
|
"cfg",
|
|
{"model": {"provider": "openai", "default": "gpt-5.5"}},
|
|
)
|
|
|
|
result = config.set_hermes_default_model("meta-llama/llama-3.1")
|
|
|
|
assert result["ok"] is True
|
|
text = config_path.read_text(encoding="utf-8")
|
|
assert "service_tier" not in text
|