From 1dd5cc43ce5080d46effbd79401def08d303bd15 Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Tue, 21 Jul 2026 00:00:52 -0700 Subject: [PATCH] fix(tests): isolate model-selection env vars so the suite passes on any runner (#6395) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suite failed 6 tests (test_sprint12 default-model, test_issue1538 Nous catalog, test_issue1567 picker capacity/symmetry) whenever it ran on a box that exports HERMES_MODEL (e.g. inside a live Hermes agent session where HERMES_MODEL is the agent's runtime model). Root cause: get_effective_default_model() in api/config.py treats HERMES_MODEL / OPENAI_MODEL / LLM_MODEL as the highest-priority default-model source — above the isolated test config — so a runner-exported value overrode the sandbox config and polluted the model picker/catalog (e.g. injecting "opus-4.8" and breaking the @nous: prefix invariant / default-model readback). Fix: scrub the three model-selection env vars at conftest module level (before any product module imports, covering in-process tests) and again in the out-of-process test_server subprocess env, and add them to the per-test _scrub_provider_env helpers in the two Nous test modules. The isolated config / HERMES_WEBUI_DEFAULT_MODEL is now authoritative regardless of the runner env. Verified: with HERMES_MODEL=opus-4.8 set, the full suite goes from 6 failed to 0 failed (13550 passed); non-vacuity confirmed (disabling the scrub reproduces the failures). Co-authored-by: nesquena-hermes --- tests/conftest.py | 21 +++++++++++++++++++ tests/test_issue1538_nous_live_catalog.py | 6 ++++++ ...e1567_nous_picker_capacity_and_symmetry.py | 5 +++++ 3 files changed, 32 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 223fb57b..d28b4099 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -181,6 +181,20 @@ os.environ['HERMES_BASE_HOME'] = str(TEST_STATE_DIR) # tests that read/write config.yaml stay inside the isolated test home. os.environ['HERMES_CONFIG_PATH'] = str(TEST_STATE_DIR / 'config.yaml') +# Model-selection env overrides must NOT leak from the runner into tests. +# get_effective_default_model() (api/config.py) treats HERMES_MODEL / OPENAI_MODEL +# / LLM_MODEL as the highest-priority default-model source — above the isolated +# test config. When the pytest process runs inside a live Hermes agent session, +# HERMES_MODEL is exported to the agent's runtime model (e.g. "opus-4.8"), which +# then overrides the sandbox config and pollutes the model picker/catalog. That +# broke test_sprint12 (default-model readback), test_issue1538 (Nous @nous: +# prefix invariant) and test_issue1567 (picker capacity/symmetry) whenever the +# suite ran on a box with HERMES_MODEL set. Strip them at module level so the +# isolated config is authoritative for the pytest process; the out-of-process +# test server env is scrubbed identically in the test_server fixture below. +for _model_env in ('HERMES_MODEL', 'OPENAI_MODEL', 'LLM_MODEL'): + os.environ.pop(_model_env, None) + @pytest.fixture(autouse=True) def _isolate_hermes_config_path(): @@ -987,6 +1001,13 @@ def test_server(): for _k in list(env): if any(_k.startswith(p) for p in _CRED_ENV_PREFIXES): del env[_k] + # Model-selection overrides must not reach the test server either. They are + # already popped from the pytest process env at module level, but strip them + # explicitly here so the subprocess default model comes only from the isolated + # config / HERMES_WEBUI_DEFAULT_MODEL below — never a runner-exported + # HERMES_MODEL (get_effective_default_model gives these env vars top priority). + for _model_env in ('HERMES_MODEL', 'OPENAI_MODEL', 'LLM_MODEL'): + env.pop(_model_env, None) # Belt-and-suspenders: keep IMDS disabled in the spawn env too (we set it # at module level above for the pytest process, but make it explicit here # so it's never accidentally cleared by an env.update later). diff --git a/tests/test_issue1538_nous_live_catalog.py b/tests/test_issue1538_nous_live_catalog.py index e2d3e403..73c0e8b8 100644 --- a/tests/test_issue1538_nous_live_catalog.py +++ b/tests/test_issue1538_nous_live_catalog.py @@ -129,6 +129,12 @@ def _scrub_provider_env(monkeypatch): "XIAOMI_API_KEY", "OPENCODE_ZEN_API_KEY", "OPENCODE_GO_API_KEY", "NOUS_API_KEY", "NVIDIA_API_KEY", "LM_API_KEY", "LMSTUDIO_API_KEY", + # Model-selection overrides: get_effective_default_model() treats these + # as the highest-priority default-model source, above config. A runner + # that exports HERMES_MODEL (e.g. a live agent session set to "opus-4.8") + # injects that id into the picker/catalog and breaks the @nous: prefix + # invariant below. Scrub so detection sees only the installed stubs. + "HERMES_MODEL", "OPENAI_MODEL", "LLM_MODEL", ): monkeypatch.delenv(var, raising=False) diff --git a/tests/test_issue1567_nous_picker_capacity_and_symmetry.py b/tests/test_issue1567_nous_picker_capacity_and_symmetry.py index 674c5957..4f30be2d 100644 --- a/tests/test_issue1567_nous_picker_capacity_and_symmetry.py +++ b/tests/test_issue1567_nous_picker_capacity_and_symmetry.py @@ -130,6 +130,11 @@ def _scrub_provider_env(monkeypatch): "XIAOMI_API_KEY", "OPENCODE_ZEN_API_KEY", "OPENCODE_GO_API_KEY", "NOUS_API_KEY", "NVIDIA_API_KEY", "LM_API_KEY", "LMSTUDIO_API_KEY", + # Model-selection overrides: get_effective_default_model() treats these + # as the highest-priority default-model source, above config. A runner + # exporting HERMES_MODEL (e.g. a live agent session on "opus-4.8") injects + # that id into the picker/catalog and breaks capacity/symmetry assertions. + "HERMES_MODEL", "OPENAI_MODEL", "LLM_MODEL", ): monkeypatch.delenv(var, raising=False)