Merge pull request #5561 from nesquena/fix/suite-flakes

Fix the two chronic full-suite test flakes (nous_picker + service_tier)

tests/test_issue1567_nous_picker_capacity_and_symmetry.py

@@ -471,17 +471,27 @@ class TestNousLiveFetchEmpty:
                 "the curated static fallback so the picker isn't empty in "
                 "test envs that lack the agent package."
             )
-            # The original curated 4 entries must all be present.
-            nous_ids = {m["id"] for m in nous_groups[0]["models"]}
+            # Every curated @nous: entry must survive the static fallback — but
+            # when the curated list exceeds _NOUS_FEATURED_THRESHOLD (>25) the
+            # picker dropdown intentionally shows only the featured subset and
+            # stashes the rest under "extra_models" (for /model autocomplete).
+            # The invariant is "no curated entry is DROPPED", so check the full
+            # returned set (visible models + extra_models), not just the visible
+            # dropdown slice (which drifts to failure as the curated list grows
+            # past the threshold — the root cause of the pre-fix flake).
+            _nous_group = nous_groups[0]
+            nous_ids = {m["id"] for m in _nous_group.get("models", [])}
+            nous_ids |= {m["id"] for m in _nous_group.get("extra_models", [])}
             _curated_nous_ids = {
                 m["id"] for m in config._PROVIDER_MODELS.get("nous", [])
                 if m["id"].startswith("@nous:")
             }
             assert _curated_nous_ids.issubset(nous_ids), (
-                f"Static fallback is missing curated @nous: entries. "
-                f"Expected subset {_curated_nous_ids}, got {nous_ids}"
+                f"Static fallback dropped curated @nous: entries. "
+                f"Missing {_curated_nous_ids - nous_ids} "
+                f"(visible+extra_models total {len(nous_ids)})"
             )
-            assert len(nous_groups[0]["models"]) >= 4, (
+            assert len(nous_ids) >= 4, (
                 "Static fallback should expose at least the curated 4-entry "
                 "list from _PROVIDER_MODELS['nous']."
             )

tests/test_issue4536_service_tier.py

@@ -1,7 +1,45 @@
 """Tests for issue #4536 — main-model service_tier persistence and guarded forwarding."""
 
+import pytest
+
 
 class TestIssue4536ServiceTier:
+    @pytest.fixture(autouse=True)
+    def _isolate_config_globals(self):
+        """Make these tests hermetic against leaked module-global config caches.
+
+        Two shared caches in api.config can serve a PRIOR test's config into this
+        one, causing a full-suite-only flake (passes in isolation):
+
+        1. `_yaml_file_cache` — memoized YAML parse keyed on
+           (str(config_path), st_mtime_ns, st_size). pytest reuses tmp_path base
+           dirs across the session, so a prior test's config.yaml can share both
+           the path string AND the (mtime_ns, size) of this test's tiny file
+           (same coarse mtime tick) → set_hermes_default_model() reads the STALE
+           cached dict (e.g. an lmstudio config) and writes THAT back, so the
+           on-disk assertion sees the wrong provider. This is the actual root
+           cause of the #4536 service_tier flake.
+        2. `cfg` / `_cfg_fingerprint` — if a prior test rebound `config.cfg`,
+           get_auxiliary_models() reads the stale dict after reload_config().
+
+        Clear both before AND after each test so neither an inherited leak nor
+        our own run can poison a neighbor.
+        """
+        from api import config
+
+        def _reset():
+            with config._cfg_lock:
+                config._cfg_cache.clear()
+                config.cfg = config._cfg_cache
+                config._cfg_fingerprint = None
+                config._cfg_mtime = 0.0
+            with config._yaml_file_cache_lock:
+                config._yaml_file_cache.clear()
+
+        _reset()
+        yield
+        _reset()
+
     def test_main_service_tier_roundtrip_via_auxiliary_endpoint(self, monkeypatch, tmp_path):
         """service_tier set on main model should persist in config and return via /api/model/auxiliary payload."""
         from api import config