Merge PR #7632 into stage/0924-k

api/config.py

@@ -576,6 +576,7 @@ def _refresh_config_cache(config_path: Path | None = None) -> None:
     # Remember the old mtime so we can tell whether config actually changed
     # vs. first-ever load (mtime == 0.0, e.g. server start or profile switch).
     _old_cfg_mtime = _cfg_mtime
+    _old_cfg_path = _cfg_path
     _cfg_path = config_path
     _cfg_mtime = 0.0
     try:
@@ -632,10 +633,11 @@ def _refresh_config_cache(config_path: Path | None = None) -> None:
     _cfg_fingerprint = _fingerprint_config(_cfg_cache)
     # Bust the models cache so the next request sees fresh config values.
     # Only delete the disk cache when config has actually changed -- not on
-    # first-ever load (when _old_cfg_mtime == 0.0, i.e. server start or
-    # profile switch) -- preserving the disk cache so the next restart
+    # first-ever load (when _old_cfg_mtime == 0.0, i.e. server start) and not
+    # on a path change (per-client profile switch leaves the old profile's
+    # mtime) -- preserving the disk cache so the next restart
     # still hits the fast path without a cold run.
-    if _old_cfg_mtime != 0.0:
+    if _old_cfg_mtime != 0.0 and _old_cfg_path == config_path:
         _delete_models_cache_on_disk()
 
 
@@ -7606,7 +7608,7 @@ _MODELS_CACHE_SCHEMA_VERSION = 3
 _models_cache_path = STATE_DIR / "models_cache.json"
 
 
-def _get_models_cache_path() -> Path:
+def _get_models_cache_path(profile: str | None = None) -> Path:
     """Return the /api/models disk-cache path for the *active* profile (#3957).
 
     WebUI profile switching is per-client/cookie scoped (issue #798), but the
@@ -7630,12 +7632,13 @@ def _get_models_cache_path() -> Path:
     The named-profile path is derived from ``_models_cache_path`` (the
     module-level default), not from ``STATE_DIR`` directly, so the path stays
     correct if the default is repointed (e.g. tests monkeypatch
-    ``_models_cache_path`` to an isolated tmp file).
+    ``_models_cache_path`` to an isolated tmp file). Pass *profile* to get
+    another profile's path (profile delete/create).
     """
     try:
         from api.profiles import get_active_profile_name, _is_root_profile
 
-        name = (get_active_profile_name() or "").strip()
+        name = (profile or get_active_profile_name() or "").strip()
         if not name or _is_root_profile(name):
             return _models_cache_path
         # Defensive filename sanitization: the cookie-derived profile name is
@@ -7922,6 +7925,96 @@ def _auth_store_semantic_fingerprint(path: Path) -> dict:
     return fp
 
 
+def _active_profile_home() -> Path:
+    try:
+        from api.profiles import get_active_hermes_home as _gah
+
+        return _gah()
+    except ImportError:
+        return _DEFAULT_HERMES_HOME
+
+
+def _models_cache_env_fingerprint(path: Path) -> list:
+    """``[key, HMAC(value)]`` per non-empty ``.env`` entry, parsed like provider detection.
+
+    Values are keyed-hashed with the WebUI signing key so the cache file never holds a secret.
+    """
+    import hmac
+    from api.auth import _signing_key
+    from api.providers import _load_env_file
+
+    key = _signing_key()
+    return [
+        [k, hmac.new(key, v.encode("utf-8"), hashlib.sha256).hexdigest()]
+        for k, v in sorted(_load_env_file(Path(path).expanduser()).items())
+        if v
+    ]
+
+
+def _declares_model_provider_kind(plugin_dir: Path) -> bool:
+    # Same parse as the agent's providers._declares_model_provider_kind: PyYAML, then a line scan.
+    for filename in ("plugin.yaml", "plugin.yml"):
+        manifest = plugin_dir / filename
+        if not manifest.is_file():
+            continue
+        try:
+            text = manifest.read_text(encoding="utf-8", errors="replace")
+        except Exception:
+            return False
+        try:
+            import yaml as _yaml
+
+            data = _yaml.safe_load(text)
+            if isinstance(data, dict):
+                return str(data.get("kind", "")).strip() == "model-provider"
+        except Exception:
+            pass
+        for line in text.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("#") or ":" not in stripped:
+                continue
+            key, _, value = stripped.partition(":")
+            if key.strip() == "kind":
+                return value.strip().strip("\"'") == "model-provider"
+        return False
+    return False
+
+
+def _models_cache_plugin_fingerprint(home: Path) -> list:
+    """``[dir, file stamps]`` per model-provider plugin, discovered like providers._scan_home_layer."""
+    found = []
+    plugins_root = Path(home).expanduser() / "plugins"
+    for base, flat in ((plugins_root / "model-providers", False), (plugins_root, True)):
+        try:
+            children = sorted(base.iterdir())
+        except OSError:
+            continue
+        for child in children:
+            if not child.is_dir() or child.name.startswith(("_", ".")):
+                continue
+            if flat and (child.name == "model-providers" or not _declares_model_provider_kind(child)):
+                continue
+            found.append([str(child.relative_to(plugins_root)), _plugin_tree_stamps(child)])
+    return found
+
+
+def _plugin_tree_stamps(plugin_dir: Path) -> list:
+    # The loader execs __init__.py, which may import siblings or read data files; skip bytecode.
+    stamps = []
+    for root, dirs, files in os.walk(plugin_dir):
+        dirs[:] = sorted(d for d in dirs if d != "__pycache__" and not d.startswith("."))
+        for name in sorted(files):
+            if name.endswith((".pyc", ".pyo")):
+                continue
+            path = os.path.join(root, name)
+            try:
+                st = os.stat(path)
+            except OSError:
+                continue
+            stamps.append([os.path.relpath(path, plugin_dir), st.st_mtime_ns, st.st_size])
+    return stamps
+
+
 def _models_cache_source_fingerprint() -> dict:
     """Return the current config/auth/catalog fingerprint for /api/models cache.
 
@@ -7933,9 +8026,12 @@ def _models_cache_source_fingerprint() -> dict:
     mtime/size fingerprint because it is only rewritten on deliberate user
     edits (which can change anything) and does not churn on a timer.
     """
+    home = _active_profile_home()
     return {
         "config_yaml": _models_cache_file_fingerprint(_get_config_path()),
         "auth_json": _auth_store_semantic_fingerprint(_get_auth_store_path()),
+        "env": _models_cache_env_fingerprint(home / ".env"),
+        "plugins": _models_cache_plugin_fingerprint(home),
         "catalog": _models_cache_catalog_fingerprint(),
     }
 
@@ -8089,10 +8185,10 @@ def _load_stale_models_cache_from_disk() -> dict | None:
     The main cache loader enforces metadata stamps for a full cold-path cache hit.
     This helper intentionally does not apply that stricter policy, so we can still
     recover a useful fallback payload when the strict loader rejected cache because
-    metadata or fingerprint fields are stale. It DOES still enforce the schema
-    version: a cross-schema cache can have an incompatible groups/badge shape, so
-    serving it to the picker could surface a broken catalog — schema mismatch is a
-    hard reject even on the fallback path.
+    the WebUI version stamp is stale. It DOES still enforce the schema version (a
+    cross-schema cache can have an incompatible groups/badge shape) and the source
+    fingerprint: a snapshot built from other config/auth/.env/plugin sources is a
+    wrong catalog, not merely an old one, so it is never served.
     """
     try:
         import json as _j
@@ -8106,6 +8202,8 @@ def _load_stale_models_cache_from_disk() -> dict | None:
             return None
         if cache.get("_schema_version") != _MODELS_CACHE_SCHEMA_VERSION:
             return None
+        if cache.get("_source_fingerprint") != _models_cache_source_fingerprint():
+            return None
         aliases = cache.get("aliases")
         if not isinstance(aliases, dict):
             # The disk cache save path does not persist `aliases`, so a cache
@@ -8196,7 +8294,7 @@ def _get_fresh_memory_models_cache(now: float) -> dict | None:
     return None
 
 
-def invalidate_models_cache():
+def invalidate_models_cache(*, delete_disk: bool = True):
     """Force the TTL cache for get_available_models() to be cleared.
 
     Call this after modifying config.cfg in-memory (e.g. in tests) so
@@ -8209,6 +8307,8 @@ def invalidate_models_cache():
     that call invalidate_models_cache() still get back the previous test's
     result from the disk cache because the disk hit is checked before the memory
     cache rebuild runs.
+
+    ``delete_disk=False`` keeps the fingerprint-guarded disk snapshot (profile switch).
     """
     global _cache_build_in_progress, _available_models_cache, _available_models_cache_ts
     global _available_models_live_rebuild_ts, _available_models_cache_source_fingerprint, _cache_build_cv
@@ -8227,6 +8327,7 @@ def invalidate_models_cache():
         _CREDENTIAL_POOL_CACHE.clear()
     # Also delete the disk cache so the next cold build starts fresh.
     # Disk delete is outside the lock — file I/O shouldn't block other readers.
+    if delete_disk:
         _delete_models_cache_on_disk()
     try:
         from api.plugin_providers import invalidate_plugin_model_provider_cache

api/profiles.py

@@ -2708,6 +2708,7 @@ def create_profile_api(name: str, clone_from: str = None,
         model_provider=model_provider,
     )
 
+    _drop_profile_models_cache(name)
     # Invalidate cached root-profile-name lookup; create_profile may have added
     # a new profile that flips is_default semantics on the agent side (#1612).
     _SKILLS_STATS_CACHE.clear()
@@ -2736,6 +2737,16 @@ def create_profile_api(name: str, clone_from: str = None,
     }
 
 
+def _drop_profile_models_cache(name: str) -> None:
+    """A deleted or new profile must never inherit a same-name models snapshot."""
+    from api.config import _get_models_cache_path
+
+    try:
+        _get_models_cache_path(name).unlink(missing_ok=True)
+    except OSError:
+        logger.debug("Failed to drop models cache for profile %s", name, exc_info=True)
+
+
 def delete_profile_api(name: str) -> dict:
     """Delete a profile. Switches to default first if it's the active one.
 
@@ -2769,6 +2780,7 @@ def delete_profile_api(name: str) -> dict:
         else:
             raise ValueError(f"Profile '{name}' does not exist.")
 
+    _drop_profile_models_cache(name)
     # Drop cached root-profile-name lookup — list_profiles_api() shape changed.
     _SKILLS_STATS_CACHE.clear()
     _invalidate_list_profiles_cache()

api/routes.py

@@ -17155,8 +17155,9 @@ def handle_post(handler, parsed) -> bool:
             # Invalidate the models cache so the very next /api/models request
             # rebuilds from the new profile's config.yaml rather than returning
             # the old profile's cached model list (#1200 — profile-switch model bug).
+            # The per-profile disk snapshot is fingerprint-guarded, so keep it.
             from api.config import invalidate_models_cache
-            invalidate_models_cache()
+            invalidate_models_cache(delete_disk=False)
             try:
                 from api.gateway_watcher import restart_watcher_for_profile
                 restart_watcher_for_profile(name)

docs/architecture/models-cache-invalidation.md

@@ -20,7 +20,7 @@ which the #7556 review flagged as an undocumented runtime contract.
   `_sync_models_cache_provenance()` must run at every site that publishes or
   invalidates the snapshot, so the tuple can never tear.
 
-## The three source axes
+## The five source axes
 
 `_models_cache_source_fingerprint()` is the single chokepoint. A cache is served
 only when every axis matches the value recorded in the cache — the disk reader
@@ -31,8 +31,13 @@ the fingerprint captured at publish time.
 | --- | --- | --- |
 | `config_yaml` | stat identity: `mtime_ns` + size (`_models_cache_file_fingerprint`) | The file is rewritten only on deliberate user edits, and any edit can change the provider/model set, so the cheap conservative identity wins. |
 | `auth_json` | content hash with a volatile-key deny-list (`_auth_store_semantic_fingerprint`, `_AUTH_FINGERPRINT_VOLATILE_KEYS`) | The credential store is rewritten roughly every 14 minutes by credential-pool / OAuth refresh; none of those rotating fields feed `detected_providers` or the returned catalog, and stat identity made the 24h cache churn on every refresh (RCA `t_d127953d` / `t_16551f61`). |
+| `env` | `[key, HMAC-SHA256(signing key, value)]` per non-empty `.env` entry, parsed by `providers._load_env_file` (`_models_cache_env_fingerprint`); plaintext values are never recorded | Env keys decide `detected_providers` and values such as `LM_BASE_URL` decide which endpoint is probed, so both key and value changes invalidate. |
+| `plugins` | `[relpath, mtime_ns, size]` of every non-bytecode file of each model-provider plugin, discovered like `providers._scan_home_layer`, flat manifests parsed with PyYAML like the agent (`_models_cache_plugin_fingerprint`) | The loader execs plugin code that builds `fallback_models`, so any file edit must invalidate, even without a `version` bump. |
 | `catalog` | baked-in provider catalog sha256 (`_PROVIDER_MODELS` + `_PROVIDER_DISPLAY`) plus the Codex local catalog (`_codex_models_cache_fingerprint`, `_CODEX_CACHE_FINGERPRINT_VOLATILE_KEYS`) | A restart after a catalog change must not keep serving a persisted payload for up to 24h (#2443). Codex rewrites `~/.codex/models_cache.json` on its own timer, bumping `mtime_ns` and size while models, `etag`, and `client_version` stay identical, so the Codex axis hashes **content** with only the refresh timestamps (`fetched_at`, `updated_at`) removed (#7540, #7556). |
 
+The over-budget stale fallback (`_load_stale_models_cache_from_disk`) tolerates a stale
+`_webui_version` but never a source-fingerprint mismatch: such a snapshot is a wrong catalog.
+
 ## Invariant: deny-lists are one-directional
 
 - Both volatile-key sets are deny-lists, never allow-lists. They may remove only
@@ -66,6 +71,22 @@ the fingerprint captured at publish time.
   TTL expires. When the runtime version cannot be resolved (early boot), that
   check is skipped rather than wedging the boot.
 
+## Invalidation paths: memory vs. disk
+
+| Path | In-memory snapshot | Disk snapshot |
+| --- | --- | --- |
+| `invalidate_models_cache()` (default `delete_disk=True`) | dropped | **deleted** |
+| `invalidate_models_cache(delete_disk=False)` (`POST /api/profile/switch`) | dropped | kept |
+| `invalidate_provider_models_cache(provider_id)` | dropped | **deleted** |
+| `_get_fresh_memory_models_cache()` on fingerprint mismatch / invalid shape | dropped | untouched |
+| config-reload branch in `get_available_models()` | dropped | deleted by `_refresh_config_cache()` only when the *same* `config.yaml` path was already loaded and changed; a first load or path change (per-client switch) keeps it |
+
+The switch keeps the disk snapshot because it is keyed per profile and
+`_is_loadable_disk_cache()` rejects it unless every axis above matches, so any
+new catalog input must become an axis first. `delete_profile_api()` and
+`create_profile_api()` unlink `models_cache.<name>.json`, so a recreated profile
+never inherits the old catalog.
+
 ## Change protocol
 
 1. Add or change a source axis in `_models_cache_source_fingerprint()` only —
@@ -90,6 +111,11 @@ timestamp-only churn keeps the fingerprint identical (and a session visit after
 Codex refresh needs no live rebuild), while genuine changes — a new model, a
 visibility change, any catalog field, any unknown field — still invalidate.
 
+`tests/test_profile_switch_models_disk_cache.py` covers the switch: the disk
+snapshot survives it and is served without a live rebuild, each source-axis
+change or a delete/recreate forces a fresh rebuild, and a same-path config
+edit still deletes the snapshot.
+
 ## References
 
 Issues/PRs: #2443, #7540, #7556, #7558. RCAs: `t_d127953d`, `t_16551f61`.

tests/test_gateway_watcher_profile.py

@@ -316,7 +316,7 @@ def test_profile_switch_restarts_watcher_best_effort(monkeypatch):
     monkeypatch.setattr(routes, "read_body", lambda handler: {"name": "demo"})
     monkeypatch.setattr(profiles, "_validate_profile_name", lambda name: None)
     monkeypatch.setattr(profiles, "switch_profile", lambda name, process_wide=False: {"ok": True, "name": name})
-    monkeypatch.setattr(config, "invalidate_models_cache", lambda: calls.append("cache"))
+    monkeypatch.setattr(config, "invalidate_models_cache", lambda **kw: calls.append(("cache", kw)))
     monkeypatch.setattr(gateway_watcher, "restart_watcher_for_profile", lambda name: calls.append(("watcher", name)))
 
     handler = _FakeHandler()
@@ -324,7 +324,7 @@ def test_profile_switch_restarts_watcher_best_effort(monkeypatch):
 
     assert handler.status == 200
     assert handler.get_json() == {"ok": True, "name": "demo"}
-    assert calls == ["cache", ("watcher", "demo")]
+    assert calls == [("cache", {"delete_disk": False}), ("watcher", "demo")]
     assert any(k == "Set-Cookie" for k, _v in handler.sent_headers)
 
 
@@ -335,7 +335,7 @@ def test_profile_switch_response_survives_watcher_restart_failure(monkeypatch):
     monkeypatch.setattr(routes, "read_body", lambda handler: {"name": "demo"})
     monkeypatch.setattr(profiles, "_validate_profile_name", lambda name: None)
     monkeypatch.setattr(profiles, "switch_profile", lambda name, process_wide=False: {"ok": True, "name": name})
-    monkeypatch.setattr(config, "invalidate_models_cache", lambda: None)
+    monkeypatch.setattr(config, "invalidate_models_cache", lambda *_a, **_kw: None)
     monkeypatch.setattr(
         gateway_watcher,
         "restart_watcher_for_profile",

tests/test_issue3928_models_budget_fallback.py

@@ -136,16 +136,12 @@ def _build_stale_disk_cache_payload() -> dict:
         "aliases": {
             "chat": "ollama-cloud/chat-1",
         },
-        # Current schema (so the stale-cache loader's schema guard accepts it),
-        # but a deliberately stale _webui_version + fingerprint so the STRICT
-        # loader rejects it — this is exactly the "recoverable stale cache" case.
+        # Current schema and sources, but a stale _webui_version so the STRICT loader
+        # rejects it: the "recoverable stale cache" case. A source-fingerprint
+        # mismatch is a wrong catalog and is covered by the rejection test below.
         "_schema_version": cfg._MODELS_CACHE_SCHEMA_VERSION,
         "_webui_version": "v999",
-        "_source_fingerprint": {
-            "catalog": "stale",
-            "config_yaml": {"size": 42},
-            "auth_json": {"size": 7},
-        },
+        "_source_fingerprint": cfg._models_cache_source_fingerprint(),
     }
 
 
@@ -401,6 +397,19 @@ def test_load_stale_models_cache_from_disk_rejects_cross_schema(
     assert cfg._load_stale_models_cache_from_disk() is None
 
 
+def test_load_stale_models_cache_from_disk_rejects_source_fingerprint_mismatch(
+    monkeypatch,
+    isolate_models_catalog_state,
+):
+    payload = _build_stale_disk_cache_payload()
+    payload["_source_fingerprint"] = "other-sources"
+    models_cache_path = isolate_models_catalog_state["models_cache_path"]
+    monkeypatch.setattr(cfg, "_get_models_cache_path", lambda: models_cache_path)
+    models_cache_path.write_text(json.dumps(payload), encoding="utf-8")
+
+    assert cfg._load_stale_models_cache_from_disk() is None
+
+
 def test_load_stale_models_cache_reconstructs_aliases_from_config(
     monkeypatch,
     isolate_models_catalog_state,

tests/test_profile_switch_models_disk_cache.py

@@ -0,0 +1,232 @@
+"""/api/profile/switch keeps the target profile's models disk snapshot only while its sources are unchanged."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+import threading
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+import api.config as cfg
+import api.profiles as profiles
+
+
+def _catalog(label: str) -> dict:
+    group = {"provider": "OpenAI", "provider_id": "openai", "models": [{"id": label, "label": label, "supports_fast_tier": False}]}
+    return {"active_provider": "openai", "default_model": label, "configured_model_badges": {}, "groups": [group], "aliases": {}}
+
+
+@pytest.fixture
+def env(tmp_path: Path, monkeypatch):
+    monkeypatch.delenv("HERMES_CONFIG_PATH", raising=False)
+    default_home = tmp_path / ".hermes"
+    demo_home = default_home / "profiles" / "demo"
+    demo_home.mkdir(parents=True)
+    (default_home / "config.yaml").write_text("model:\n  default: default-model\n", encoding="utf-8")
+    _write_demo_config(demo_home)
+    monkeypatch.setattr(profiles, "_DEFAULT_HERMES_HOME", default_home)
+    monkeypatch.setattr(profiles, "_active_profile", "default")
+    monkeypatch.setattr(profiles, "_is_isolated_profile_mode", lambda: False)
+    monkeypatch.setattr(profiles._tls, "profile", None, raising=False)
+    monkeypatch.setattr(cfg, "_models_cache_path", tmp_path / "models_cache.json")
+    monkeypatch.setattr(cfg, "_invoke_models_rebuild", lambda _builder: _catalog("fresh-model"))
+    for attr in ("_cfg_mtime", "_cfg_path", "_cfg_fingerprint"):
+        monkeypatch.setattr(cfg, attr, getattr(cfg, attr), raising=False)
+    saved_cfg = dict(cfg._cfg_cache)
+    cfg.invalidate_models_cache(delete_disk=False)
+    cfg.reload_config()  # the default profile's config is loaded, as at server boot
+    yield SimpleNamespace(home=demo_home, cache=tmp_path / "models_cache.demo.json")
+    profiles.clear_request_profile()
+    cfg.invalidate_models_cache(delete_disk=False)
+    cfg._cfg_cache.clear()
+    cfg._cfg_cache.update(saved_cfg)
+
+
+def _write_demo_config(home: Path, default: str = "demo-model", mtime: int = 1_000_000) -> None:
+    (home / "config.yaml").write_text(f"model:\n  default: {default}\n", encoding="utf-8")
+    os.utime(home / "config.yaml", (mtime, mtime))
+
+
+def _save_demo_snapshot(env) -> None:
+    profiles.set_request_profile("demo")
+    try:
+        cfg._save_models_cache_to_disk(_catalog("snapshot-model"))
+    finally:
+        profiles.clear_request_profile()
+    assert env.cache.exists()
+
+
+def _fetch_as_demo() -> dict:
+    profiles.set_request_profile("demo")
+    try:
+        return cfg.get_available_models()
+    finally:
+        profiles.clear_request_profile()
+
+
+def _switch_to_demo_and_fetch() -> str:
+    profiles.switch_profile("demo", process_wide=False)
+    cfg.invalidate_models_cache(delete_disk=False)  # as /api/profile/switch does
+    return _fetch_as_demo()["default_model"]
+
+
+def _write_plugin(home: Path, version: str = "1.0.0", *, flat: bool = False, code: str | None = None, sub: str | None = None, kind: str = "model-provider") -> Path:
+    d = home / "plugins" / ("" if flat else "model-providers") / "acme"
+    d.mkdir(parents=True, exist_ok=True)
+    (d / "plugin.yaml").write_text(f"name: acme\nkind: {kind}\nversion: {version}\n", encoding="utf-8")
+    for name, text in (("__init__.py", code), ("models.py", sub)):
+        if text is not None:
+            (d / name).write_text(text, encoding="utf-8")
+            os.utime(d / name, (1_000_000, 1_000_000))
+    return d
+
+
+def _env(text: str):
+    return lambda home: (home / ".env").write_text(text, encoding="utf-8")
+
+
+# (id, source state before the snapshot is saved, change made after it, model served after the switch)
+_CASES = [
+    ("unchanged", None, None, "snapshot-model"),
+    ("env-value-rotated", _env("DEEPSEEK_API_KEY=sk-one\n"), _env("DEEPSEEK_API_KEY=sk-two\n"), "fresh-model"),
+    ("env-base-url-changed", _env("LM_BASE_URL=http://127.0.0.1:1234/v1\n"), _env("LM_BASE_URL=http://127.0.0.1:5678/v1\n"), "fresh-model"),
+    ("env-key-added", None, _env("DEEPSEEK_API_KEY=sk-new\n"), "fresh-model"),
+    ("env-key-removed", _env("DEEPSEEK_API_KEY=sk-old\nOTHER=1\n"), _env("OTHER=1\n"), "fresh-model"),
+    # provider detection reads `export KEY=` as key `export KEY`, so the credential is gone
+    ("env-export-prefix", _env("DEEPSEEK_API_KEY=sk-old\n"), _env("export DEEPSEEK_API_KEY=sk-old\n"), "fresh-model"),
+    ("plugin-installed", None, lambda h: _write_plugin(h), "fresh-model"),
+    ("plugin-installed-flat", None, lambda h: _write_plugin(h, flat=True), "fresh-model"),
+    ("plugin-installed-flat-yaml-comment", None, lambda h: _write_plugin(h, flat=True, kind="model-provider # valid YAML comment"), "fresh-model"),
+    ("plugin-removed", lambda h: _write_plugin(h), lambda h: shutil.rmtree(h / "plugins"), "fresh-model"),
+    ("plugin-version-bumped", lambda h: _write_plugin(h, flat=True), lambda h: _write_plugin(h, "2.0.0", flat=True), "fresh-model"),
+    ("plugin-code-edited", lambda h: _write_plugin(h, code="M = (1,)\n"), lambda h: (h / "plugins/model-providers/acme/__init__.py").write_text("M = (1, 2)\n"), "fresh-model"),
+    ("plugin-code-edited-flat", lambda h: _write_plugin(h, flat=True, code="M = (1,)\n"), lambda h: (h / "plugins/acme/__init__.py").write_text("M = (1, 2)\n"), "fresh-model"),
+    ("plugin-submodule-edited", lambda h: _write_plugin(h, code="from .models import M\n", sub="M = (1,)\n"), lambda h: (h / "plugins/model-providers/acme/models.py").write_text("M = (1, 2)\n"), "fresh-model"),
+]
+
+
+@pytest.mark.parametrize("before, change, expected", [c[1:] for c in _CASES], ids=[c[0] for c in _CASES])
+def test_switch_serves_snapshot_only_while_sources_are_unchanged(env, before, change, expected):
+    if before:
+        before(env.home)
+    _save_demo_snapshot(env)
+    if change:
+        change(env.home)
+    assert _switch_to_demo_and_fetch() == expected
+    assert "sk-" not in env.cache.read_text(encoding="utf-8")  # the fingerprint never records a secret
+
+
+@pytest.mark.parametrize("change", [_env("LM_BASE_URL=http://127.0.0.1:5678/v1\n"), lambda h: _write_plugin(h, flat=True)], ids=["env", "plugin"])
+def test_slow_rebuild_never_serves_fingerprint_mismatched_snapshot_as_stale_fallback(env, monkeypatch, change):
+    _env("LM_BASE_URL=http://127.0.0.1:1234/v1\n")(env.home)
+    _save_demo_snapshot(env)
+    change(env.home)
+    profiles.set_request_profile("demo")
+    try:
+        assert cfg._load_models_cache_from_disk() is None
+        assert cfg._load_stale_models_cache_from_disk() is None
+    finally:
+        profiles.clear_request_profile()
+    release = threading.Event()
+    monkeypatch.setattr(cfg, "_LIVE_REBUILD_BUDGET_SECONDS", 0.05, raising=False)
+    monkeypatch.setattr(cfg, "_invoke_models_rebuild", lambda _b: release.wait(5) and _catalog("fresh-model"))
+    try:
+        assert _switch_to_demo_and_fetch() != "snapshot-model"
+    finally:
+        release.set()
+
+
+def test_stale_fallback_still_serves_snapshot_when_only_webui_version_differs(env, monkeypatch):
+    _save_demo_snapshot(env)
+    monkeypatch.setattr(cfg, "_current_webui_version", lambda: "v-other")
+    profiles.set_request_profile("demo")
+    try:
+        assert cfg._load_models_cache_from_disk() is None
+        assert cfg._load_stale_models_cache_from_disk()["default_model"] == "snapshot-model"
+    finally:
+        profiles.clear_request_profile()
+
+
+@pytest.mark.parametrize("manifest", [
+    "kind: model-provider # valid YAML comment\n",
+    "kind: model-provider\n",
+    "kind: 'model-provider'\n",
+    "name: x\nkind: tool\n",
+    "meta:\n  kind: model-provider\n",
+    "# kind: model-provider\n",
+    "kind: [unclosed\nkind: model-provider\n",
+])
+def test_manifest_kind_parse_matches_agent(tmp_path, manifest):
+    agent = pytest.importorskip("providers")
+    (tmp_path / "plugin.yaml").write_text(manifest, encoding="utf-8")
+    assert cfg._declares_model_provider_kind(tmp_path) == agent._declares_model_provider_kind(tmp_path)
+
+
+def test_same_profile_config_edit_still_deletes_disk_cache(env, monkeypatch):
+    _save_demo_snapshot(env)
+    assert _switch_to_demo_and_fetch() == "snapshot-model"
+    _write_demo_config(env.home, "demo-model-2", 2_000_000)
+    cfg.invalidate_models_cache(delete_disk=False)
+    cache_at_rebuild = []
+    monkeypatch.setattr(cfg, "_invoke_models_rebuild", lambda _b: cache_at_rebuild.append(env.cache.exists()) or _catalog("x"))
+    _fetch_as_demo()
+    assert cache_at_rebuild == [False]
+
+
+@pytest.mark.parametrize("delete_disk", [True, False])
+def test_invalidate_drops_memory_and_deletes_disk_only_by_default(env, delete_disk):
+    _save_demo_snapshot(env)
+    _fetch_as_demo()
+    profiles.set_request_profile("demo")
+    try:
+        cfg.invalidate_models_cache(delete_disk=delete_disk)
+    finally:
+        profiles.clear_request_profile()
+    assert cfg._available_models_cache is None and cfg._models_cache_provenance is None
+    assert env.cache.exists() is not delete_disk
+
+
+def test_env_fingerprint_keys_match_provider_env_loader(env):
+    from api.providers import _load_env_file
+
+    _env("export A=1\nB='x'\nC=\n# D=1\n")(env.home)
+    loaded = sorted(k for k, v in _load_env_file(env.home / ".env").items() if v)
+    fp = cfg._models_cache_env_fingerprint(env.home / ".env")
+    assert [k for k, _ in fp] == loaded
+    assert all(len(h) == 64 and h not in ("1", "x") for _, h in fp)
+
+
+def test_env_fingerprint_is_keyed_not_a_plain_value_hash(env):
+    import hashlib
+
+    _env("DEEPSEEK_API_KEY=sk-secret\n")(env.home)
+    (_, digest), = cfg._models_cache_env_fingerprint(env.home / ".env")
+    assert digest != hashlib.sha256(b"sk-secret").hexdigest()
+
+
+def test_plugin_bytecode_cache_does_not_churn_fingerprint(env):
+    plugin = _write_plugin(env.home, code="X = 1\n")
+    before = cfg._models_cache_plugin_fingerprint(env.home)
+    (plugin / "__pycache__").mkdir()
+    (plugin / "__pycache__" / "__init__.cpython-311.pyc").write_bytes(b"x")
+    assert cfg._models_cache_plugin_fingerprint(env.home) == before
+
+
+@pytest.mark.parametrize("recreate", [False, True])
+def test_delete_and_recreate_profile_drop_its_models_cache(env, monkeypatch, recreate):
+    monkeypatch.setitem(sys.modules, "hermes_cli.profiles", None)
+    monkeypatch.setattr(profiles, "list_profiles_api", lambda: [])
+    _save_demo_snapshot(env)
+    stale = env.cache.read_text(encoding="utf-8")
+    profiles.delete_profile_api("demo")
+    assert not env.home.exists() and not env.cache.exists()
+    if recreate:
+        env.cache.write_text(stale, encoding="utf-8")  # left behind by an older build or another process
+        profiles.create_profile_api("demo")
+        assert not env.cache.exists()
+        _write_demo_config(env.home)
+        assert _switch_to_demo_and_fetch() == "fresh-model"

tests/test_trusted_header_auth.py

@@ -438,7 +438,7 @@ def test_profile_switch_accepts_bound_profile(monkeypatch):
     monkeypatch.setattr(routes, "_check_csrf", lambda _handler: True)
     monkeypatch.setattr(routes, "read_body", lambda _handler: {"name": "devops"})
     monkeypatch.setattr("api.profiles.switch_profile", lambda name, process_wide=False: {"ok": True, "profile": name})
-    monkeypatch.setattr("api.config.invalidate_models_cache", lambda: None)
+    monkeypatch.setattr("api.config.invalidate_models_cache", lambda *_a, **_kw: None)
     monkeypatch.setattr("api.gateway_watcher.restart_watcher_for_profile", lambda _name: None)
 
     routes.handle_post(handler, SimpleNamespace(path="/api/profile/switch", query=""))
@@ -851,7 +851,7 @@ def test_consumers_route_through_auth_owner(monkeypatch):
     monkeypatch.setattr(routes, "_check_csrf", lambda _handler: True)
     monkeypatch.setattr(routes, "read_body", lambda _handler: {"name": "devops"})
     monkeypatch.setattr("api.profiles.switch_profile", lambda name, process_wide=False: {"ok": True, "profile": name})
-    monkeypatch.setattr("api.config.invalidate_models_cache", lambda: None)
+    monkeypatch.setattr("api.config.invalidate_models_cache", lambda *_a, **_kw: None)
     monkeypatch.setattr("api.gateway_watcher.restart_watcher_for_profile", lambda _name: None)
 
     routes.handle_get(handler, SimpleNamespace(path="/api/auth/status", query=""))