Validate profile model selections

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### Fixed
+
+- New profile creation now validates submitted `default_model` / `model_provider` values against the server-side `/api/models` catalog before writing them to the profile config. Hand-crafted requests with nonexistent models now return a clear error instead of creating a profile that fails later at runtime. Empty model selection still means "use the active profile default." Refs #2240.
+
 ### Added
 
 - **PR #2099** by @dobby-d-elf — Adds an opt-in `Settings → Preferences → Fade text effect` toggle (off by default). When enabled, newly streamed output tokens are revealed through an adaptive playout buffer and animated with an opacity-only fade similar to ChatGPT and other frontier LLM apps. Implementation details: fade locked per stream to avoid mid-stream toggle rewind; reduced-motion users get non-animated text; live cursor hidden while fade is active; custom renderer on `streaming-markdown` parser wraps only newly-appended words; animated spans replace themselves with plain text on `animationend` (no long-lived wrapper buildup in long responses); unsafe streamed `href`/`src` values blocked in fade renderer `set_attr` path. Performance tuning: 200ms base fade duration scaling to 350ms for fast output, 16ms word stagger, 320ms done-drain wait cap, 160 wps visual cap, max 2-3 words/frame, brief pauses after sentence punctuation. Default-off means existing users see no change. 293-line regression test pinning the contract.

api/profiles.py

@@ -986,6 +986,85 @@ def _split_webui_provider_model_value(default_model: Optional[str], model_provid
     return model, provider
 
 
+def _strip_webui_provider_prefix(model_id: object) -> str:
+    value = str(model_id or "").strip()
+    if value.startswith("@") and ":" in value:
+        return value.rsplit(":", 1)[1]
+    return value
+
+
+def _profile_model_selection_exists(
+    available_models: object,
+    default_model: Optional[str],
+    model_provider: Optional[str],
+) -> bool:
+    """Return True when a profile default model/provider exists in /api/models."""
+    if not default_model and not model_provider:
+        return True
+    if not isinstance(available_models, dict):
+        return False
+
+    provider_seen = False
+    model_seen = False
+    for group in available_models.get("groups", []) or []:
+        if not isinstance(group, dict):
+            continue
+        provider_id = str(group.get("provider_id") or "").strip()
+        if model_provider and provider_id != model_provider:
+            continue
+        if model_provider and provider_id == model_provider:
+            provider_seen = True
+        for model in group.get("models", []) or []:
+            if not isinstance(model, dict):
+                continue
+            model_id = str(model.get("id") or "").strip()
+            if not model_id:
+                continue
+            if default_model and (
+                model_id == default_model
+                or _strip_webui_provider_prefix(model_id) == default_model
+            ):
+                model_seen = True
+                if model_provider:
+                    return True
+        if not default_model and provider_seen:
+            return True
+
+    if model_provider and not provider_seen:
+        return False
+    return bool(model_seen)
+
+
+def _get_available_models_for_profile_validation() -> dict:
+    from api.config import get_available_models
+
+    return get_available_models()
+
+
+def _validate_profile_model_selection(
+    default_model: Optional[str],
+    model_provider: Optional[str],
+    available_models: Optional[dict] = None,
+) -> None:
+    """Reject profile model defaults that do not exist in the server catalog."""
+    if not default_model and not model_provider:
+        return
+    catalog = (
+        available_models
+        if available_models is not None
+        else _get_available_models_for_profile_validation()
+    )
+    if _profile_model_selection_exists(catalog, default_model, model_provider):
+        return
+    if default_model and model_provider:
+        raise ValueError(
+            f"Selected model '{default_model}' is not available for provider '{model_provider}'"
+        )
+    if default_model:
+        raise ValueError(f"Selected model '{default_model}' is not available")
+    raise ValueError(f"Selected model provider '{model_provider}' is not available")
+
+
 def _write_model_defaults_to_config(
     profile_dir: Path,
     *,
@@ -1033,6 +1112,7 @@ def create_profile_api(name: str, clone_from: str = None,
     if clone_from is not None and not _is_root_profile(clone_from):
         _validate_profile_name(clone_from)
     default_model, model_provider = _split_webui_provider_model_value(default_model, model_provider)
+    _validate_profile_model_selection(default_model, model_provider)
 
     try:
         from hermes_cli.profiles import create_profile

tests/test_issue749_profile_create_model_picker.py

@@ -2,6 +2,7 @@
 
 from pathlib import Path
 
+import pytest
 import yaml
 
 import api.profiles as profiles
@@ -72,3 +73,100 @@ def test_profile_model_config_writer_preserves_existing_model_settings(tmp_path)
     assert saved["model"]["base_url"] == "https://gateway.example/v1"
     assert saved["model"]["default"] == "gpt-5.5"
     assert saved["model"]["provider"] == "openai-codex"
+
+
+def test_profile_model_selection_accepts_catalog_model_with_provider():
+    catalog = {
+        "groups": [
+            {
+                "provider": "OpenAI Codex",
+                "provider_id": "openai-codex",
+                "models": [{"id": "gpt-5.5", "label": "GPT-5.5"}],
+            }
+        ]
+    }
+
+    profiles._validate_profile_model_selection(
+        "gpt-5.5",
+        "openai-codex",
+        available_models=catalog,
+    )
+
+
+def test_profile_model_selection_accepts_provider_qualified_picker_value():
+    catalog = {
+        "groups": [
+            {
+                "provider": "Research Gateway",
+                "provider_id": "custom:research-gateway",
+                "models": [
+                    {
+                        "id": "@custom:research-gateway:claude-opus-4.6",
+                        "label": "claude-opus-4.6",
+                    }
+                ],
+            }
+        ]
+    }
+
+    default_model, model_provider = profiles._split_webui_provider_model_value(
+        "@custom:research-gateway:claude-opus-4.6",
+        "custom:research-gateway",
+    )
+
+    profiles._validate_profile_model_selection(
+        default_model,
+        model_provider,
+        available_models=catalog,
+    )
+
+
+def test_profile_model_selection_rejects_unknown_model_provider_pair():
+    catalog = {
+        "groups": [
+            {
+                "provider": "OpenAI Codex",
+                "provider_id": "openai-codex",
+                "models": [{"id": "gpt-5.5", "label": "GPT-5.5"}],
+            }
+        ]
+    }
+
+    with pytest.raises(ValueError, match="not available for provider"):
+        profiles._validate_profile_model_selection(
+            "missing-model",
+            "openai-codex",
+            available_models=catalog,
+        )
+
+
+def test_profile_create_rejects_unknown_model_before_creating_profile(monkeypatch):
+    calls = []
+
+    monkeypatch.setattr(
+        profiles,
+        "_get_available_models_for_profile_validation",
+        lambda: {
+            "groups": [
+                {
+                    "provider": "OpenAI Codex",
+                    "provider_id": "openai-codex",
+                    "models": [{"id": "gpt-5.5", "label": "GPT-5.5"}],
+                }
+            ]
+        },
+    )
+    monkeypatch.setattr(
+        profiles,
+        "_create_profile_fallback",
+        lambda *args, **kwargs: calls.append((args, kwargs)),
+    )
+
+    with pytest.raises(ValueError, match="Selected model 'missing-model'"):
+        profiles.create_profile_api(
+            "research",
+            default_model="missing-model",
+            model_provider="openai-codex",
+        )
+
+    assert calls == []