fix(reasoning): degrade 'max' via provider ceiling, not sourced list
Gate found removing the max->xhigh coercion let 'max' reach provider adapters whose native ladder tops out lower, degrading WORSE than before: Gemini treats unknown 'max' as medium; pre-adaptive Claude (3.7/4.5) manual-thinking lacks a 'max' budget and falls to 8k. Fix: coerce_reasoning_effort_for_model now applies the hard provider ceiling (_filter_reasoning_efforts_for_provider) FIRST and degrades down the ladder when the ceiling excludes the requested level — even when the sourced capability list is empty (unrecognized) or wrongly advertises the level. Gemini + all Google aliases and pre-4.6 Anthropic now cap 'max' at xhigh; adaptive Claude 4.6+/DeepSeek/etc. preserve it. Regression tests added. Co-authored-by: perejaslav <perejaslav@users.noreply.github.com>
This commit is contained in:
1 parent
d56123f796
commit
96b04359ab
2 files changed
+85
No files matched your search
@@ -3251,9 +3251,38 @@ def _filter_reasoning_efforts_for_provider(
|
||||
return [eff for eff in normalized if eff in {"low", "medium", "high"}]
|
||||
if bare.startswith("gpt-5"):
|
||||
return [eff for eff in normalized if eff != "max"]
|
||||
# 'max' is a WebUI-level ceiling; providers whose native ladder tops out lower
|
||||
# must NOT advertise it, otherwise a stored/CLI 'max' degrades WORSE than the
|
||||
# prior max->xhigh coercion (Gemini's adapter treats unknown 'max' as medium;
|
||||
# pre-adaptive Anthropic manual-thinking lacks a 'max' budget and falls to 8k).
|
||||
# Dropping 'max' here lets the existing downgrade ladder land on xhigh/high.
|
||||
if provider in {"gemini", "google", "google-gemini", "google-vertex", "vertex"}:
|
||||
return [eff for eff in normalized if eff != "max"]
|
||||
if provider in {"anthropic", "claude", "anthropic-claude"} and _is_pre_adaptive_anthropic(bare):
|
||||
return [eff for eff in normalized if eff != "max"]
|
||||
return normalized
|
||||
|
||||
|
||||
def _is_pre_adaptive_anthropic(bare_model: str) -> bool:
|
||||
"""True for Claude models that predate the adaptive-thinking (4.6+) generation.
|
||||
|
||||
Adaptive models (Opus/Sonnet 4.6+, 4.7, …) accept 'max'; earlier manual-thinking
|
||||
Claudes (3.5/3.7/4.0–4.5) do not and must degrade 'max' to xhigh rather than
|
||||
fall through the manual-thinking budget table to its 8k default.
|
||||
"""
|
||||
m = (bare_model or "").lower()
|
||||
if "claude" not in m:
|
||||
return False
|
||||
# Extract a version like 4.6 / 4-5 / 3.7 from the model name.
|
||||
import re as _re
|
||||
match = _re.search(r"(\d+)[.\-](\d+)", m)
|
||||
if not match:
|
||||
# Unversioned/opus-latest style — treat as adaptive (current flagship).
|
||||
return False
|
||||
major, minor = int(match.group(1)), int(match.group(2))
|
||||
return (major, minor) < (4, 6)
|
||||
|
||||
|
||||
def _heuristic_reasoning_efforts(model_id: str, provider_id: str) -> list[str]:
|
||||
"""Fallback when hermes_cli is unavailable."""
|
||||
model = _strip_provider_hint_for_reasoning(model_id).lower()
|
||||
@@ -3638,6 +3667,32 @@ def coerce_reasoning_effort_for_model(
|
||||
provider_id=provider_id,
|
||||
base_url=base_url,
|
||||
)
|
||||
# Hard provider ceilings must win regardless of what the sourced capability
|
||||
# list says. resolve_model_reasoning_efforts() draws from hermes_cli /
|
||||
# models.dev / heuristics, and those can (a) return [] for an unrecognized
|
||||
# model or (b) wrongly advertise a WebUI-only level like 'max' for a provider
|
||||
# whose native ladder tops out lower. _filter_reasoning_efforts_for_provider
|
||||
# encodes the known ceilings (openai-codex gpt-5, Gemini, pre-adaptive
|
||||
# Anthropic all cap below 'max'); if it actively EXCLUDES the requested level,
|
||||
# honor that ceiling and degrade down the ladder even when the sourced list is
|
||||
# empty or (mistakenly) includes the level. This keeps a stored/CLI 'max' from
|
||||
# reaching an adapter that would silently downgrade it worse than xhigh/high
|
||||
# (Gemini→medium, legacy Claude manual-thinking→8k). For providers with NO
|
||||
# ceiling rule the filter returns the full list unchanged, so genuinely
|
||||
# unknown models still preserve the configured effort (#3505 behavior).
|
||||
ceiling = _filter_reasoning_efforts_for_provider(
|
||||
list(VALID_REASONING_EFFORTS), str(model_id or ""), str(provider_id or "")
|
||||
)
|
||||
if ceiling and raw not in ceiling:
|
||||
ladder = list(VALID_REASONING_EFFORTS) # ascending: minimal..xhigh..max
|
||||
try:
|
||||
raw_idx = ladder.index(raw)
|
||||
except ValueError:
|
||||
raw_idx = None
|
||||
if raw_idx is not None:
|
||||
for level in reversed(ladder[:raw_idx]): # strictly lower, highest first
|
||||
if level in ceiling:
|
||||
return level
|
||||
# An empty list is ambiguous: resolve_model_reasoning_efforts() returns []
|
||||
# both for models KNOWN not to support reasoning AND for models we simply
|
||||
# don't recognize (custom providers, aggregator-rewritten ids, brand-new
|
||||
|
||||
@@ -327,3 +327,33 @@ def test_get_reasoning_status_coerces_stale_max_to_xhigh(monkeypatch):
|
||||
)
|
||||
assert status["reasoning_effort"] == "xhigh"
|
||||
assert status["reasoning_effort"] != "max"
|
||||
|
||||
|
||||
def test_max_effort_degrades_to_xhigh_for_gemini():
|
||||
# Gemini's native ladder tops out below 'max'; its adapter would silently
|
||||
# treat an unknown 'max' as medium. A stored/CLI 'max' must degrade to xhigh
|
||||
# (the highest supported), not fall through to a worse level. (#4627 gate)
|
||||
for model in ("gemini-3-pro", "gemini-3-flash"):
|
||||
assert cfg.coerce_reasoning_effort_for_model(
|
||||
"max", model_id=model, provider_id="gemini"
|
||||
) == "xhigh", f"{model} max must degrade to xhigh"
|
||||
|
||||
|
||||
def test_max_effort_degrades_to_xhigh_for_pre_adaptive_anthropic():
|
||||
# Pre-adaptive Claude (3.7 / 4.0-4.5) uses manual thinking whose budget table
|
||||
# lacks 'max' and falls back to 8k; 'max' must degrade to xhigh instead. (#4627 gate)
|
||||
for model in ("claude-3-7-sonnet", "claude-sonnet-4-5"):
|
||||
assert cfg.coerce_reasoning_effort_for_model(
|
||||
"max", model_id=model, provider_id="anthropic"
|
||||
) == "xhigh", f"{model} max must degrade to xhigh"
|
||||
|
||||
|
||||
def test_max_effort_preserved_for_adaptive_anthropic_and_deepseek():
|
||||
# Adaptive Claude (4.6+) and DeepSeek genuinely support 'max' — it must NOT degrade.
|
||||
for model in ("claude-opus-4.6", "claude-opus-4.7"):
|
||||
assert cfg.coerce_reasoning_effort_for_model(
|
||||
"max", model_id=model, provider_id="anthropic"
|
||||
) == "max", f"{model} must preserve max"
|
||||
assert cfg.coerce_reasoning_effort_for_model(
|
||||
"max", model_id="deepseek-reasoner", provider_id="deepseek"
|
||||
) == "max"
|
||||
Reference in new issue
Block a user