diff --git a/api/config.py b/api/config.py index 24530ef7..0eb26136 100644 --- a/api/config.py +++ b/api/config.py @@ -3061,7 +3061,39 @@ def resolve_model_provider(model_id: str, *, explicitly_picked: bool = False) -> and provider_hint.lower() in _custom_endpoint_slugs_for_base_url(config_base_url) ): return _finalize(bare_model, config_provider, config_base_url) - return _finalize(bare_model, provider_hint, _get_provider_base_url(provider_hint)) + # _get_provider_base_url() only reads `providers:` and the ACTIVE + # `model.base_url`, so a named custom provider registered solely in + # `custom_providers:` resolved to None there. Prefer that entry's own + # endpoint so a non-active @custom: hint routes to its own URL + # instead of falling back to the default endpoint (HTTP 400 "Invalid + # model format or no credentials for provider: "). + # + # Match via _unique_custom_provider_entry on the module-level `cfg` list: + # it is pure and lock-safe, so it stays callable when resolve_model_provider() + # is invoked while the caller already holds the non-reentrant _cfg_lock — + # a get_config()-based resolver would self-deadlock there. It also does NOT + # guess: an unmatched slug (e.g. a host:port-derived one absent from + # custom_providers) keeps the prior None so no stale endpoint is persisted + # for it (#4728). A colliding slug still fails closed via the raise. + # + # When an exact custom_providers[] entry exists, that row is authoritative + # for its slug: use its stripped base_url directly (including None when + # blank). It must NOT fall through to _get_provider_base_url(), which + # could pair a same-slug keyed `providers:` endpoint with this entry's + # credentials (violating same-entry endpoint/key parity). Only fall back + # to _get_provider_base_url() when no exact custom_providers[] row exists. + if provider_hint.startswith("custom:"): + entry = _unique_custom_provider_entry( + cfg.get('custom_providers', []), + _custom_provider_slug_key(provider_hint), + ) + if entry is not None: + base_url = str(entry.get('base_url') or '').strip() or None + else: + base_url = _get_provider_base_url(provider_hint) + else: + base_url = _get_provider_base_url(provider_hint) + return _finalize(bare_model, provider_hint, base_url) if "/" in model_id: prefix, bare = model_id.split("/", 1) @@ -3251,41 +3283,482 @@ def resolve_model_provider(model_id: str, *, explicitly_picked: bool = False) -> return _finalize(model_id, config_provider, config_base_url) -def resolve_custom_provider_connection(provider_id: str) -> tuple[str | None, str | None]: - """Return (api_key, base_url) for a named ``custom:*`` provider. +def _custom_record_base_url(record: object) -> str | None: + """Return a custom record's OWN endpoint, or None when it declares none.""" + if not isinstance(record, dict): + return None + return str(record.get("base_url") or "").strip() or None - Supports ``custom_providers[].api_key`` as either a literal key or - ``${ENV_VAR}``, and ``custom_providers[].key_env`` as an env-var hint. - Returns ``(None, None)`` when no named custom provider matches. + +def _resolve_custom_record_key( + raw_api_key: object, + raw_key_env: object, + provider_hint: object = None, +) -> str | None: + """Static credential declared by ONE custom record. + + Accepts a literal key, a ``${ENV_VAR}`` reference or a ``key_env`` env-var + hint, then falls back to the ``CUSTOM__API_KEY`` convention. Reading + every form from the SAME record is what keeps an endpoint and a credential + from being resolved out of two different authorities. """ - pid = str(provider_id or "").strip().lower() - if not pid.startswith("custom:"): - return None, None + api_key = None + if raw_api_key is not None: + key_text = str(raw_api_key).strip() + if key_text.startswith("${") and key_text.endswith("}") and len(key_text) > 3: + api_key = _thread_local_env_value(key_text[2:-1]).strip() or None + elif key_text: + api_key = key_text + if not api_key: + key_env = str(raw_key_env or "").strip() + if key_env: + api_key = _thread_local_env_value(key_env).strip() or None + if not api_key and provider_hint: + api_key = _lookup_custom_api_key_env(provider_hint) + return api_key - slug = _custom_provider_slug_key(pid) - if not slug: - return None, None - # Read the live config snapshot to avoid stale module-level cache edge - # cases after profile switches or runtime config edits. - cfg_data = get_config() +# Explicit outcomes of a named ``custom:`` selection. +# +# The status travels with the resolved bundle so a caller can tell "a record +# OWNS this route" from "nothing owns this route" instead of inferring it from +# an empty URL/key pair. That difference is load-bearing: an unowned named route +# must fail closed — no ambient endpoint, no ambient credential, no ambient +# pool/transport, no rewrite to the generic ``custom`` provider and no keyless +# placeholder — rather than borrow whatever the ambient runtime resolved. +CUSTOM_SELECTION_EXACT = "exact" # exact custom_providers[] row +CUSTOM_SELECTION_KEYED = "keyed" # identity-owned providers:/model: record +CUSTOM_SELECTION_RESOLVER = "resolver" # injected connection_resolver reported a pair +CUSTOM_SELECTION_MISSING = "missing" # no authority owns this slug +CUSTOM_SELECTION_MALFORMED = "malformed" # ``custom:`` with no slug behind it +# ``ambiguous`` is never RETURNED: a slug collision raises +# AmbiguousCustomProviderError out of _unique_custom_provider_entry, so the +# actionable rename message reaches the user (routes turn it into a 400) instead +# of being silently degraded into a keyless or ambient-authority send. +CUSTOM_SELECTION_AMBIGUOUS = "ambiguous" +# Statuses for which NO authority owns the route. +CUSTOM_SELECTION_UNOWNED = (CUSTOM_SELECTION_MISSING, CUSTOM_SELECTION_MALFORMED) - def _resolve_key(raw_api_key, raw_key_env, provider_hint=None) -> str | None: - api_key = None - if raw_api_key is not None: - key_text = str(raw_api_key).strip() - if key_text.startswith("${") and key_text.endswith("}") and len(key_text) > 3: - api_key = _thread_local_env_value(key_text[2:-1]).strip() or None - elif key_text: - api_key = key_text - if not api_key: - key_env = str(raw_key_env or "").strip() - if key_env: - api_key = _thread_local_env_value(key_env).strip() or None - if not api_key and provider_hint: - api_key = _lookup_custom_api_key_env(provider_hint) - return api_key +# ── Terminal routing verdicts ──────────────────────────────────────────────── +# +# Hermes Agent does NOT read an incomplete connection pair as a refusal. Its +# ``agent/agent_init.py:_init_openai_client()`` honours the explicit endpoint and +# credential the constructor was handed only when BOTH are truthy: +# +# if api_key and base_url: +# client_kwargs = _explicit_client_kwargs(...) +# else: +# client_kwargs = _routed_client_kwargs(...) +# +# and ``_routed_client_kwargs()`` resolves a provider AGAIN — through the +# centralized router, then the init-time fallback chain. So a bundle that "fails +# closed" by clearing its endpoint and/or its credential is not terminal at the +# constructor boundary; clearing those fields is precisely the signal to route +# somewhere else, which is how an unroutable custom slug ends up talking to the +# ambient or fallback provider after all. +# +# A route that cannot be resolved therefore has to be represented as an explicit +# verdict rather than as an ordinary constructor-ready dict with a hole in it. +# :data:`CUSTOM_ROUTE_ERROR_FIELD` carries that verdict on every merged bundle +# (``None`` when the route is fine), and every consumer that builds an AIAgent — +# or writes the agent cache, or hands the bundle to an auxiliary client — must +# stop on it and emit a controlled failure instead. +CUSTOM_ROUTE_UNOWNED = "unowned_custom_provider" +CUSTOM_ROUTE_NO_CREDENTIAL = "custom_provider_credential_unresolved" +CUSTOM_ROUTE_NO_ENDPOINT = "custom_provider_endpoint_unresolved" + +# Key the verdict travels under on a merged bundle. ``None`` == routable. +CUSTOM_ROUTE_ERROR_FIELD = "route_error" + + +class CustomProviderRouteError(ValueError): + """Raised when a named ``custom:`` route must not reach a provider client. + + The route resolved to no usable ``(api_key, base_url)`` pair, and passing + that pair to AIAgent would re-enter provider routing rather than fail (see + the :data:`CUSTOM_ROUTE_ERROR_FIELD` note above). Raising is how the refusal + stays terminal across the constructor boundary. + + Subclasses ``ValueError`` for the same reason + :class:`AmbiguousCustomProviderError` does: existing ``except ValueError`` / + broad-``except`` handlers in the non-streaming routes already turn it into a + controlled 400 / deterministic fallback instead of a traceback. + """ + + def __init__(self, message: str, *, reason: str, provider: str | None = None, hint: str = ""): + super().__init__(message) + # Forwarded verbatim by HTTP handlers, exactly like the ambiguous-slug + # error's ``.message``. + self.message = message + self.reason = reason + self.provider = provider + self.hint = hint + + +def _custom_route_verdict(reason: str, provider: str | None) -> dict: + """Build the terminal verdict recorded on an unroutable bundle.""" + name = str(provider or "").strip() or "custom" + if reason == CUSTOM_ROUTE_UNOWNED: + message = ( + f"Custom provider '{name}' is not configured: no custom_providers[] row, " + "keyed providers[] record or model: authority owns that name." + ) + hint = ( + "Add the provider under Settings -> Providers (or config.yaml " + "custom_providers[]), or pick a configured provider, then send again." + ) + elif reason == CUSTOM_ROUTE_NO_CREDENTIAL: + message = ( + f"Custom provider '{name}' declares a credential source that produced no " + "API key, so the route has no usable connection." + ) + hint = ( + "Check the provider's api_key / key_env / key_cmd / credential_pool " + "setting and the environment variable it names, then send again." + ) + else: + message = ( + f"Custom provider '{name}' resolved no endpoint, so the route has no " + "usable connection." + ) + hint = ( + "Set a base_url for the provider under Settings -> Providers (or " + "config.yaml custom_providers[]), then send again." + ) + return {"reason": reason, "provider": name, "message": message, "hint": hint} + + +def custom_provider_route_error(bundle: object) -> dict | None: + """Return ``bundle``'s terminal verdict, or ``None`` when it is routable. + + The verdict dict carries ``reason`` (one of :data:`CUSTOM_ROUTE_UNOWNED`, + :data:`CUSTOM_ROUTE_NO_CREDENTIAL`, :data:`CUSTOM_ROUTE_NO_ENDPOINT`), + ``provider``, a user-facing ``message`` and a ``hint``. + """ + if not isinstance(bundle, dict): + return None + verdict = bundle.get(CUSTOM_ROUTE_ERROR_FIELD) + return verdict if isinstance(verdict, dict) else None + + +def raise_for_custom_provider_route(bundle: dict) -> dict: + """Return ``bundle`` when routable; raise :class:`CustomProviderRouteError` otherwise. + + The single chokepoint every AIAgent-constructing consumer goes through, so + "this route is unresolvable" cannot degrade into "resolve it some other way" + at the constructor. + """ + verdict = custom_provider_route_error(bundle) + if verdict is None: + return bundle + raise CustomProviderRouteError( + verdict["message"], + reason=verdict["reason"], + provider=verdict["provider"], + hint=verdict["hint"], + ) + + +# Identity fields a record can use to NAME the custom provider it belongs to. +# ``name`` is the friendly name a ``custom_providers[]`` entry carries, so a +# ``providers:`` record spelled the same way is claiming the same identity. A +# ``model:`` block's ``name`` is the MODEL's name and never claims a provider, +# which is why that call site passes ``allow_name=False``. +_CUSTOM_RECORD_IDENTITY_FIELDS = ("provider_key", "provider", "custom_provider", "name") + + +def _custom_record_claims_slug(record: object, slug: str, *, allow_name: bool = True) -> bool: + """True when ``record`` names ``slug`` as its OWN identity. + + A generic record — ``providers['custom']`` or a ``model:`` block whose + provider is the bare string ``custom`` — belongs to no slug by itself, so it + is an authority for ``custom:`` only when it names that slug. Without + this test any unknown route adopts whichever generic record happens to be + configured, which is a credential handed to an endpoint the user never named. + """ + if not isinstance(record, dict) or not slug: + return False + for field in _CUSTOM_RECORD_IDENTITY_FIELDS: + if field == "name" and not allow_name: + continue + value = record.get(field) + if not isinstance(value, str) or not value.strip(): + continue + if _custom_provider_slug_key(value) == slug: + return True + return False + + +def _custom_record_owns_connection(record: dict, pid: str) -> bool: + """True when ``record`` declares ANY field the connection bundle carries. + + The bundle is more than a URL/key pair: ``key_cmd`` mints a per-request + bearer, ``credential_pool`` supplies a rotating one, and ``api_mode`` and the + ACP command/args decide the wire protocol and the transport process. A record + keyed to this slug that declares ONLY those still owns the route — reporting + it ``missing`` would fail the route closed and then clear the very fields it + declares (see :func:`merge_custom_provider_runtime_bundle`). + """ + base_url = _custom_record_base_url(record) + if base_url: + return True + # DECLARATION, not resolution: a ``key_env`` naming an unset variable or a + # pool that is momentarily empty still names this slug's credential source. + # Judging by "did a static key resolve?" hands the route to the ambient + # authority instead of surfacing the misconfiguration. + if _custom_record_declares_credential(record, base_url, None): + return True + if _resolve_custom_record_key(record.get("api_key"), record.get("key_env"), pid): + return True + if _custom_record_api_mode(record): + return True + if _custom_record_acp_transport(record): + return True + return False + + +# ── raw ``providers:`` records (the standard Hermes v12 config shape) ──── +# +# The installed Agent's authoritative matcher for a named custom route is +# ``hermes_cli.runtime_provider_custom._match_new_style_provider()``. It scans +# the ENABLED entries of the RAW ``providers:`` mapping and takes the first +# whose alias set contains the requested name, where the alias set is minted by +# ``hermes_cli.providers.custom_provider_aliases()`` from BOTH the entry's +# display ``name`` and its config KEY. So the standard v12 shape +# +# model: +# provider: custom:omni +# providers: +# omni: +# base_url: https://omni.example/v1 +# key_env: OMNI_GATE_KEY +# +# routes through ``providers['omni']``: that record names the identity by its +# own config key, exactly the way ``providers['custom:']`` does. Looking +# only at the ``custom:``-prefixed spellings made every send on that config fail +# closed as ``unowned_custom_provider`` while the Agent resolved it fine. +# +# The alias/enabled rules are MIRRORED here rather than imported for the same +# reason :data:`_API_MODE_ALIASES` is: selection has to keep working when the +# installed runtime module is unavailable or replaced by a test double. + + +def _agent_custom_provider_slug(value: object) -> str: + """``custom:`` identity the Agent mints for ``value``. + + Mirror of ``hermes_cli.providers.custom_provider_slug()``: lowercase, spaces + to dashes, prefixed unless it already carries one. Deliberately NOT + :func:`_custom_provider_slug_key`'s normalization — this one reproduces the + Agent's alias vocabulary, and both are consulted by + :func:`_custom_record_names_identity`. + """ + identity = str(value or "").strip().lower().replace(" ", "-") + if not identity: + return "" + return identity if identity.startswith("custom:") else f"custom:{identity}" + + +def _custom_provider_alias_set(display_name: object, provider_key: object) -> frozenset[str]: + """Every identity the Agent accepts for ONE custom-provider record. + + Mirror of ``hermes_cli.providers.custom_provider_aliases()``, including its + legacy ``custom:custom:`` spelling, so a record the Agent would route + to is not reported unowned here. + """ + aliases: set[str] = set() + for value in (display_name, provider_key): + raw = str(value or "").strip().lower() + if not raw: + continue + normalized = raw.replace(" ", "-") + aliases.update({raw, normalized, _agent_custom_provider_slug(normalized)}) + if normalized.startswith("custom:"): + suffix = normalized.split(":", 1)[1] + if suffix: + aliases.update({suffix, f"custom:{normalized}"}) + aliases.discard("") + return frozenset(aliases) + + +def _custom_record_names_identity(display_name: object, provider_key: object, pid: str, slug: str) -> bool: + """True when a record keyed ``provider_key`` / named ``display_name`` IS ``pid``. + + Two vocabularies, because two producers mint these ids. The Agent's alias set + is what decides a CLI send, so honouring it is what keeps the two in + agreement. :func:`_custom_provider_slug_key` is what the WebUI itself mints + from a friendly name (``Local (127.0.0.1:11434)`` -> ``local-127.0.0.1-11434``), + so a record the WebUI wrote is recognised by its own rules too. Both are + identity tests on the record's OWN name/key — neither widens selection to a + record that names some other provider. + """ + if pid in _custom_provider_alias_set(display_name, provider_key): + return True + return any( + _custom_provider_slug_key(value) == slug + for value in (display_name, provider_key) + if str(value or "").strip() + ) + + +# ``enabled:`` words YAML may hand us as strings, matching +# hermes_cli.config_providers._FALSE_WORDS. +_PROVIDER_DISABLED_WORDS = frozenset({"false", "0", "no", "off"}) + + +def _raw_provider_record_enabled(record: object) -> bool: + """Mirror of ``hermes_cli.config_providers.is_provider_enabled()``. + + Default True; only an explicit falsey ``enabled`` hides the entry. A disabled + entry is invisible to the Agent's resolver, so it must not own a WebUI route + either — the route fails closed instead of quietly using a row the user + switched off. + """ + if not isinstance(record, dict): + return False + flag = record.get("enabled", True) + if isinstance(flag, bool): + return flag + if isinstance(flag, str): + return flag.strip().lower() not in _PROVIDER_DISABLED_WORDS + return bool(flag) + + +# ``_entry_url``'s precedence in hermes_cli.runtime_provider_custom. +_RAW_PROVIDER_URL_FIELDS = ("api", "url", "base_url") + + +def _normalized_raw_provider_record(record: dict, ep_name: str) -> dict: + """Return ``record`` respelled in the field names this module's readers use. + + ONE record in, ONE record out: every value is copied from ``record`` and + nothing is sourced from anywhere else, so the endpoint, the credential, the + wire protocol, the pool, the ACP transport, the capabilities and the request + fields the caller then lifts all still belong to that single authority. Only + the SPELLINGS differ between the raw ``providers:`` shape and the + ``custom_providers[]`` shape the field readers were written against: + + ``api`` / ``url`` -> ``base_url`` (the Agent's ``_entry_url`` precedence) + ``api_key_env`` -> ``key_env`` + ```` -> ``provider_key``, when the record names none itself + + ``transport`` -> ``api_mode`` and ``command`` / ``args`` -> ``acp_*`` need no + rewrite: :func:`_custom_record_api_mode` and + :func:`_custom_record_acp_transport` already accept both spellings in place. + + Respelling a COPY rather than the live record keeps the config snapshot + unmutated, so a second resolution of the same slug sees the same input. + """ + normalized = dict(record) + for field in _RAW_PROVIDER_URL_FIELDS: + value = record.get(field) + if isinstance(value, str) and value.strip(): + normalized["base_url"] = value.strip() + break + if not str(normalized.get("key_env") or "").strip(): + api_key_env = record.get("api_key_env") + if isinstance(api_key_env, str) and api_key_env.strip(): + normalized["key_env"] = api_key_env.strip() + if not str(normalized.get("provider_key") or "").strip() and str(ep_name or "").strip(): + # The record's OWN config key is its identity. Stamping it keeps the + # credential-pool lookup and the ``key_cmd`` token process labelled with + # this provider instead of the generic ``custom``. + normalized["provider_key"] = str(ep_name).strip() + return normalized + + +def _unique_raw_provider_record(providers_cfg: object, pid: str, slug: str) -> dict | None: + """The ONE enabled raw ``providers:`` record that owns ``pid``, else None. + + Skips the two keys that already have dedicated, higher-precedence candidates + in :func:`_select_custom_provider_record` — ``providers['custom:']`` + and the generic ``providers['custom']`` — so one record can never be counted + as two authorities, nor collide with itself. + + Fails closed on a genuine collision: when two DISTINCT raw records both name + this identity and both declare connection fields, an endpoint and a + credential could be lifted from different rows, which is the same + split-authority pairing :func:`_unique_custom_provider_entry` raises on. A + record that names the identity but declares nothing is not a competing + authority, so it neither wins nor blocks — selection simply falls through to + the next candidate, and to the terminal verdict when there is none. + """ + if not isinstance(providers_cfg, dict): + return None + owning: list[tuple[str, dict]] = [] + for ep_name, record in providers_cfg.items(): + key = str(ep_name or "").strip() + if not key or key.lower() in {pid, "custom"}: + continue + if not isinstance(record, dict) or not record: + continue + if not _raw_provider_record_enabled(record): + continue + if not _custom_record_names_identity(record.get("name") or key, key, pid, slug): + continue + normalized = _normalized_raw_provider_record(record, key) + if _custom_record_owns_connection(normalized, pid): + owning.append((key, normalized)) + if len(owning) >= 2: + keys = [key for key, _record in owning] + raise AmbiguousCustomProviderError( + f"Custom provider records {keys!r} under providers: all name the provider " + f"slug {slug!r}; an endpoint and API key could be resolved from different " + f"records. Rename one so each custom provider has a unique slug." + ) + return owning[0][1] if owning else None + + +def _select_custom_provider_record( + pid: str, + slug: str, + cfg_data: dict, +) -> tuple[dict | None, str, bool, str]: + """Return ``(record, source, is_exact, status)`` — the ONE authority for ``pid``. + + Selection is IDENTITY-OWNED: a candidate qualifies only by naming this slug + (or by being the bare-``custom`` / explicitly-matching ``model:`` authority), + never by being the only row around. Selection order is the WebUI routing + contract: + + 1. the exact ``custom_providers[]`` row whose name normalizes to ``slug`` + (authoritative for its slug even against a same-slug keyed record, and + even when its ``base_url`` is blank — see #1806); + 2. otherwise the keyed ``providers['custom:']`` record, then the raw + ``providers:`` record that names this identity by its own config key + or display name (``providers: {omni: ...}`` for ``custom:omni`` — the + standard Hermes v12 shape, matched with the installed Agent's own alias + rules; see :func:`_unique_raw_provider_record`), then a record that NAMES + this slug (the generic ``providers['custom']`` entry whose own + name/provider_key normalizes to it, or a ``model:`` block whose provider + is ``custom:``), and last the generic ``providers['custom']`` entry + when ``model.provider`` names this slug — the active-provider shape the + WebUI itself writes. Each is taken as a COMPLETE record rather than + field-by-field, and each is eligible as soon as it declares ANY field the + connection bundle carries, not just a static key or a base_url (see + :func:`_custom_record_owns_connection`). + + What is deliberately NOT eligible is the generic ``providers['custom']`` + record or a bare-``custom`` ``model:`` block that names no slug at all. Those + are catch-alls, and while they matched every lookup, ``custom:ghost`` took + whichever one was configured and inherited its endpoint, credential, pool, + ``api_mode`` and ACP transport — the same wrong-authority pairing as the + sole-row fallback below, sourced from ``providers:``/``model:`` instead. + + There is deliberately NO "the list holds exactly one row, so use it" + fallback. That rule resolved ``custom:ghost`` to the endpoint AND credential + of a sole unrelated row named ``omni`` — the same wrong-authority pairing the + exact-row rule exists to prevent, and a credential leak to an endpoint the + user never named. An unknown slug owns nothing, so it reports ``missing`` + and the caller fails closed. This matches the point-of-return rule + ``resolve_model_provider`` already applies (#4728: no unique entry -> no + base_url, never a guess). + + ``source`` is ``custom_providers`` / ``providers`` / ``model`` (or ``""`` + when nothing matched) and names the authority that owns every field the + caller then lifts off ``record``. ``status`` is one of the + ``CUSTOM_SELECTION_*`` values (``ambiguous`` raises instead of returning). + """ custom_providers = cfg_data.get("custom_providers", []) if not isinstance(custom_providers, list): custom_providers = [] @@ -3295,47 +3768,1010 @@ def resolve_custom_provider_connection(provider_id: str) -> tuple[str | None, st # and credential are always resolved from the SAME entry. matched_entry = _unique_custom_provider_entry(custom_providers, slug) if matched_entry is not None: - base_url = str(matched_entry.get("base_url") or "").strip() or None - api_key = _resolve_key(matched_entry.get("api_key"), matched_entry.get("key_env"), pid) - return api_key, base_url + return matched_entry, "custom_providers", True, CUSTOM_SELECTION_EXACT - # If exactly one custom provider is configured, use it as a pragmatic - # fallback for mismatched slugs (e.g. punctuation differences). - if len(custom_providers) == 1 and isinstance(custom_providers[0], dict): - entry = custom_providers[0] - return ( - _resolve_key(entry.get("api_key"), entry.get("key_env"), pid), - str(entry.get("base_url") or "").strip() or None, - ) - - # Fallbacks for setups that don't use custom_providers names directly. + # Fallbacks for setups that don't use custom_providers names directly. Every + # one of them is keyed on this provider's OWN identity. providers_cfg = cfg_data.get("providers", {}) provider_specific = providers_cfg.get(pid, {}) if isinstance(providers_cfg, dict) else {} provider_custom = providers_cfg.get("custom", {}) if isinstance(providers_cfg, dict) else {} + if ( + isinstance(provider_specific, dict) + and provider_specific + and not _raw_provider_record_enabled(provider_specific) + ): + # ``providers['custom:']`` names THIS slug by its own exact key, so + # switching it off is a statement about this route and not merely about + # one candidate among several. Falling through to the generic ``custom`` + # record or the ``model:`` block would honour the disable by routing the + # turn somewhere ELSE — the ambient provider's endpoint and credential + # under the name the user just took offline. The slug's own authority + # said no, so the route is terminal here. + return None, "", False, CUSTOM_SELECTION_MISSING + model_cfg = cfg_data.get("model", {}) model_provider = str(model_cfg.get("provider") or "").strip().lower() if isinstance(model_cfg, dict) else "" - fallback_base = None - for candidate in (provider_specific, provider_custom, model_cfg): - if isinstance(candidate, dict): - _base = str(candidate.get("base_url") or "").strip() - if _base: - fallback_base = _base - break + # The generic ``providers['custom']`` record and a ``model:`` block whose + # provider is the bare string ``custom`` are catch-alls: they name no slug of + # their own. While they were eligible for EVERY ``custom:`` lookup, + # ``custom:ghost`` selected whichever one happened to be configured and + # inherited its endpoint, credential, pool, api_mode and ACP transport — the + # user's prompt and a credential sent to a provider they never named. + # Candidates are ordered by how specifically they name THIS identity. + named_by_model = model_provider in {pid, slug} + # A disabled record is invisible to the Agent's resolver, so it is not an + # authority here either — at EVERY rung, not just the raw-``providers:`` scan + # that already filtered for it. + has_generic_custom = ( + isinstance(provider_custom, dict) + and bool(provider_custom) + and _raw_provider_record_enabled(provider_custom) + ) + # The generic record is keyed ``custom`` in ``providers:``, so it names the + # literal ``custom:custom`` route by its own key; for any other slug it has + # to say so itself. + generic_claims_slug = has_generic_custom and ( + slug == "custom" or _custom_record_claims_slug(provider_custom, slug) + ) - fallback_key = None - if isinstance(provider_specific, dict): - fallback_key = _resolve_key(provider_specific.get("api_key"), provider_specific.get("key_env"), pid) - if not fallback_key and isinstance(provider_custom, dict): - fallback_key = _resolve_key(provider_custom.get("api_key"), provider_custom.get("key_env"), pid) - if not fallback_key and isinstance(model_cfg, dict) and model_provider in {"custom", pid, slug}: - fallback_key = _resolve_key(model_cfg.get("api_key"), model_cfg.get("key_env"), pid) + def _candidates(): + """Identity-owned candidates, most specific first — evaluated LAZILY. - if fallback_key or fallback_base: - return fallback_key, fallback_base or None + Laziness is load-bearing for the raw-``providers:`` scan alone: that one + RAISES on an alias collision, and a config whose route is already decided + by the exact ``providers['custom:']`` key must not be failed closed + by a collision further down a list it never reaches. + """ + if isinstance(provider_specific, dict) and provider_specific: + # Keyed by this provider's own id: ``providers['custom:']``. + yield provider_specific, "providers" + # The standard v12 shape: a raw ``providers:`` record that names + # this identity by its own config key or display name (``providers: + # {omni: ...}`` for ``custom:omni``), matched with the installed Agent's + # alias rules. This is identity ownership, not an ambient-field escape — + # a record that names some OTHER provider never appears here, and an + # unknown slug still matches nothing and falls through to the terminal + # verdict below. + raw_record = _unique_raw_provider_record(providers_cfg, pid, slug) + if raw_record is not None: + yield raw_record, "providers" + if generic_claims_slug: + yield provider_custom, "providers" + if isinstance(model_cfg, dict) and model_cfg and _raw_provider_record_enabled(model_cfg) and ( + named_by_model + or (model_provider == "custom" and _custom_record_claims_slug(model_cfg, slug, allow_name=False)) + ): + yield model_cfg, "model" + if has_generic_custom and named_by_model and not generic_claims_slug: + # Last: the active-provider shape the WebUI itself writes, where the + # generic record holds the configuration of whichever custom provider + # ``model.provider`` names — here, THIS slug. That is a provenance + # tie rather than a declaration, so anything naming the slug outright + # (above) outranks it. + yield provider_custom, "providers" - return None, None + for cand, source in _candidates(): + # A record that names this slug is its authority as soon as it declares + # ANY connection field — a static key and a base_url are not the only + # things a record can own. One that declares only ``key_cmd``, a + # credential pool, an ``api_mode`` or an ACP transport is still THIS + # slug's record; reporting it missing would clear those very fields as + # foreign in merge_custom_provider_runtime_bundle. + # + # The ``enabled`` re-check is the single chokepoint every candidate has + # to pass: each branch above gates on it too, and one selection path + # added later without that gate is exactly how a switched-off row's + # endpoint and secret got back onto a live route. + if _raw_provider_record_enabled(cand) and _custom_record_owns_connection(cand, pid): + return cand, source, False, CUSTOM_SELECTION_KEYED + + return None, "", False, CUSTOM_SELECTION_MISSING + + +def resolve_custom_provider_connection( + provider_id: str, + *, + return_provenance: bool = False, +) -> tuple[str | None, str | None] | tuple[str | None, str | None, bool]: + """Return (api_key, base_url) for a named ``custom:*`` provider. + + Supports ``custom_providers[].api_key`` as either a literal key or + ``${ENV_VAR}``, and ``custom_providers[].key_env`` as an env-var hint. + Returns ``(None, None)`` when no named custom provider matches. + If ``return_provenance=True``, returns ``(api_key, base_url, is_exact)`` + indicating whether the connection was resolved from an exact matching + ``custom_providers[]`` entry. + + This is the URL/key VIEW of the authoritative record. Agent-construction + paths want :func:`resolve_custom_provider_bundle` instead: the record also + owns ``api_mode``, ``key_cmd``, pool credentials and ACP transport fields, + and a caller that takes only two of them still builds a mixed-authority + agent. + """ + pid = str(provider_id or "").strip().lower() + if not pid.startswith("custom:"): + if return_provenance: + return None, None, False + return None, None + + slug = _custom_provider_slug_key(pid) + if not slug: + if return_provenance: + return None, None, False + return None, None + + # Read the live config snapshot to avoid stale module-level cache edge + # cases after profile switches or runtime config edits. + record, _source, is_exact, _status = _select_custom_provider_record(pid, slug, get_config()) + if record is None: + # Nothing owns this slug. Returning ``(None, None)`` is the whole point: + # an unknown named route must not inherit an unrelated row's endpoint or + # credential (see _select_custom_provider_record). + if return_provenance: + return None, None, False + return None, None + + base_url = _custom_record_base_url(record) + api_key = _resolve_custom_record_key(record.get("api_key"), record.get("key_env"), pid) + if return_provenance: + return api_key, base_url, is_exact + return api_key, base_url + + +# Local OpenAI-compatible servers frequently run without authentication, so a +# missing key must not fail before the first request: hand the SDK a harmless +# placeholder and let the endpoint accept it or return its own auth error. It is +# applied ONLY after the authoritative record has been resolved in full and +# reported itself genuinely keyless (no api_key, no key_env, no key_cmd, no +# host-gated env key, no credential pool) — substituting it while any of those +# could still mint a credential is what turned a working ``key_cmd`` endpoint +# into a 401. +KEYLESS_CUSTOM_API_KEY = "dummy-key" + +# The constructor-routing fields AIAgent takes beside provider/base_url/api_key. +# They travel with the connection: a bundle that replaces the endpoint and the +# credential but leaves these behind builds an agent whose wire protocol +# (api_mode), transport (ACP subprocess) or credential source (pool) still +# points at the previous authority. +CUSTOM_CONNECTION_SIDE_FIELDS = ("api_mode", "acp_command", "acp_args", "credential_pool") + +# Alias spellings accepted for a record's ``api_mode`` / ``transport``, mirroring +# hermes_cli.config_providers._canonical_api_mode. Resolved locally rather than +# imported so a record's own transport survives even when the installed runtime +# module is unavailable (or replaced by a test double). +_API_MODE_ALIASES = { + "chat-completions": "chat_completions", + "chatcompletions": "chat_completions", + "openai": "chat_completions", + "responses": "codex_responses", + "openai_responses": "codex_responses", + "openai-responses": "codex_responses", + "anthropic": "anthropic_messages", + "anthropic-messages": "anthropic_messages", + "messages": "anthropic_messages", + "bedrock": "bedrock_converse", + "bedrock-converse": "bedrock_converse", +} +_VALID_API_MODES = { + "chat_completions", + "codex_responses", + "anthropic_messages", + "bedrock_converse", + "codex_app_server", +} + + +def _custom_record_api_mode(record: dict) -> str | None: + """Return the wire protocol a custom record declares for ITSELF, else None. + + ``transport:`` is the v12-migration spelling of ``api_mode:``; hand-edited + configs still use either. An unrecognized value returns None so the runtime + keeps its own host/provider detection instead of being handed nonsense. + """ + for field in ("api_mode", "transport"): + raw = record.get(field) + if not isinstance(raw, str): + continue + cleaned = raw.strip() + if not cleaned: + continue + canonical = _API_MODE_ALIASES.get(cleaned.lower(), cleaned).lower() + if canonical in _VALID_API_MODES: + return canonical + return None + + +def _custom_record_acp_transport(record: dict) -> dict: + """Return the ACP subprocess transport a custom record declares for ITSELF.""" + owned: dict = {} + command = record.get("acp_command") or record.get("command") + if isinstance(command, str) and command.strip(): + owned["acp_command"] = command.strip() + args = record.get("acp_args") + if args is None: + args = record.get("args") + if isinstance(args, (list, tuple)) and len(args) > 0: + owned["acp_args"] = list(args) + return owned + + +def _custom_record_pool_runtime(base_url: str | None, record: dict) -> dict | None: + """Runtime dict from the credential pool that owns ``base_url``, else None. + + Delegates to the installed runtime's own pool lookup so pool ownership, + ordering and the loopback placeholder behave exactly as they do for a CLI + send. Best-effort: builds that do not expose the helper (or a stubbed + runtime module) simply report no pool. + """ + if not base_url: + return None + try: + import hermes_cli.runtime_provider as _runtime_provider + + resolve_pool = getattr(_runtime_provider, "_try_resolve_from_custom_pool", None) + if resolve_pool is None: + return None + return resolve_pool( + base_url, + "custom", + _custom_record_api_mode(record), + provider_name=str(record.get("provider_key") or record.get("name") or "") or None, + ) + except Exception: + return None + + +def _host_gated_env_key(base_url: str | None) -> str | None: + """Env credential the installed runtime would accept for ``base_url``, else None. + + Host-GATED on purpose (GHSA-76xc-57q6-vm5m): the helper only yields + OPENAI/OPENROUTER/````_API_KEY when the endpoint's host is the + authoritative one, so this cannot leak a cloud key to an unrelated custom + endpoint. Consulted here only so "is this endpoint genuinely keyless?" has + the same answer in WebUI as it does at runtime. + """ + if not base_url: + return None + try: + import hermes_cli.runtime_provider as _runtime_provider + + candidates = getattr(_runtime_provider, "_host_gated_env_key_candidates", None) + if candidates is None: + return None + for candidate in candidates(base_url, ollama=False): + cleaned = str(candidate or "").strip() + if cleaned: + return cleaned + except Exception: + return None + return None + + +def _custom_record_key_cmd_provider(base_url: str | None, record: dict, pid: str): + """Per-request token provider for a record's ``key_cmd``, else None. + + ``key_cmd`` names a command that PRINTS a short-lived bearer; both wire + clients accept a callable api_key and mint per request. It is a real + credential source, so a record that declares one is NOT keyless and must + never be handed :data:`KEYLESS_CUSTOM_API_KEY`. + """ + key_cmd = str(record.get("key_cmd") or "").strip() + if not key_cmd: + return None + try: + from agent.command_token_source import build_command_token_provider + + return build_command_token_provider( + key_cmd, str(record.get("name") or record.get("provider_key") or pid or "custom") + ) + except Exception: + logger.debug("key_cmd token provider unavailable for %s", pid, exc_info=True) + return None + + +def _custom_record_declares_credential( + record: dict, + base_url: str | None, + pool_runtime: dict | None, +) -> bool: + """True when the record DECLARES any credential source, resolved or not. + + "Keyless" is a positive claim that an endpoint wants no authentication, and + it is the only gate on :data:`KEYLESS_CUSTOM_API_KEY`. Deriving it from + "``api_key`` came back empty" conflates two opposite situations: an + unauthenticated local server, and an authenticated endpoint whose declared + credential did not resolve (an ``${ENV}`` that is unset, a ``key_env`` + naming a missing variable, a configured pool that yielded nothing, an + unbuildable ``key_cmd``). The second must NOT be handed the placeholder — + that turns a missing-credential misconfiguration into an opaque 401 from the + endpoint and hides the real cause. + + So the question asked here is declaration, not resolution: the RAW + ``api_key`` (literal or ``${ENV}``), ``key_env``, ``key_cmd``, a configured + ``credential_pool``, a pool that actually resolved for this endpoint, and a + host-gated env credential the runtime would accept for it. + """ + for field in ("api_key", "key_env", "key_cmd"): + if str(record.get(field) or "").strip(): + return True + # A configured pool is a declaration even when empty or exhausted: the user + # pointed this endpoint at a credential source. + if record.get("credential_pool") is not None: + return True + if pool_runtime: + return True + if _host_gated_env_key(base_url): + return True + return False + + +def _unowned_custom_provider_bundle(pid: str, slug: str, status: str) -> dict: + """Bundle for a named ``custom:*`` route that NO authority owns. + + Every connection field is empty and ``keyless`` is False, so the merge below + cannot substitute :data:`KEYLESS_CUSTOM_API_KEY`: "nobody owns this route" is + not a claim that the route is unauthenticated. + """ + return { + "provider_id": pid, + "slug": slug, + "source": "", + "status": status, + "is_exact": False, + "record": None, + "base_url": None, + "api_key": None, + # Nothing owns the route, so nothing positively owns an endpoint for it + # either. Stated rather than inferred: the merge must never read an + # endpoint that reached it from somewhere else as this route's own. + "endpoint_owned": False, + "keyless": False, + "owned": {}, + } + + +def resolve_custom_provider_bundle( + provider_id: str, + *, + connection_resolver=None, +) -> dict | None: + """Return the COMPLETE connection bundle a named ``custom:*`` record owns. + + ``None`` only when ``provider_id`` is not a named custom provider at all. + When it IS one but nothing owns it, the result is an explicit unowned bundle + (``status`` ``missing``/``malformed``, every connection field empty and + ``keyless`` False) rather than ``None``, so the merge below can fail the + route closed instead of silently leaving the ambient runtime's endpoint, + credential and pool in place. Otherwise a dict with: + + ``base_url`` + the record's own endpoint (``None`` when it declares none). + ``endpoint_owned`` + True only when the SELECTED record supplied that endpoint itself. It is + the positive half of the provenance the merge needs: an endpoint already + sitting in a caller's bundle, an endpoint that merely compares equal, and + an absent runtime dict are all silence, and silence must never be read as + "this endpoint belongs to the selected record". False here means the + route has no endpoint of its own, whatever else is in flight. + ``api_key`` + the record's own credential, resolved through the SAME ladder the + runtime uses for a named custom provider: pool credential, then literal + / ``${ENV}`` / ``key_env`` / ``CUSTOM__API_KEY``, then a host-gated + env key — with ``key_cmd`` overriding the static forms because it mints a + fresh bearer per request. May be a callable (``key_cmd`` token provider). + ``keyless`` + True only when the record DECLARES no credential source at all, i.e. the + endpoint is genuinely unauthenticated. A declared-but-unresolved + credential (unset ``${ENV}``, missing ``key_env`` variable, empty pool, + unbuildable ``key_cmd``) leaves this False — see + :func:`_custom_record_declares_credential`. This is the ONLY gate on + :data:`KEYLESS_CUSTOM_API_KEY`. + ``owned`` + the subset of :data:`CUSTOM_CONNECTION_SIDE_FIELDS` this record supplies + itself. Callers keep these and must not overwrite them with the ambient + runtime's values; fields ABSENT here are simply unowned, not proven + foreign (see :func:`merge_custom_provider_runtime_bundle`). + ``source`` / ``is_exact`` / ``record`` / ``status`` + the provenance of the selection, so a caller can tell an exact + ``custom_providers[]`` row from a keyed fallback — and either of them + from a route nothing owns (``status`` in + :data:`CUSTOM_SELECTION_UNOWNED`). + + ``connection_resolver`` lets a caller inject its own module-bound reference + to :func:`resolve_custom_provider_connection`, so monkeypatching that name in + the caller's namespace still takes effect. An injected resolver can only + report a URL/key pair — there is no record behind it — so the bundle it + yields owns no side fields and the caller keeps the runtime's. + """ + pid = str(provider_id or "").strip().lower() + if not pid.startswith("custom:"): + return None + slug = _custom_provider_slug_key(pid) + if not slug: + # ``custom:`` with nothing behind it names no provider, so no record can + # own it. Report it explicitly rather than as "not a custom route": the + # caller asked to route somewhere and must fail closed, not inherit the + # ambient connection. + return _unowned_custom_provider_bundle(pid, slug, CUSTOM_SELECTION_MALFORMED) + + if connection_resolver is not None and connection_resolver is not resolve_custom_provider_connection: + # Tolerate resolvers that predate/omit the provenance kwarg (older builds + # and test doubles that patch in a plain two-value resolver). + try: + conn = connection_resolver(pid, return_provenance=True) + except TypeError: + conn = connection_resolver(pid) + if len(conn) == 3: + api_key, base_url, is_exact = conn + else: + api_key, base_url = conn + is_exact = False + if not (is_exact or api_key or base_url): + # The injected resolver is the whole authority on this path, and it + # reported nothing for the slug — so nothing owns the route. + return _unowned_custom_provider_bundle(pid, slug, CUSTOM_SELECTION_MISSING) + return { + "provider_id": pid, + "slug": slug, + "source": "resolver", + "status": CUSTOM_SELECTION_RESOLVER, + "is_exact": bool(is_exact), + "record": None, + "base_url": base_url, + "api_key": api_key, + "endpoint_owned": bool(base_url), + # A resolver reports a URL/key pair and nothing else; with no record + # behind it there is no declaration to inspect, so an absent key is + # the only keyless signal available here. + "keyless": not api_key, + "owned": {}, + } + + record, source, is_exact, status = _select_custom_provider_record(pid, slug, get_config()) + if record is None: + return _unowned_custom_provider_bundle(pid, slug, status) + + base_url = _custom_record_base_url(record) + owned: dict = {} + + api_mode = _custom_record_api_mode(record) + if api_mode: + owned["api_mode"] = api_mode + owned.update(_custom_record_acp_transport(record)) + + # Pool first, exactly like the runtime's named-custom path: a pooled + # endpoint's credential AND its pool object come from the same lookup. + api_key = None + pool_runtime = _custom_record_pool_runtime(base_url, record) + if pool_runtime: + api_key = pool_runtime.get("api_key") or None + if pool_runtime.get("credential_pool") is not None: + owned["credential_pool"] = pool_runtime.get("credential_pool") + + if not api_key: + api_key = _resolve_custom_record_key(record.get("api_key"), record.get("key_env"), pid) + if not api_key: + api_key = _host_gated_env_key(base_url) + + # ``key_cmd`` beats a static api_key / key_env (short-lived bearers go stale + # mid-session), but never displaces a pooled credential, which the pool + # itself already rotates. + if not pool_runtime: + token_provider = _custom_record_key_cmd_provider(base_url, record, pid) + if token_provider is not None: + api_key = token_provider + + if "credential_pool" not in owned and record.get("credential_pool") is not None: + owned["credential_pool"] = record.get("credential_pool") + + return { + "provider_id": pid, + "slug": slug, + "source": source, + "status": status, + "is_exact": is_exact, + "record": record, + "base_url": base_url, + "api_key": api_key, + # POSITIVE endpoint provenance, resolved from the selected record and + # from nothing else. The merge below pairs this record's credential with + # an endpoint only while this is True. + "endpoint_owned": bool(base_url), + "keyless": not api_key + and not _custom_record_declares_credential(record, base_url, pool_runtime), + "owned": owned, + } + + +def _connection_identity(runtime_provider: dict, resolved_provider: str | None) -> str | None: + """Name the provider the already-resolved connection fields belong to. + + The runtime provider dict names itself, so it wins when present. With no + runtime dict the caller resolved the connection for ``resolved_provider`` + (every such call site passes ``requested=resolved_provider``), so that is + the identity behind the fields. + """ + rt_provider = str((runtime_provider or {}).get("provider") or "").strip() + if rt_provider: + return rt_provider.lower() + return str(resolved_provider or "").strip().lower() or None + + +def _custom_bundle_endpoint_matches(bundle: dict, runtime_provider: dict) -> bool: + """True when the runtime resolved the SAME endpoint the record owns. + + This is the provenance test that decides whether the ambient runtime's side + fields are same-authority (keep) or foreign (clear). A record that declares + no endpoint of its own cannot prove the runtime is foreign, so its runtime + fields stand. + + Endpoint equality alone is NOT provenance. Two providers are free to share a + base_url — a gateway fronting several accounts, a local proxy, the same host + reached with different keys — so a URL match between a record and the ambient + runtime says only that both point at one host, never that one authority + resolved both. When the runtime dict NAMES itself and that name is some other + provider, the identities settle it directly and the shared URL proves + nothing. + """ + if not isinstance(runtime_provider, dict) or not runtime_provider: + return False + rt_provider = str(runtime_provider.get("provider") or "").strip().lower() + if rt_provider: + slug = str(bundle.get("slug") or "").strip().lower() + owned_names = { + str(bundle.get("provider_id") or "").strip().lower(), + slug, + f"custom:{slug}" if slug else "", + # The generic ``custom`` runtime is the shape the WebUI writes for + # whichever custom provider is active, so it is not a competing name. + "custom", + } + owned_names.discard("") + if rt_provider not in owned_names: + return False + record_base_url = bundle.get("base_url") + rt_base_url = runtime_provider.get("base_url") + if not record_base_url and not rt_base_url: + return True + if not record_base_url or not rt_base_url: + return False + return _normalize_base_url_for_match(record_base_url) == _normalize_base_url_for_match( + rt_base_url + ) + + +def _custom_runtime_endpoint_is_record_owned(bundle: dict, runtime_provider: dict) -> bool: + """True when the runtime dict positively originated from the selected RECORD. + + Endpoint equality alone is NOT selected-record provenance: two distinct + records (e.g. an exact list row and a same-slug keyed record) can share a + normalized URL while carrying different credentials. A tie requires either + an explicit source-record identity or matching record credentials; + silence, missing runtime dicts, and distinct records sharing an endpoint + never establish that the runtime spelling belongs to the selected record. + """ + if not bundle.get("base_url"): + return False + if not isinstance(runtime_provider, dict) or not runtime_provider.get("base_url"): + return False + # Credential conflict strictly rejects a tie before checking identity metadata + bundle_key = bundle.get("api_key") + rt_key = runtime_provider.get("api_key") + if bundle_key and rt_key and bundle_key != rt_key: + return False + rec = bundle.get("record") + rt_rec = runtime_provider.get("record") or runtime_provider.get("source_record") + if rt_rec is not None: + return rt_rec == rec and _custom_bundle_endpoint_matches(bundle, runtime_provider) + rec_id = bundle.get("record_id") or (rec.get("id") if isinstance(rec, dict) else None) + rt_id = runtime_provider.get("record_id") + if rt_id is not None: + return rt_id == rec_id and _custom_bundle_endpoint_matches(bundle, runtime_provider) + if bundle_key and rt_key and bundle_key == rt_key: + return _custom_bundle_endpoint_matches(bundle, runtime_provider) + return False + + +def merge_custom_provider_runtime_bundle( + resolved_provider: str | None, + resolved_api_key: str | None, + resolved_base_url: str | None, + runtime_provider: dict | None = None, + *, + lookup_provider: str | None = None, + connection_resolver=None, +) -> dict: + """Return the COMPLETE constructor-routing bundle for one send attempt. + + Keys: ``provider``, ``base_url``, ``api_key``, every field in + :data:`CUSTOM_CONNECTION_SIDE_FIELDS`, and + :data:`CUSTOM_ROUTE_ERROR_FIELD`. Callers must apply the WHOLE dict — that is + the point: the connection and the transport/protocol/pool fields are one + authority, and the agent-cache signature must be derived from the same dict + so a bundle change always mints a new agent. + + :data:`CUSTOM_ROUTE_ERROR_FIELD` is the bundle's TERMINAL verdict, and it is + not optional to check. A named ``custom:`` route that resolves no complete + ``(api_key, base_url)`` pair is not constructor-ready: Hermes Agent reads the + missing field as permission to resolve another provider, so every consumer + that builds an AIAgent (or writes the agent cache, or feeds an auxiliary + client) must run the bundle through :func:`raise_for_custom_provider_route` + first and fail the turn with a controlled provider / missing-credential + error. See the :data:`CUSTOM_ROUTE_ERROR_FIELD` commentary for the exact + constructor branch this defends. + + Every consumer that builds an AIAgent for a ``custom:`` route goes + through here so the endpoint, the credential and the routing fields all come + from ONE record. The fill-only pattern this replaces + (``if not api_key: api_key = ...``) mixed authorities whenever the runtime + provider had already supplied a truthy value from a same-slug keyed + ``providers:`` record: resolution deterministically produced the + ``custom_providers[]`` row's URL while keeping the keyed row's API key. + + Ownership rules, per side field: + + * the selected record declares it -> the record's value wins (an exact list + row's ``api_mode: anthropic_messages`` is not "ambient noise" to be + cleared, and a keyed record keeps the pool/transport it owns); + * the record declares no endpoint of its own, or the runtime resolved the + SAME endpoint -> the runtime's value is same-authority and is kept; + * otherwise the runtime resolved a DIFFERENT authority -> the field is + cleared, because passing it through is what let a custom HTTP endpoint + inherit Anthropic credential pooling and a Claude ACP subprocess. + + The ENDPOINT itself is decided by positive provenance only. A selected + config record supplies the endpoint its credential is sent to, and the one + exception is a runtime dict positively tied to that same record (same + identity, same normalized URL), whose spelling is the one it can reach. + Everything else is silence: the ``resolved_base_url`` the caller arrived + with, a URL that merely compares equal, and a missing runtime dict say + nothing about the record selected for this slug. So a record that declares + no usable ``base_url`` does not borrow one — the route is terminal with + :data:`CUSTOM_ROUTE_NO_ENDPOINT` and the bundle keeps neither the incoming + endpoint nor this record's credential, because pairing them is precisely how + a non-active provider's secret reached the ACTIVE provider's URL. + + The CREDENTIAL of an exact ``custom_providers[]`` row is exempt from the + same-endpoint rule above: it is resolved from that row's own ladder and is + never inherited from the runtime, because a same-slug keyed record may + declare the identical ``base_url`` and endpoint equality would then be + enough to hand the row somebody else's key. + + :data:`KEYLESS_CUSTOM_API_KEY` is substituted only once the selected record + reports that it DECLARES no credential source at all. A record that declares + one which produced nothing here (an unset ``${ENV}``, a ``key_env`` naming a + missing variable, an empty pool, an unbuildable ``key_cmd``) is sent without + a key rather than with a placeholder that guarantees a 401 and hides the + real cause. + + A named ``custom:`` route that NO authority owns fails closed: the + returned bundle keeps the named provider but carries no endpoint, no + credential, no pool/transport and no placeholder key, so an unknown slug can + never be routed through the ambient provider's connection. This holds even + when the ambient runtime LABELS itself with that same slug — a provider + string is self-assigned, not proof that the connection belongs to the slug. + """ + return _custom_provider_runtime_bundle_with_provenance( + resolved_provider, + resolved_api_key, + resolved_base_url, + runtime_provider, + lookup_provider=lookup_provider, + connection_resolver=connection_resolver, + )[0] + + +def _custom_provider_runtime_bundle_with_provenance( + resolved_provider: str | None, + resolved_api_key: str | None, + resolved_base_url: str | None, + runtime_provider: dict | None = None, + *, + lookup_provider: str | None = None, + connection_resolver=None, +) -> tuple[dict, dict | None]: + """:func:`merge_custom_provider_runtime_bundle` plus the record it selected. + + Returns ``(bundle, custom)`` where ``custom`` is the + :func:`resolve_custom_provider_bundle` result the merge applied (``None`` + when the route is not a named custom provider, or no record matched). + Callers that also need the provenance take it from here rather than + re-resolving: a second resolution re-reads the config and can mint a second + ``key_cmd`` token provider or select a different pool entry. + """ + _rt = runtime_provider if isinstance(runtime_provider, dict) else {} + bundle = { + "provider": resolved_provider, + "base_url": resolved_base_url, + "api_key": resolved_api_key, + "api_mode": _rt.get("api_mode"), + "acp_command": _rt.get("acp_command", _rt.get("command")), + "acp_args": _rt.get("acp_args", _rt.get("args")), + "credential_pool": _rt.get("credential_pool"), + # Routable until a named custom route proves otherwise. Non-custom + # routes legitimately leave the endpoint and/or credential to the + # runtime provider, so only the named-``custom:`` branches below can + # record a verdict here. + CUSTOM_ROUTE_ERROR_FIELD: None, + } + + lookup = lookup_provider or resolved_provider + if not (isinstance(lookup, str) and lookup.startswith("custom:")): + return bundle, None + + custom = resolve_custom_provider_bundle(lookup, connection_resolver=connection_resolver) + if custom is None: + return bundle, None + + if custom["status"] in CUSTOM_SELECTION_UNOWNED: + # NO authority owns this slug: not an exact ``custom_providers[]`` row, + # not a keyed ``providers:`` record, not a ``model:`` authority. Whatever + # is already in ``bundle`` was resolved by someone else and reached us as + # ambient state, so the whole bundle fails closed. + # + # A provider LABEL is not proof of ownership. The runtime dict's + # ``provider: "custom:ghost"`` is a self-assigned string on a connection + # this process authenticated for some other reason, and + # :func:`_connection_identity` falls back to ``resolved_provider`` — the + # very slug being looked up — so a caller that passes no runtime dict at + # all would match itself. Trusting either would let any unknown slug + # claim the ambient endpoint, credential, pool and transport simply by + # naming itself. Ownership is decided by config records, and there are + # none here. + # + # So: no endpoint and no credential; no ambient pool/transport/ + # wire-protocol carried alongside them; the provider stays the NAMED + # slug, because rewriting it to the generic ``custom`` would present an + # unresolvable route as a resolved one; and no KEYLESS_CUSTOM_API_KEY — + # ``keyless`` is False on an unowned bundle precisely so the placeholder + # cannot claim that an endpoint we never found is unauthenticated. + logger.warning( + "custom provider %s matches no custom_providers[] row, keyed " + "providers[] record or model: authority, and the resolved connection " + "belongs to %s; refusing to route it through that provider's " + "endpoint and credential", + lookup, + _connection_identity(_rt, resolved_provider) or "an unnamed provider", + ) + bundle["base_url"] = None + bundle["api_key"] = None + for field in CUSTOM_CONNECTION_SIDE_FIELDS: + bundle[field] = None + # Clearing the fields is NOT what makes this terminal — the constructor + # reads an empty pair as "route me somewhere else". Record the verdict so + # consumers stop before AIAgent instead. + bundle[CUSTOM_ROUTE_ERROR_FIELD] = _custom_route_verdict( + CUSTOM_ROUTE_UNOWNED, lookup + ) + return bundle, custom + + same_authority = _custom_bundle_endpoint_matches(custom, _rt) + # The positive tie: the runtime dict resolved the very endpoint the selected + # record declares. Never True on absence of evidence, so it is the only + # signal allowed to keep an endpoint this record did not supply itself. + endpoint_tied = _custom_runtime_endpoint_is_record_owned(custom, _rt) + + if custom["record"] is not None and not custom["endpoint_owned"]: + # A CONFIG RECORD owns this slug — an exact ``custom_providers[]`` row, a + # keyed ``providers['custom:']``, a raw ``providers:`` row (the + # standard v12 shape) or a ``model:`` authority — and it declares no + # usable endpoint. So this route HAS no endpoint: the one sitting in + # ``bundle`` was resolved for whoever the process was already talking to + # (the ACTIVE provider) and reached us as ambient state. + # + # Keeping it would pair THIS record's credential with THAT provider's + # URL — the user's prompt and a non-active provider's secret delivered to + # an endpoint neither the record nor the user named. The installed + # Agent's ``_match_new_style_provider()`` skips an endpoint-less raw + # record for exactly this reason. + # + # Nothing here may stand in for the missing endpoint: not the caller's + # ``resolved_base_url``, not a URL that merely compares equal (a gateway + # fronting two accounts is one host and two authorities), and not an + # absent runtime dict. Absence of contrary evidence is not provenance. + # So the route fails closed on its OWN name — the terminal + # ``custom_provider_endpoint_unresolved``, naming the setting to fix — + # rather than degrading into a send through the active provider. + bundle["base_url"] = None + bundle["api_key"] = None + for field in CUSTOM_CONNECTION_SIDE_FIELDS: + # The record keeps what it declares for ITSELF; every other side + # field belongs to the ambient authority whose endpoint was just + # refused, so it goes with it. Leaving those behind would hand the + # refused provider's credential pool and ACP transport to a route + # that is about to be reported unresolvable. + bundle[field] = custom["owned"].get(field) + bundle[CUSTOM_ROUTE_ERROR_FIELD] = _custom_route_verdict( + CUSTOM_ROUTE_NO_ENDPOINT, custom["provider_id"] + ) + logger.warning( + "custom provider %s resolved no endpoint of its own; refusing to pair " + "its credential with the connection resolved for %s", + custom["provider_id"], + _connection_identity(_rt, resolved_provider) or "another provider", + ) + return bundle, custom + + if custom["is_exact"]: + # An exact ``custom_providers[]`` row is authoritative for its slug: BOTH + # the endpoint (including None when the row's base_url is blank) and the + # credential are replaced, never merged — the sole exception being the + # runtime's spelling of the row's OWN endpoint, handled below. The + # credential comes from the + # ROW's own ladder (pool, api_key/``${ENV}``/``key_env``/ + # ``CUSTOM__API_KEY``, ``key_cmd``, host-gated env) and from + # nowhere else — including when ``same_authority`` holds. + # + # Endpoint equality is NOT record provenance. A same-slug keyed + # ``providers["custom:"]`` record is free to declare the very same + # ``base_url``, and the ambient runtime dict carrying that URL is then + # indistinguishable by URL from one the row itself resolved. Accepting + # ``_rt["api_key"]`` on that evidence hands the keyed row's credential to + # the exact row, which is precisely the split-authority merge this + # function exists to stop: the row's URL with the keyed row's key. + # + # So a row whose declared credential produced nothing here keeps + # ``api_key`` None and falls through to the terminal-route verdict below, + # naming the setting to fix, rather than silently borrowing a credential + # the row never declared. A row that declares NO credential at all is + # reported ``keyless`` by the resolution above and gets + # :data:`KEYLESS_CUSTOM_API_KEY` there — that is the row's own statement + # that the endpoint is unauthenticated, not an inherited key either. + bundle["api_key"] = custom["api_key"] or None + # Same endpoint rule as the keyed branch below: the row's endpoint wins + # unless the runtime dict is POSITIVELY tied to it — same identity, same + # normalized URL — in which case the runtime's spelling of the ROW's own + # endpoint is kept, because that is the form it can actually reach. This + # is not a merge: an endpoint the row did not declare can never survive + # here, since ``endpoint_tied`` is False whenever the row supplied none. + bundle["base_url"] = _rt.get("base_url") if endpoint_tied else custom["base_url"] + elif custom["record"] is not None: + # No exact row, but a CONFIG RECORD was selected: a keyed + # ``providers['custom:']``, a raw ``providers:`` row (the + # standard v12 shape) or a ``model:`` authority, picked as ONE complete + # record. The endpoint and the credential must therefore come from it + # together, decided by that record's OWN endpoint provenance and by + # nothing that happened to be in flight when it was selected. A record + # that declares NO endpoint never reaches here — the terminal branch + # above already refused it rather than let it borrow one. + # + # The record DECLARES an endpoint, so that endpoint and the credential + # sent to it are one authority's pair and the record's credential is the + # only one that may accompany it — including when the ambient runtime + # reports the SAME URL. A shared base_url is not shared provenance: a + # gateway fronting two accounts hands both providers one host and two + # different keys, and taking ``_rt["api_key"]`` there sends the ACTIVE + # provider's secret to the non-active record's endpoint. A record whose + # declared credential source resolved nothing keeps ``api_key`` None and + # falls through to the terminal verdict naming the setting to fix, rather + # than borrowing the ambient key across that host. + bundle["api_key"] = custom["api_key"] or None + # The record's endpoint wins unless the runtime dict is POSITIVELY tied + # to it — same identity, same normalized URL — in which case the + # runtime's spelling is kept, because a normalized form of the same + # endpoint is the one it can actually reach. Any other endpoint already + # in the bundle was resolved before this record was selected: absent + # provenance is not a tie, so it is displaced rather than left beside the + # credential this record just supplied. + bundle["base_url"] = _rt.get("base_url") if endpoint_tied else custom["base_url"] + else: + # No record behind the selection: an injected ``connection_resolver`` + # reported a bare URL/key pair and IS the whole authority on this path. + # There is no record to take endpoint provenance from, so this keeps the + # historical fill-only shape — the caller's already-resolved endpoint + # stands and the resolver fills only what it left empty (#2271). + if custom["base_url"]: + bundle["api_key"] = custom["api_key"] or None + elif same_authority and _rt.get("api_key"): + # The resolver reported NO endpoint, so the runtime's URL is filling + # a hole rather than being displaced — and the ambient endpoint's own + # credential is the coherent partner for it. + bundle["api_key"] = _rt.get("api_key") + elif custom["api_key"]: + bundle["api_key"] = custom["api_key"] + if custom["base_url"] and _rt.get("base_url") and not same_authority: + bundle["base_url"] = custom["base_url"] + elif not bundle["base_url"] and custom["base_url"]: + bundle["base_url"] = custom["base_url"] + + for field in CUSTOM_CONNECTION_SIDE_FIELDS: + if field in custom["owned"]: + # The selected record declares this field for ITSELF, so it wins over + # the ambient runtime even when the two disagree: an exact list row's + # ``api_mode: anthropic_messages`` is the row's wire protocol, not + # ambient noise, and a keyed record keeps the pool/ACP transport it + # declares. + bundle[field] = custom["owned"][field] + elif not same_authority: + # The record owns an endpoint the runtime did NOT resolve, so this + # value provably came from a different authority. Clearing it is what + # stops a custom HTTP endpoint inheriting Anthropic credential pooling + # and a Claude ACP subprocess. + bundle[field] = None + # else: the runtime resolved the SAME endpoint the record owns, so its + # value is same-authority and the seed above already kept it. + + if bundle["base_url"]: + # Route through the generic custom OpenAI-compatible client once the + # named provider has supplied the concrete endpoint. Keeping the provider + # as custom: would make Agent init synthesize invalid env-var hints + # like CUSTOM:SOMETHING-8000_API_KEY on keyless setups. + bundle["provider"] = "custom" + if not bundle["api_key"]: + if custom["keyless"]: + # Only now, with the record's full credential ladder exhausted + # (pool, api_key, key_env, CUSTOM__API_KEY, key_cmd, + # host-gated env) and the runtime offering nothing either, is the + # endpoint provably keyless. + bundle["api_key"] = KEYLESS_CUSTOM_API_KEY + else: + # The record DECLARES a credential source that did not yield one + # here (e.g. a ``key_cmd`` whose token provider could not be + # built). Substituting the placeholder would turn that into a + # silent 401 against an endpoint that does require auth; leave the + # credential unset so the failure names its real cause. + logger.warning( + "custom provider %s declares a credential source that produced " + "no key; sending without one rather than the keyless placeholder", + custom["provider_id"], + ) + + if not (bundle["api_key"] and bundle["base_url"]): + # A record OWNS this route, but the merge could not produce the complete + # explicit pair AIAgent needs to honour it. Leaving the hole would hand + # the send straight back to ``_routed_client_kwargs()``, which re-resolves + # the provider and can reach the ambient endpoint, a keyed row this record + # deliberately displaced, or the init-time fallback chain. Name the real + # cause instead and let the caller fail the turn. + bundle[CUSTOM_ROUTE_ERROR_FIELD] = _custom_route_verdict( + CUSTOM_ROUTE_NO_CREDENTIAL if bundle["base_url"] else CUSTOM_ROUTE_NO_ENDPOINT, + custom["provider_id"], + ) + logger.warning( + "custom provider %s resolved no routable connection (%s); refusing to " + "build an agent that would re-enter provider routing", + custom["provider_id"], + bundle[CUSTOM_ROUTE_ERROR_FIELD]["reason"], + ) + + return bundle, custom + + +def apply_custom_provider_connection_authority( + resolved_provider: str | None, + resolved_api_key: str | None, + resolved_base_url: str | None, + *, + lookup_provider: str | None = None, + connection_resolver=None, + runtime_provider: dict | None = None, +) -> tuple[str | None, str | None, str | None, bool]: + """Connection-only VIEW of :func:`merge_custom_provider_runtime_bundle`. + + Returns ``(provider, api_key, base_url, custom_owned)``, where + ``custom_owned`` reports whether a config-owned custom record supplied the + connection — False for a named route nothing owns, whose key and base_url + come back ``None`` rather than as the ambient runtime's. Kept for callers + that genuinely construct nothing else (capability/vision lookups); anything + that builds an AIAgent must take the whole bundle instead and honour its + :data:`CUSTOM_ROUTE_ERROR_FIELD` verdict, because the three connection fields + alone are neither a complete constructor contract nor a terminal refusal. + This view deliberately does NOT raise: a capability lookup asking "can this + route do vision" is not a constructor boundary and must not turn into a + failed turn. + """ + bundle, custom = _custom_provider_runtime_bundle_with_provenance( + resolved_provider, + resolved_api_key, + resolved_base_url, + runtime_provider, + lookup_provider=lookup_provider, + connection_resolver=connection_resolver, + ) + custom_owned = custom is not None and custom["status"] not in CUSTOM_SELECTION_UNOWNED + return bundle["provider"], bundle["api_key"], bundle["base_url"], custom_owned # Subprocess ACP transports (Cursor/Copilot CLI). Model IDs often contain '/' @@ -5158,10 +6594,17 @@ def set_auxiliary_model(task: str, provider: str, model: str, advanced: dict | N # (overlapping-id misroute, sibling of the resolve_model_provider # fix). For a named custom: selection, look up that # provider's OWN custom_providers[] entry directly. Note we do - # NOT route through model_with_provider_context here: the - # @custom::model form resolves base_url to None (the - # @provider path doesn't carry a custom entry's base_url), which - # would drop the base_url entirely. Fall back to the bare resolve + # NOT route through model_with_provider_context here: its + # @custom::model form re-resolves against the AMBIENT + # module-level `cfg`, while this path must read the in-lock + # `config_data` snapshot it is about to write. Those two can + # disagree whenever the cache is stale or the profile path + # changed, and ambient/global resolution while holding the + # non-reentrant _cfg_lock is exactly what the direct lookup below + # exists to avoid. (That qualified form DOES resolve a custom + # entry's own base_url as of the non-active #1806 fix -- the + # reason to stay direct here is the lock/snapshot, not a URL the + # qualified path cannot produce.) Fall back to the bare resolve # only for the unnamed `custom` case, which has no own entry. resolved_base_url = None if provider.startswith("custom:"): diff --git a/api/routes.py b/api/routes.py index 701a51d7..85b404ae 100644 --- a/api/routes.py +++ b/api/routes.py @@ -2970,6 +2970,128 @@ from api.request_diagnostics import RequestDiagnostics from api.system_health import build_system_health_payload +# ── Non-streaming custom-provider connection authority ─────────────────────── +# +# Several WebUI consumers build their own AIAgent outside the streaming path +# (POST /api/chat, manual compression, update summary, git commit message, +# handoff summary). They resolve a model/provider, ask the Hermes runtime +# provider for a connection, then had to apply the named ``custom:`` +# record's authority by hand. +# +# ``apply_custom_provider_connection_authority`` returns only the THREE +# connection fields, and three fields are not a complete constructor contract: +# AIAgent also takes ``api_mode`` (wire protocol), ``credential_pool`` +# (credential source) and ``acp_command``/``acp_args`` (subprocess transport). +# A consumer that replaced the endpoint and credential while leaving those to +# default — or, worse, to the ambient runtime — built a mixed-authority agent: +# an exact row's ``api_mode: anthropic_messages`` or its own pool never reached +# the constructor at all. These helpers carry the WHOLE bundle instead, exactly +# as ``api/streaming.py`` does for the streaming path. + +# The constructor-routing fields that travel with provider/base_url/api_key as +# one authority. Mirrors ``api.streaming._RUNTIME_BUNDLE_FIELDS`` and +# ``api.config.CUSTOM_CONNECTION_SIDE_FIELDS``. +_AGENT_BUNDLE_SIDE_FIELDS = ("api_mode", "acp_command", "acp_args", "credential_pool") + + +def _resolve_agent_connection_bundle( + resolved_provider, + resolved_api_key, + resolved_base_url, + runtime_provider=None, + *, + lookup_provider=None, +): + """Return the COMPLETE constructor-routing bundle for a non-streaming send. + + Keys: ``provider``, ``base_url``, ``api_key`` plus every field in + :data:`_AGENT_BUNDLE_SIDE_FIELDS`. Pass the whole dict to the constructor + via :func:`_agent_bundle_kwargs` — the endpoint/credential and the + transport/protocol/pool fields are ONE authority. + + ``runtime_provider`` is the dict ``resolve_runtime_provider`` returned. It + matters: the merge seeds the side fields from it and then decides, by + endpoint provenance, whether they are same-authority (keep) or the ambient + provider's (clear). Omitting it silently drops that signal. + + ``lookup_provider`` preserves the pre-canonicalization ``custom:`` + identity, since the merge rewrites a resolved bundle's provider to the + generic ``custom``. + + Raises :class:`api.config.CustomProviderRouteError` when the merge returns a + TERMINAL route verdict — a named ``custom:`` that resolved no complete + ``(api_key, base_url)`` pair. This is the single chokepoint for every + non-streaming and auxiliary consumer precisely because an incomplete bundle + is NOT a refusal at the constructor: AIAgent's ``_init_openai_client()`` + only honours an explicit pair when BOTH fields are truthy and otherwise + calls ``_routed_client_kwargs()``, which re-resolves a provider and can + reach the ambient endpoint or the init-time fallback chain. Returning the + bundle with a hole in it would therefore route the send somewhere the user + never asked for; raising here keeps the refusal terminal for all five call + sites (POST /api/chat, manual compression, update summary, git commit + message, handoff summary) without each having to remember to check. + + The exception subclasses ``ValueError``, so the existing ``except + ValueError`` / broad-``except`` handlers at those call sites already turn it + into a controlled 400 or a deterministic non-LLM fallback. + """ + return api_config.raise_for_custom_provider_route( + api_config.merge_custom_provider_runtime_bundle( + resolved_provider, + resolved_api_key, + resolved_base_url, + runtime_provider, + lookup_provider=lookup_provider or resolved_provider, + ) + ) + + +def _agent_bundle_kwargs(agent_cls, bundle): + """Return the bundle's side-field kwargs supported by ``agent_cls``. + + ``api_mode``/``acp_command``/``acp_args``/``credential_pool`` were added to + AIAgent over several releases, so gate each on the constructor signature the + way the streaming path does rather than raising TypeError against an older + hermes-agent build. Values come from the BUNDLE, never from the runtime + provider dict: a custom-provider override clears these, and reading them off + the runtime would re-introduce the authority the merge just replaced. + """ + import inspect as _inspect + + try: + params = set(_inspect.signature(agent_cls.__init__).parameters) + except (TypeError, ValueError): + return {} + return { + field: bundle[field] + for field in _AGENT_BUNDLE_SIDE_FIELDS + if field in params + } + + +def _auxiliary_main_runtime(bundle, model): + """Return the ``main_runtime`` an auxiliary client must receive for a bundle. + + When the auxiliary client answers, AIAgent is never built, so this dict is + the ONLY place the resolved authority reaches the wire. It therefore carries + the same whole bundle :func:`_agent_bundle_kwargs` hands the constructor — + endpoint and credential plus every field in + :data:`_AGENT_BUNDLE_SIDE_FIELDS`. Sending only provider/model/base_url/ + api_key silently downgraded an exact row's ``api_mode`` + (``anthropic_messages`` fell back to chat completions) and dropped the + credential pool/ACP transport that belong to the same record. + """ + runtime = { + "provider": bundle["provider"], + "model": model, + "base_url": bundle["base_url"], + "api_key": bundle["api_key"], + } + for field in _AGENT_BUNDLE_SIDE_FIELDS: + runtime[field] = bundle[field] + return runtime + + def _kanban_unknown_endpoint(handler, parsed, method: str) -> bool: """Return a Kanban-specific 404 for stale clients/obsolete endpoint shapes.""" return bad( @@ -14907,6 +15029,96 @@ def _resolve_new_session_workspace(body, visible_prev_session_id, profile=None): ) return str(workspace) + +def _llm_update_summary(system_prompt: str, user_prompt: str, active_profile: str | None = None) -> str: + from api import profiles as profiles_api + + profile = active_profile or profiles_api.get_active_profile_name() or "default" + + with profiles_api.profile_env_for_background_worker( + profile, + "update summary", + logger_override=logger, + ): + from api.config import ( + get_effective_default_model, + resolve_model_provider, + ) + + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ] + + _main_model, _main_provider, _main_base_url = resolve_model_provider(get_effective_default_model()) + _main_api_key = None + _rt = None + try: + from api.oauth import resolve_runtime_provider_with_anthropic_env_lock + from hermes_cli.runtime_provider import resolve_runtime_provider + + _rt = resolve_runtime_provider_with_anthropic_env_lock( + resolve_runtime_provider, + requested=_main_provider, + ) + _main_api_key = _rt.get("api_key") + if not _main_provider: + _main_provider = _rt.get("provider") + if not _main_base_url: + _main_base_url = _rt.get("base_url") + except Exception as _e: + logger.debug("update summary runtime provider resolution failed: %s", _e) + # Atomic custom-provider authority (see the /api/chat note): the record + # that supplies the endpoint must also supply the credential — and the + # wire protocol, credential pool and ACP transport that go with it. + _bundle = _resolve_agent_connection_bundle( + _main_provider, _main_api_key, _main_base_url, _rt + ) + _main_provider = _bundle["provider"] + _main_api_key = _bundle["api_key"] + _main_base_url = _bundle["base_url"] + + main_runtime = _auxiliary_main_runtime(_bundle, _main_model) + + ensure_agent_runtime_current() + try: + from agent.auxiliary_client import get_text_auxiliary_client + + aux_client, aux_model = get_text_auxiliary_client( + "compression", + main_runtime=main_runtime, + ) + if aux_client is not None and aux_model: + response = aux_client.chat.completions.create( + model=aux_model, + messages=messages, + ) + return str(response.choices[0].message.content or "").strip() + except Exception as _e: + logger.debug("update summary auxiliary model failed; falling back to main model: %s", _e) + + AIAgent = require_ai_agent_class() + + agent = AIAgent( + model=_main_model, + provider=_main_provider, + base_url=_main_base_url, + api_key=_main_api_key, + platform="webui", + quiet_mode=True, + enabled_toolsets=[], + session_id=f"updates-summary-{uuid.uuid4().hex[:8]}", + **_agent_bundle_kwargs(AIAgent, _bundle), + ) + result = agent.run_conversation( + user_message=user_prompt, + system_message=system_prompt, + conversation_history=[], + task_id=f"updates-summary-{uuid.uuid4().hex[:8]}", + ) + return str(result.get("final_response") or "").strip() + + def handle_post(handler, parsed) -> bool: """Handle all POST routes. Returns True if handled, False for 404.""" diag = RequestDiagnostics.maybe_start("POST", parsed.path, logger=logger, print_fn=getattr(handler, '_safe_webui_print', None)) @@ -17369,99 +17581,6 @@ def handle_post(handler, parsed) -> bool: updates = body.get("updates") if isinstance(body, dict) else {} target = body.get("target") if isinstance(body, dict) else None - def _llm_update_summary(system_prompt: str, user_prompt: str) -> str: - from api import profiles as profiles_api - - active_profile = profiles_api.get_active_profile_name() or "default" - - with profiles_api.profile_env_for_background_worker( - active_profile, - "update summary", - logger_override=logger, - ): - from api.config import ( - get_effective_default_model, - resolve_model_provider, - resolve_custom_provider_connection, - ) - - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}, - ] - - _main_model, _main_provider, _main_base_url = resolve_model_provider(get_effective_default_model()) - _main_api_key = None - try: - from api.oauth import resolve_runtime_provider_with_anthropic_env_lock - from hermes_cli.runtime_provider import resolve_runtime_provider - - _rt = resolve_runtime_provider_with_anthropic_env_lock( - resolve_runtime_provider, - requested=_main_provider, - ) - _main_api_key = _rt.get("api_key") - if not _main_provider: - _main_provider = _rt.get("provider") - if not _main_base_url: - _main_base_url = _rt.get("base_url") - except Exception as _e: - logger.debug("update summary runtime provider resolution failed: %s", _e) - if isinstance(_main_provider, str) and _main_provider.startswith("custom:"): - _cp_key, _cp_base = resolve_custom_provider_connection(_main_provider) - if not _main_api_key and _cp_key: - _main_api_key = _cp_key - if not _main_base_url and _cp_base: - _main_base_url = _cp_base - - main_runtime = { - "provider": _main_provider, - "model": _main_model, - "base_url": _main_base_url, - "api_key": _main_api_key, - } - - ensure_agent_runtime_current() - try: - from agent.auxiliary_client import get_text_auxiliary_client - - # Update summaries are a short text-compression/summarization task. - # Reuse the documented auxiliary.compression slot instead of - # inventing a WebUI-only auxiliary task name that users cannot - # discover in the Hermes Agent setup/config UI. - aux_client, aux_model = get_text_auxiliary_client( - "compression", - main_runtime=main_runtime, - ) - if aux_client is not None and aux_model: - response = aux_client.chat.completions.create( - model=aux_model, - messages=messages, - ) - return str(response.choices[0].message.content or "").strip() - except Exception as _e: - logger.debug("update summary auxiliary model failed; falling back to main model: %s", _e) - - AIAgent = require_ai_agent_class() - - agent = AIAgent( - model=_main_model, - provider=_main_provider, - base_url=_main_base_url, - api_key=_main_api_key, - platform="webui", - quiet_mode=True, - enabled_toolsets=[], - session_id=f"updates-summary-{uuid.uuid4().hex[:8]}", - ) - result = agent.run_conversation( - user_message=user_prompt, - system_message=system_prompt, - conversation_history=[], - task_id=f"updates-summary-{uuid.uuid4().hex[:8]}", - ) - return str(result.get("final_response") or "").strip() - return j(handler, summarize_update_payload(updates, llm_callback=_llm_update_summary, target=target)) # ── CLI session import (POST) ── @@ -24525,16 +24644,14 @@ def _handle_chat_sync(handler, body): AIAgent = require_ai_agent_class() with CHAT_LOCK: - from api.config import ( - resolve_model_provider, - resolve_custom_provider_connection, - ) + from api.config import resolve_model_provider _model, _provider, _base_url = resolve_model_provider( model_with_provider_context(s.model, getattr(s, "model_provider", None)) ) # Resolve API key via Hermes runtime provider (matches gateway behaviour) _api_key = None + _rt = None try: from api.oauth import resolve_runtime_provider_with_anthropic_env_lock from hermes_cli.runtime_provider import resolve_runtime_provider @@ -24554,12 +24671,35 @@ def _handle_chat_sync(handler, body): f"[webui] WARNING: resolve_runtime_provider failed: {_e}", flush=True, ) - if isinstance(_provider, str) and _provider.startswith("custom:"): - _cp_key, _cp_base = resolve_custom_provider_connection(_provider) - if not _api_key and _cp_key: - _api_key = _cp_key - if not _base_url and _cp_base: - _base_url = _cp_base + # Apply the named custom provider's OWN record atomically, as one + # COMPLETE bundle. The fill-only form this replaced kept a truthy + # runtime value, so a slug present in BOTH custom_providers[] and + # providers: sent the list row's URL with the keyed row's API key; + # the connection-only view that followed still truncated the record's + # api_mode / credential_pool / ACP transport before the constructor. + try: + _bundle = _resolve_agent_connection_bundle( + _provider, _api_key, _base_url, _rt + ) + except api_config.CustomProviderRouteError as _route_err: + # The named route resolved no usable connection. Constructing + # AIAgent with the incomplete pair would send this turn through + # ``_routed_client_kwargs()`` to whatever provider init resolves + # next, so answer with the actionable cause instead. 400, not + # 500: it is a user-fixable provider misconfiguration, exactly + # like the ambiguous-slug collision. + logger.warning( + "Chat blocked by unroutable custom provider: %s", _route_err.message + ) + return j(handler, { + "error": _route_err.message, + "type": "custom_provider_unroutable", + "reason": _route_err.reason, + "hint": _route_err.hint, + }, status=400) + _provider = _bundle["provider"] + _api_key = _bundle["api_key"] + _base_url = _bundle["base_url"] agent = AIAgent( model=_model, provider=_provider, @@ -24571,6 +24711,7 @@ def _handle_chat_sync(handler, body): quiet_mode=True, enabled_toolsets=_resolve_cli_toolsets(), session_id=s.session_id, + **_agent_bundle_kwargs(AIAgent, _bundle), ) from api.streaming import ( _WEBUI_PROGRESS_PROMPT, @@ -25138,7 +25279,6 @@ def _llm_git_commit_message(system_prompt: str, user_prompt: str, session=None) from api.config import ( get_effective_default_model, model_with_provider_context, - resolve_custom_provider_connection, resolve_model_provider, ) @@ -25151,6 +25291,7 @@ def _llm_git_commit_message(system_prompt: str, user_prompt: str, session=None) ) _main_model, _main_provider, _main_base_url = resolve_model_provider(model_for_resolution) _main_api_key = None + _rt = None try: from api.oauth import resolve_runtime_provider_with_anthropic_env_lock from hermes_cli.runtime_provider import resolve_runtime_provider @@ -25166,23 +25307,21 @@ def _llm_git_commit_message(system_prompt: str, user_prompt: str, session=None) _main_base_url = _rt.get("base_url") except Exception as _e: logger.debug("git commit message runtime provider resolution failed: %s", _e) - if isinstance(_main_provider, str) and _main_provider.startswith("custom:"): - _cp_key, _cp_base = resolve_custom_provider_connection(_main_provider) - if not _main_api_key and _cp_key: - _main_api_key = _cp_key - if not _main_base_url and _cp_base: - _main_base_url = _cp_base + # Atomic custom-provider authority (see the /api/chat note): the record + # that supplies the endpoint must also supply the credential — and the + # wire protocol, credential pool and ACP transport that go with it. + _bundle = _resolve_agent_connection_bundle( + _main_provider, _main_api_key, _main_base_url, _rt + ) + _main_provider = _bundle["provider"] + _main_api_key = _bundle["api_key"] + _main_base_url = _bundle["base_url"] messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ] - main_runtime = { - "provider": _main_provider, - "model": _main_model, - "base_url": _main_base_url, - "api_key": _main_api_key, - } + main_runtime = _auxiliary_main_runtime(_bundle, _main_model) ensure_agent_runtime_current() try: from agent.auxiliary_client import get_text_auxiliary_client @@ -25211,6 +25350,7 @@ def _llm_git_commit_message(system_prompt: str, user_prompt: str, session=None) quiet_mode=True, enabled_toolsets=[], session_id=f"git-commit-message-{uuid.uuid4().hex[:8]}", + **_agent_bundle_kwargs(AIAgent, _bundle), ) result = agent.run_conversation( user_message=user_prompt, @@ -27230,6 +27370,7 @@ def _handle_session_compress(handler, body): ) resolved_api_key = None + _rt = None try: _rt = resolve_runtime_provider_with_anthropic_env_lock( _runtime_provider.resolve_runtime_provider, @@ -27243,12 +27384,16 @@ def _handle_session_compress(handler, body): except Exception as _e: logger.warning("resolve_runtime_provider failed for compression: %s", _e) - if isinstance(resolved_provider, str) and resolved_provider.startswith("custom:"): - _cp_key, _cp_base = _cfg.resolve_custom_provider_connection(resolved_provider) - if not resolved_api_key and _cp_key: - resolved_api_key = _cp_key - if not resolved_base_url and _cp_base: - resolved_base_url = _cp_base + # Atomic custom-provider authority: the deterministic list-row URL must + # not be paired with a keyed/runtime API key (see the /api/chat note), + # and the record's own api_mode / credential_pool / ACP transport must + # reach the constructor with it rather than being truncated away. + _bundle = _resolve_agent_connection_bundle( + resolved_provider, resolved_api_key, resolved_base_url, _rt + ) + resolved_provider = _bundle["provider"] + resolved_api_key = _bundle["api_key"] + resolved_base_url = _bundle["base_url"] if not resolved_api_key: return bad(handler, "No provider configured -- cannot compress.") @@ -27277,6 +27422,7 @@ def _handle_session_compress(handler, body): quiet_mode=True, enabled_toolsets=_resolve_cli_toolsets(), session_id=sid, + **_agent_bundle_kwargs(AIAgent, _bundle), ) compressed = agent.context_compressor.compress( original_messages, @@ -27905,7 +28051,7 @@ def _handle_handoff_summary(handler, body): # of the resolve_model_provider fix). model_with_provider_context # encodes it as @custom:A:model so the resolver honors the session's # endpoint; base_url is backfilled from that provider's own custom - # entry by the resolve_custom_provider_connection block below. + # entry by the custom-provider authority block below. session_model_provider = getattr(s_obj, "model_provider", None) except Exception: pass @@ -27916,6 +28062,7 @@ def _handle_handoff_summary(handler, body): resolved_model, resolved_provider, resolved_base_url = _cfg.resolve_model_provider(model_for_resolution) resolved_api_key = None + _rt = None try: _rt = resolve_runtime_provider_with_anthropic_env_lock( _runtime_provider.resolve_runtime_provider, @@ -27929,12 +28076,16 @@ def _handle_handoff_summary(handler, body): except Exception as _e: logger.warning("resolve_runtime_provider failed for handoff summary: %s", _e) - if isinstance(resolved_provider, str) and resolved_provider.startswith("custom:"): - _cp_key, _cp_base = _cfg.resolve_custom_provider_connection(resolved_provider) - if not resolved_api_key and _cp_key: - resolved_api_key = _cp_key - if not resolved_base_url and _cp_base: - resolved_base_url = _cp_base + # Atomic custom-provider authority (see the /api/chat note): the session's + # own custom_providers[] row owns BOTH the endpoint and the credential, + # plus the api_mode / credential_pool / ACP transport that travel with + # them as one constructor bundle. + _bundle = _resolve_agent_connection_bundle( + resolved_provider, resolved_api_key, resolved_base_url, _rt + ) + resolved_provider = _bundle["provider"] + resolved_api_key = _bundle["api_key"] + resolved_base_url = _bundle["base_url"] if not resolved_api_key: summary_text = _fallback_handoff_summary(msgs) @@ -27965,6 +28116,7 @@ def _handle_handoff_summary(handler, body): quiet_mode=True, enabled_toolsets=[], session_id=sid, + **_agent_bundle_kwargs(AIAgent, _bundle), ) summary_system_prompt = ( diff --git a/api/streaming.py b/api/streaming.py index ba2223fd..beab649b 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -45,6 +45,13 @@ from api.config import ( SESSION_AGENT_LOCKS, SESSION_AGENT_LOCKS_LOCK, resolve_model_provider, resolve_custom_provider_connection, + apply_custom_provider_connection_authority, + merge_custom_provider_runtime_bundle, + CustomProviderRouteError, + CUSTOM_ROUTE_NO_CREDENTIAL, + CUSTOM_ROUTE_NO_ENDPOINT, + custom_provider_route_error, + raise_for_custom_provider_route, model_with_provider_context, warm_models_catalog_provenance_if_cold, load_settings, @@ -271,7 +278,6 @@ def _cancel_event_payload( # (api/profiles.py:715). _ENV_LOCK = threading.Lock() -_KEYLESS_CUSTOM_API_KEY = "dummy-key" _STREAM_WRITEBACK_DIAG_DEFAULT_THRESHOLD_MS = 250.0 _STREAMING_CRON_PROFILE_HOME: contextvars.ContextVar[str | None] = contextvars.ContextVar( @@ -570,50 +576,121 @@ def _apply_profile_home_context_to_streaming_model( return model, provider_context, False -def _resolve_custom_provider_runtime_overrides( +def _resolve_custom_provider_connection_authority( resolved_provider: str | None, resolved_api_key: str | None, resolved_base_url: str | None, profile_name: str | None = None, -) -> tuple[str | None, str | None, str | None]: - """Return provider/key/base_url overrides for ``custom:*`` endpoints. + custom_provider_lookup: str | None = None, +) -> tuple[str | None, str | None, str | None, bool]: + """Return ``(provider, api_key, base_url, custom_owned)`` for ``custom:*``. - Hermes Agent treats named custom providers as routing hints around an - OpenAI-compatible base URL. Local OpenAI-compatible servers often run - without authentication, so a missing key should not fail before the first - request; pass a harmless placeholder to the SDK and let the endpoint accept - it or return its own auth error. + Thin profile-scoped wrapper around + :func:`api.config.apply_custom_provider_connection_authority`, which holds the + atomic-replacement rules (exact ``custom_providers[]`` row owns BOTH the + endpoint and the credential; keyed/``model:`` fallbacks only fill gaps, as one + same-record bundle). - ``profile_name`` binds the SESSION's own profile while - ``resolve_custom_provider_connection`` reads the endpoint AND the credential. - That helper resolves both from ONE ``get_config()``/env snapshot, so binding - the profile here keeps them from splitting across profiles: on a detached - worker thread (which doesn't inherit the request-profile TLS/env) an unbound - call would read the DEFAULT profile and pair a named profile's endpoint with - the default profile's API key (finding #3). No-op for the default/root - profile; when ``profile_name`` is None the ambient scope (if any) is used. + ``profile_name`` binds the SESSION's own profile while the connection is + resolved. That lookup reads the endpoint AND the credential from ONE + ``get_config()``/env snapshot, so binding the profile here keeps them from + splitting across profiles: on a detached worker thread (which doesn't inherit + the request-profile TLS/env) an unbound call would read the DEFAULT profile + and pair a named profile's endpoint with the default profile's API key + (finding #3). No-op for the default/root profile; when ``profile_name`` is + None the ambient scope (if any) is used. + + ``custom_provider_lookup`` carries the session's pre-canonicalization + identity, so a provider already rewritten to plain ``"custom"`` by an earlier + resolve still selects its own named record on a retry. + + ``custom_owned`` is True when a config-owned custom record supplied the + connection; see :func:`_resolve_runtime_connection_bundle` for why callers + must then clear the runtime-owned side fields. """ - if not (isinstance(resolved_provider, str) and resolved_provider.startswith("custom:")): - return resolved_provider, resolved_api_key, resolved_base_url + lookup_provider = custom_provider_lookup or resolved_provider + if not (isinstance(lookup_provider, str) and lookup_provider.startswith("custom:")): + return resolved_provider, resolved_api_key, resolved_base_url, False from api import profiles as _profiles_api with _profiles_api.profile_scope_for_detached_worker( profile_name, "custom provider connection", logger_override=logger ): - _cp_key, _cp_base = resolve_custom_provider_connection(resolved_provider) - if not resolved_api_key and _cp_key: - resolved_api_key = _cp_key - if not resolved_base_url and _cp_base: - resolved_base_url = _cp_base - if resolved_base_url: - # Route through the generic custom OpenAI-compatible client once the - # named provider has supplied the concrete endpoint. Keeping the - # provider as custom: would make Agent init synthesize invalid - # env-var hints like CUSTOM:SOMETHING-8000_API_KEY on keyless setups. - resolved_provider = "custom" - if not resolved_api_key: - resolved_api_key = _KEYLESS_CUSTOM_API_KEY - return resolved_provider, resolved_api_key, resolved_base_url + return apply_custom_provider_connection_authority( + resolved_provider, + resolved_api_key, + resolved_base_url, + lookup_provider=lookup_provider, + # Pass THIS module's binding so tests (and any future shim) that patch + # ``streaming.resolve_custom_provider_connection`` still take effect. + connection_resolver=resolve_custom_provider_connection, + ) + + +def _resolve_custom_provider_runtime_overrides( + resolved_provider: str | None, + resolved_api_key: str | None, + resolved_base_url: str | None, + profile_name: str | None = None, + custom_provider_lookup: str | None = None, +) -> tuple[str | None, str | None, str | None]: + """Return provider/key/base_url overrides for ``custom:*`` endpoints. + + Legacy three-value view of + :func:`_resolve_custom_provider_connection_authority`. Prefer + :func:`_resolve_runtime_connection_bundle` in agent-construction paths: the + three fields alone are NOT a complete constructor bundle, so a caller that + keeps the ambient runtime's ``credential_pool`` / ``api_mode`` / ACP + command+args beside them still sends a mixed-authority agent to the custom + endpoint. + """ + return _resolve_custom_provider_connection_authority( + resolved_provider, + resolved_api_key, + resolved_base_url, + profile_name=profile_name, + custom_provider_lookup=custom_provider_lookup, + )[:3] + + +# The constructor-routing fields AIAgent takes from the resolved runtime +# provider. They travel together: a bundle that replaces the endpoint/credential +# but leaves these behind builds an agent whose transport (ACP subprocess), +# wire protocol (api_mode) or credential source (pool) still points at the +# previous authority. +_RUNTIME_BUNDLE_FIELDS = ('api_mode', 'acp_command', 'acp_args', 'credential_pool') + + +def _resolve_runtime_connection_bundle( + resolved_provider: str | None, + resolved_api_key: str | None, + resolved_base_url: str | None, + runtime_provider: dict | None, + profile_name: str | None = None, + custom_provider_lookup: str | None = None, +) -> dict: + """Return the COMPLETE constructor-routing bundle for one send attempt. + + Keys: ``provider``, ``base_url``, ``api_key`` plus every field in + :data:`_RUNTIME_BUNDLE_FIELDS`. Callers must apply the whole dict — that is + the point: the endpoint/credential and the transport/protocol/pool fields + are one authority, and the agent-cache signature must be derived from the + same dict so a bundle change always mints a new agent. + """ + lookup_provider = custom_provider_lookup or resolved_provider + from api import profiles as _profiles_api + + with _profiles_api.profile_scope_for_detached_worker( + profile_name, "custom provider connection", logger_override=logger + ): + return merge_custom_provider_runtime_bundle( + resolved_provider, + resolved_api_key, + resolved_base_url, + runtime_provider, + lookup_provider=lookup_provider, + connection_resolver=resolve_custom_provider_connection, + ) def _same_base_url_endpoint(url_a: str, url_b: str) -> bool: @@ -645,6 +722,7 @@ def _runtime_preferred_base_url( runtime_provider: dict | None, resolved_provider: str | None, configured_base_url: str | None, + session_requested_provider: str | None = None, ) -> str | None: """Prefer the runtime-normalized base_url, but never override an explicit configured endpoint that points somewhere genuinely different. @@ -670,7 +748,8 @@ def _runtime_preferred_base_url( return runtime_base_url provider_id = str( - resolved_provider + session_requested_provider + or resolved_provider or (runtime_provider or {}).get("provider") or "" ).strip().lower() @@ -1429,6 +1508,41 @@ def _result_reports_compression_snapshot_stale(result) -> bool: ) +def _custom_provider_route_classification(error) -> dict: + """WebUI apperror classification for a terminal custom-provider route. + + ``error`` is either a :class:`CustomProviderRouteError` or the verdict dict + :func:`custom_provider_route_error` read off a bundle — the retry paths hold + the verdict without ever raising, so both shapes classify identically. + """ + if isinstance(error, dict): + reason = error.get('reason') + hint = error.get('hint') or '' + message = error.get('message') or '' + else: + reason = getattr(error, 'reason', None) + hint = getattr(error, 'hint', '') or '' + message = getattr(error, 'message', None) or str(error) + if reason == CUSTOM_ROUTE_NO_CREDENTIAL: + label = 'Provider credential unavailable' + elif reason == CUSTOM_ROUTE_NO_ENDPOINT: + label = 'Provider endpoint unavailable' + else: + label = 'Provider not configured' + return { + 'label': label, + 'type': 'provider_unroutable', + # The STRUCTURED verdict (``custom_provider_endpoint_unresolved`` / + # ``custom_provider_no_credential``) travels beside the prose so the + # emitted apperror can carry it too. ``type`` alone says only "this route + # is unroutable"; which of the two settings to fix lives here, and the + # non-streaming /api/chat refusal already answers it as ``reason``. + 'reason': reason, + 'hint': hint, + 'message': message, + } + + def _classify_provider_error( err_str: str, exc=None, @@ -1451,6 +1565,12 @@ def _classify_provider_error( err_str = str(_probe_text or err_str or '') _err_lower = err_str.lower() _exc_name = type(exc).__name__ if exc is not None else '' + if isinstance(exc, CustomProviderRouteError): + # An unroutable named ``custom:`` route. Classified BEFORE any text + # matching so it can never be read as a 401 — the auth branch would run + # credential self-heal, which re-resolves a provider and is exactly the + # re-routing this verdict exists to prevent. + return _custom_provider_route_classification(exc) _result_is_compression_snapshot_stale = ( _result_reports_compression_snapshot_stale(result) ) @@ -8745,6 +8865,54 @@ def _agent_cache_api_key_sig(resolved_api_key, credential_pool) -> str: return _hashlib.sha256((resolved_api_key or '').encode()).hexdigest()[:16] +def _compute_agent_cache_signature( + resolved_model: str | None, + resolved_api_key: str | None, + resolved_base_url: str | None, + resolved_provider: str | None, + runtime_bundle: dict | None, + max_iterations_cfg=None, + max_tokens_cfg=None, + fallback_resolved=None, + toolsets=None, + reasoning_config=None, + main_request_overrides=None, + _main_request_overrides=None, + prefill_context=None, + profile_home: str | None = None, + safe_profile_runtime_env: dict | None = None, +) -> str: + """Return the SHA256 signature identifying an agent's constructor routing state.""" + import hashlib as _hashlib + import json as _json + _bundle = runtime_bundle if isinstance(runtime_bundle, dict) else {} + _credential_pool = _bundle.get('credential_pool') + _env = safe_profile_runtime_env if isinstance(safe_profile_runtime_env, dict) else {} + _main_request_overrides = main_request_overrides if main_request_overrides is not None else _main_request_overrides + _sig_blob = _json.dumps([ + resolved_model or '', + _agent_cache_api_key_sig(resolved_api_key, _credential_pool), + resolved_base_url or '', + resolved_provider or '', + _bundle.get('api_mode') or '', + _bundle.get('acp_command') or '', + _bundle.get('acp_args') or [], + bool(_credential_pool), + max_iterations_cfg or '', + max_tokens_cfg or '', + fallback_resolved or {}, + sorted(toolsets) if toolsets else [], + reasoning_config or {}, + _main_request_overrides or {}, + _public_prefill_context_status(prefill_context), + profile_home or '', + _env.get('TERMINAL_ENV', '') or '', + _env.get('TERMINAL_SSH_HOST', '') or '', + _env.get('TERMINAL_SSH_USER', '') or '', + ], sort_keys=True) + return _hashlib.sha256(_sig_blob.encode()).hexdigest()[:16] + + def _lifecycle_commit_session_memory(session_id: str, *, agent=None, wait: bool = False) -> bool: from api.session_lifecycle import commit_session_memory @@ -10348,6 +10516,9 @@ def _run_agent_streaming( # Resolve API key via Hermes runtime provider (matches gateway behaviour). # Pass the resolved provider so non-default providers get their own credentials. resolved_api_key = None + # Default to an empty runtime dict so the constructor-routing + # bundle below stays buildable when resolution raises. + _rt = {} try: from api.oauth import resolve_runtime_provider_with_anthropic_env_lock from hermes_cli.runtime_provider import resolve_runtime_provider @@ -10371,11 +10542,38 @@ def _run_agent_streaming( # Preserve the pre-canonicalization identity so image routing can # still select the exact custom_providers entry after the rewrite # to "custom" below. + # + # Resolve ONE complete constructor-routing bundle here and let + # every downstream field (including the agent-cache signature) + # read from it. Resolving only provider/key/base_url left the + # runtime-owned credential_pool / api_mode / ACP command+args + # beside a replaced custom endpoint, so the agent was built with + # a mixed authority: the list row's URL and key, but the ambient + # provider's transport, wire protocol and credential source. _session_requested_provider = resolved_provider - resolved_provider, resolved_api_key, resolved_base_url = _resolve_custom_provider_runtime_overrides( - resolved_provider, resolved_api_key, resolved_base_url, + _runtime_bundle = _resolve_runtime_connection_bundle( + resolved_provider, resolved_api_key, resolved_base_url, _rt, profile_name=_resolved_profile_name, + custom_provider_lookup=_session_requested_provider, ) + # Stop HERE on a terminal route verdict — before the agent + # kwargs, before _AIAgent(), before the agent-cache write. A + # named custom: that resolved no complete + # (api_key, base_url) pair is not "fail closed" at the + # constructor: AIAgent honours an explicit pair only when BOTH + # are truthy and otherwise calls _routed_client_kwargs(), which + # re-resolves a provider and can land on the ambient endpoint or + # the init-time fallback chain. Raising sends this turn to the + # outer handler, which classifies it via + # _custom_provider_route_classification() and emits a controlled + # provider / missing-credential error — and, because we never + # reach SESSION_AGENT_CACHE, the cache is never poisoned with an + # agent built on an unroutable bundle that later turns would + # silently reuse. + raise_for_custom_provider_route(_runtime_bundle) + resolved_provider = _runtime_bundle['provider'] + resolved_api_key = _runtime_bundle['api_key'] + resolved_base_url = _runtime_bundle['base_url'] # Read per-profile config at call time (not module-level snapshot). # The streaming worker is a detached thread that does NOT inherit the @@ -10582,15 +10780,18 @@ def _run_agent_streaming( _agent_kwargs['max_tokens'] = _max_tokens_cfg if 'request_overrides' in _agent_params and _main_request_overrides: _agent_kwargs['request_overrides'] = _main_request_overrides - # Params added in newer hermes-agent — skip if not supported + # Params added in newer hermes-agent — skip if not supported. + # Read from the resolved bundle, NOT from _rt: a custom-provider + # override clears these, and taking them straight off the runtime + # provider would re-introduce the authority it replaced. if 'api_mode' in _agent_params: - _agent_kwargs['api_mode'] = _rt.get('api_mode') + _agent_kwargs['api_mode'] = _runtime_bundle['api_mode'] if 'acp_command' in _agent_params: - _agent_kwargs['acp_command'] = _rt.get('command') + _agent_kwargs['acp_command'] = _runtime_bundle['acp_command'] if 'acp_args' in _agent_params: - _agent_kwargs['acp_args'] = _rt.get('args') + _agent_kwargs['acp_args'] = _runtime_bundle['acp_args'] if 'credential_pool' in _agent_params: - _agent_kwargs['credential_pool'] = _rt.get('credential_pool') + _agent_kwargs['credential_pool'] = _runtime_bundle['credential_pool'] # Pin Honcho memory sessions to the stable WebUI session ID. # Without this, 'per-session' Honcho strategy creates a new Honcho # session on every streaming request because HonchoSessionManager is @@ -10605,41 +10806,26 @@ def _run_agent_streaming( agent = _AIAgent(**_agent_kwargs) logger.debug('[webui] Created ephemeral agent for session %s', session_id) else: - import hashlib as _hashlib - import json as _json from api.config import SESSION_AGENT_CACHE, SESSION_AGENT_CACHE_LOCK - _credential_pool = _rt.get('credential_pool') - _sig_blob = _json.dumps([ - resolved_model or '', - _agent_cache_api_key_sig(resolved_api_key, _credential_pool), - resolved_base_url or '', - resolved_provider or '', - _rt.get('api_mode') or '', - _rt.get('command') or '', - _rt.get('args') or [], - bool(_credential_pool), - _max_iterations_cfg or '', - _max_tokens_cfg or '', - _fallback_resolved or {}, - sorted(_toolsets) if _toolsets else [], - _reasoning_config or {}, - _main_request_overrides or {}, - _public_prefill_context_status(_prefill_context), - # #1897: profile_home is part of the agent's identity because - # AIAgent caches `_cached_system_prompt` from `load_soul_md()` - # at construction time, sourced from HERMES_HOME. Same-session - # profile switches keep `session_id` stable, so without this - # field the cached agent silently retains the previous - # profile's SOUL.md (and any other profile-scoped context). - _profile_home or '', - # Terminal backend identity: sessions switching between - # remote (SSH) and local backends must not reuse a cached - # agent that carries stale terminal env vars (#5937). - _safe_profile_runtime_env.get('TERMINAL_ENV', '') or '', - _safe_profile_runtime_env.get('TERMINAL_SSH_HOST', '') or '', - _safe_profile_runtime_env.get('TERMINAL_SSH_USER', '') or '', - ], sort_keys=True) - _agent_sig = _hashlib.sha256(_sig_blob.encode()).hexdigest()[:16] + # Signature fields come from the SAME bundle the constructor + # received, so a cleared/overridden field always mints a new + # agent instead of reusing one built on the prior authority. + _agent_sig = _compute_agent_cache_signature( + resolved_model, + resolved_api_key, + resolved_base_url, + resolved_provider, + _runtime_bundle, + max_iterations_cfg=_max_iterations_cfg, + max_tokens_cfg=_max_tokens_cfg, + fallback_resolved=_fallback_resolved, + toolsets=_toolsets, + reasoning_config=_reasoning_config, + main_request_overrides=_main_request_overrides, + prefill_context=_prefill_context, + profile_home=_profile_home, + safe_profile_runtime_env=_safe_profile_runtime_env, + ) agent = None _identity_mismatch_entry = None @@ -11497,7 +11683,8 @@ def _run_agent_streaming( if not resolved_provider: resolved_provider = _heal_rt.get('provider') resolved_base_url = _runtime_preferred_base_url( - _heal_rt, resolved_provider, configured_base_url + _heal_rt, resolved_provider, configured_base_url, + session_requested_provider=_session_requested_provider, ) # Preserve the session's original pre-canonicalization # provider identity (captured at first resolve) so a @@ -11506,21 +11693,64 @@ def _run_agent_streaming( # (e.g. the provider was first discovered on this heal). if not _session_requested_provider: _session_requested_provider = resolved_provider - resolved_provider, resolved_api_key, resolved_base_url = _resolve_custom_provider_runtime_overrides( - resolved_provider, resolved_api_key, resolved_base_url, + # Rebuild the COMPLETE bundle, not just the three + # connection fields: the self-heal re-resolve can + # return a different runtime authority, and a + # custom-provider override must clear its + # credential_pool / api_mode / ACP fields here too. + _runtime_bundle = _resolve_runtime_connection_bundle( + resolved_provider, resolved_api_key, resolved_base_url, _heal_rt, profile_name=_resolved_profile_name, + custom_provider_lookup=_session_requested_provider, ) + # The re-resolve can turn a previously routable named + # route terminal (the record's key_cmd stopped + # minting, the pool drained, the row was edited + # mid-turn). Abandon the retry BEFORE _AIAgent and + # before the SESSION_AGENT_CACHE write below: healing + # a 401 by handing the constructor an incomplete pair + # is exactly the re-routing this verdict exists to + # stop, and caching that agent would carry it into + # every later turn. Raising reaches the outer handler, + # which emits the controlled provider / + # missing-credential error instead of the 401. + raise_for_custom_provider_route(_runtime_bundle) + resolved_provider = _runtime_bundle['provider'] + resolved_api_key = _runtime_bundle['api_key'] + resolved_base_url = _runtime_bundle['base_url'] # Rebuild agent kwargs and create a fresh agent _agent_kwargs['api_key'] = resolved_api_key _agent_kwargs['base_url'] = resolved_base_url _agent_kwargs['model'] = resolved_model _agent_kwargs['provider'] = resolved_provider _replace_session_db_in_kwargs(_agent_kwargs, _state_db_path) + if 'api_mode' in _agent_params: + _agent_kwargs['api_mode'] = _runtime_bundle['api_mode'] + if 'acp_command' in _agent_params: + _agent_kwargs['acp_command'] = _runtime_bundle['acp_command'] + if 'acp_args' in _agent_params: + _agent_kwargs['acp_args'] = _runtime_bundle['acp_args'] if 'credential_pool' in _agent_params: - _agent_kwargs['credential_pool'] = _heal_rt.get('credential_pool') + _agent_kwargs['credential_pool'] = _runtime_bundle['credential_pool'] agent = _AIAgent(**_agent_kwargs) with STREAMS_LOCK: AGENT_INSTANCES[stream_id] = agent + _agent_sig = _compute_agent_cache_signature( + resolved_model, + resolved_api_key, + resolved_base_url, + resolved_provider, + _runtime_bundle, + max_iterations_cfg=_max_iterations_cfg, + max_tokens_cfg=_max_tokens_cfg, + fallback_resolved=_fallback_resolved, + toolsets=_toolsets, + reasoning_config=_reasoning_config, + main_request_overrides=_main_request_overrides, + prefill_context=_prefill_context, + profile_home=_profile_home, + safe_profile_runtime_env=_safe_profile_runtime_env, + ) from api.config import SESSION_AGENT_CACHE as _SAC, SESSION_AGENT_CACHE_LOCK as _SAC_L with _SAC_L: _SAC[session_id] = (agent, _agent_sig) @@ -12747,9 +12977,24 @@ def _run_agent_streaming( _exc_is_compression_snapshot_stale = ( _classification['type'] == 'compression_snapshot_stale' ) + _exc_is_provider_unroutable = _classification['type'] == 'provider_unroutable' + + # The structured terminal reason for an unroutable named custom route, + # stamped onto the emitted payload below. Kept separate from _exc_type + # because every unroutable route shares one type and differs only here. + _exc_route_reason = None # The user hint still points to Settings / `hermes model` from _classify_provider_error(). - if _exc_is_quota: + if _exc_is_provider_unroutable: + # Checked FIRST so the terminal route verdict can never be flattened + # into the generic 'Error' tail of this chain: its hint names the + # exact provider setting to fix, which is the only actionable thing + # the user gets on an unroutable custom:. + _exc_label, _exc_type, _exc_hint = ( + _classification['label'], _classification['type'], _classification['hint'], + ) + _exc_route_reason = _classification.get('reason') + elif _exc_is_quota: _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], ) @@ -12763,6 +13008,15 @@ def _run_agent_streaming( ) elif _exc_is_auth: _heal_stale_classification = None + # Set when the self-heal re-resolve produces a TERMINAL route + # verdict. Unlike the two in-flight sites, this branch runs INSIDE + # the outer exception handler, so raising here would escape the + # generator with no error event at all. Record the verdict instead + # and let the abandoned heal fall through to the emission below, + # which prefers this classification over the generic + # "Authentication error" — the 401 is a symptom, the unroutable + # named provider is the cause the user can actually fix. + _heal_route_classification = None if not _self_healed: # ── Credential self-heal on 401 (#1401) ── # Bind the session's profile so the self-heal re-resolve AND the @@ -12786,7 +13040,8 @@ def _run_agent_streaming( if not resolved_provider: resolved_provider = _heal_rt.get('provider') resolved_base_url = _runtime_preferred_base_url( - _heal_rt, resolved_provider, configured_base_url + _heal_rt, resolved_provider, configured_base_url, + session_requested_provider=_session_requested_provider, ) # Preserve the session's original pre-canonicalization provider # identity (captured at first resolve) so a named custom:slug @@ -12794,10 +13049,41 @@ def _run_agent_streaming( # Only initialize when empty. if not _session_requested_provider: _session_requested_provider = resolved_provider - resolved_provider, resolved_api_key, resolved_base_url = _resolve_custom_provider_runtime_overrides( - resolved_provider, resolved_api_key, resolved_base_url, + # Rebuild the COMPLETE bundle (see the returned-error retry + # above): replacing only provider/key/base_url would leave + # the pre-heal credential_pool / api_mode / ACP fields on a + # custom endpoint that owns none of them. + _runtime_bundle = _resolve_runtime_connection_bundle( + resolved_provider, resolved_api_key, resolved_base_url, _heal_rt, profile_name=_resolved_profile_name, + custom_provider_lookup=_session_requested_provider, ) + _heal_route_verdict = custom_provider_route_error(_runtime_bundle) + if _heal_route_verdict is not None: + # The refreshed credential did not produce a routable + # named connection. Abandon the heal BEFORE _AIAgent and + # before the SESSION_AGENT_CACHE write below — building + # here would hand the constructor an incomplete pair and + # let _routed_client_kwargs() pick a provider of its own, + # and caching that agent would keep doing so on every + # later turn in this session. + _heal_route_classification = ( + _custom_provider_route_classification(_heal_route_verdict) + ) + logger.warning( + '[webui] self-heal (except path): abandoning retry — %s', + _heal_route_classification['message'], + ) + # Clearing _heal_rt closes the retry block below. We must + # NOT raise from inside the outer exception handler: that + # would escape the generator and the client would get no + # error event at all. + _heal_rt = None + + if _heal_rt is not None: + resolved_provider = _runtime_bundle['provider'] + resolved_api_key = _runtime_bundle['api_key'] + resolved_base_url = _runtime_bundle['base_url'] # Build a fresh agent with the new credentials _heal_kwargs = dict(_agent_kwargs) if '_agent_kwargs' in dir() else {} _heal_kwargs['api_key'] = resolved_api_key @@ -12805,11 +13091,33 @@ def _run_agent_streaming( _heal_kwargs['model'] = resolved_model _heal_kwargs['provider'] = resolved_provider _replace_session_db_in_kwargs(_heal_kwargs, _state_db_path) + if 'api_mode' in _agent_params: + _heal_kwargs['api_mode'] = _runtime_bundle['api_mode'] + if 'acp_command' in _agent_params: + _heal_kwargs['acp_command'] = _runtime_bundle['acp_command'] + if 'acp_args' in _agent_params: + _heal_kwargs['acp_args'] = _runtime_bundle['acp_args'] if 'credential_pool' in _agent_params: - _heal_kwargs['credential_pool'] = _heal_rt.get('credential_pool') + _heal_kwargs['credential_pool'] = _runtime_bundle['credential_pool'] _heal_agent = _AIAgent(**_heal_kwargs) with STREAMS_LOCK: AGENT_INSTANCES[stream_id] = _heal_agent + _agent_sig = _compute_agent_cache_signature( + resolved_model, + resolved_api_key, + resolved_base_url, + resolved_provider, + _runtime_bundle, + max_iterations_cfg=_max_iterations_cfg, + max_tokens_cfg=_max_tokens_cfg, + fallback_resolved=_fallback_resolved, + toolsets=_toolsets, + reasoning_config=_reasoning_config, + main_request_overrides=_main_request_overrides, + prefill_context=_prefill_context, + profile_home=_profile_home, + safe_profile_runtime_env=_safe_profile_runtime_env, + ) from api.config import SESSION_AGENT_CACHE as _SAC2, SESSION_AGENT_CACHE_LOCK as _SAC2_L with _SAC2_L: _SAC2[session_id] = (_heal_agent, _agent_sig) @@ -12925,7 +13233,19 @@ def _run_agent_streaming( except Exception as _retry_exc2: logger.warning('[webui] self-heal (except path): retry failed: %s', _retry_exc2) # Fall through to emit the original error - if _heal_stale_classification is not None: + if _heal_route_classification is not None: + # The heal was abandoned because the named route is unroutable. + # Report THAT, not the 401 that triggered the heal: telling the + # user to check credentials when the provider resolves no + # endpoint (or declares a credential source that yields nothing) + # sends them to the wrong setting. + _exc_label = _heal_route_classification['label'] + _exc_type = _heal_route_classification['type'] + _exc_hint = _heal_route_classification['hint'] + _exc_route_reason = _heal_route_classification.get('reason') + if _heal_route_classification['message']: + err_str = _heal_route_classification['message'] + elif _heal_stale_classification is not None: _exc_label = _heal_stale_classification['label'] _exc_type = _heal_stale_classification['type'] _exc_hint = _heal_stale_classification['hint'] @@ -12963,6 +13283,10 @@ def _run_agent_streaming( 'The conversation changed while context compression was being prepared.' ) _error_payload = _provider_error_payload(err_str, _exc_type, _exc_hint) + if _exc_route_reason: + # Clients (and regressions) branch on the exact verdict rather than + # on the human-readable message, which is free to be reworded. + _error_payload['reason'] = _exc_route_reason if s is not None: if _checkpoint_stop is not None: _checkpoint_stop.set() diff --git a/tests/test_agent_max_turns_parity.py b/tests/test_agent_max_turns_parity.py index 888e34ed..4665f74f 100644 --- a/tests/test_agent_max_turns_parity.py +++ b/tests/test_agent_max_turns_parity.py @@ -13,6 +13,153 @@ REPO = Path(__file__).resolve().parent.parent STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8") +def _compute_agent_cache_signature_source() -> str: + """Return the source of the `_compute_agent_cache_signature()` helper. + + The cache-signature blob used to be inlined in the streaming send path; it + now lives in this helper so the initial send and both self-heal retry paths + derive the signature from the same final runtime bundle. + """ + start = STREAMING_PY.index("def _compute_agent_cache_signature(") + end = STREAMING_PY.index("\ndef ", start) + return STREAMING_PY[start:end] + + +def _signature_blob() -> str: + """Return the `_json.dumps([...])` field list the signature hashes.""" + helper = _compute_agent_cache_signature_source() + blob_start = helper.index("_sig_blob = _json.dumps") + blob_end = helper.index("], sort_keys=True)", blob_start) + return helper[blob_start:blob_end] + + +def _production_signature_calls() -> list[tuple[int, str]]: + """Return `(offset, source)` for every production signature call site. + + Paren-balanced so the whole multi-line keyword-argument list is captured, + and every live call site is returned so a retry path cannot silently drop a + field that the initial send still passes. Commented-out occurrences are + skipped -- a disabled retry call must read as missing, not as present. + """ + marker = "_agent_sig = _compute_agent_cache_signature(" + calls: list[tuple[int, str]] = [] + pos = STREAMING_PY.find(marker) + while pos != -1: + line_start = STREAMING_PY.rfind("\n", 0, pos) + 1 + if STREAMING_PY[line_start:pos].strip(): + # Commented-out (or otherwise non-statement) occurrence: it no + # longer runs, so it must not count as a live call site. + pos = STREAMING_PY.find(marker, pos + 1) + continue + depth = 0 + for idx in range(pos + len(marker) - 1, len(STREAMING_PY)): + char = STREAMING_PY[idx] + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + calls.append((pos, STREAMING_PY[pos:idx + 1])) + break + else: + raise AssertionError("unterminated _compute_agent_cache_signature( call") + pos = STREAMING_PY.find(marker, pos + 1) + return calls + + +# Lifecycle anchors for the streaming send path. Pinning one signature call to +# each region keeps the oracle honest: a retry call that silently disappears +# fails the region check instead of hiding behind the calls that remain. +INITIAL_SEND_MARKER = "# ── Agent cache: reuse across messages in the same session ──" +RETURNED_ERROR_SELF_HEAL_MARKER = ( + "logger.info('[webui] self-heal: retrying stream after credential refresh')" +) +RAISED_EXCEPTION_SELF_HEAL_MARKER = ( + "logger.info('[webui] self-heal (except path): " + "retrying stream after credential refresh')" +) + +# Each region ends at the cache write that stores the signature it just +# computed. Bounding the region at the write -- rather than at the next +# lifecycle anchor or at EOF -- is what proves the signature is recomputed +# *before* it is cached: a call moved below its own write lands outside the +# region and fails, instead of passing while the write consumes a stale +# `_agent_sig`. +INITIAL_SEND_CACHE_WRITE_MARKER = ( + "SESSION_AGENT_CACHE[session_id] = (agent, _agent_sig)" +) +RETURNED_ERROR_SELF_HEAL_CACHE_WRITE_MARKER = ( + "_SAC[session_id] = (agent, _agent_sig)" +) +RAISED_EXCEPTION_SELF_HEAL_CACHE_WRITE_MARKER = ( + "_SAC2[session_id] = (_heal_agent, _agent_sig)" +) + + +def _marker_offset(marker: str) -> int: + """Return the single offset of a lifecycle marker in streaming.py.""" + offset = STREAMING_PY.index(marker) + assert STREAMING_PY.find(marker, offset + 1) == -1, ( + "lifecycle marker is no longer unique in streaming.py:\n" + marker + ) + return offset + + +def _signature_calls_by_region( + calls: list[tuple[int, str]], +) -> dict[str, tuple[int, str]]: + """Map each lifecycle region to the single signature call inside it. + + The regions are the initial send, the returned-error self-heal retry and + the raised-exception self-heal retry; each runs from its lifecycle anchor + to the cache write that consumes `_agent_sig`. Requiring exactly one call + inside those bounds means neither a dropped retry call (masked by its + surviving siblings) nor a call recomputed after its own cache write (which + would leave the write storing a stale signature) can read as correct. + """ + initial = _marker_offset(INITIAL_SEND_MARKER) + initial_cache_write = _marker_offset(INITIAL_SEND_CACHE_WRITE_MARKER) + returned_error = _marker_offset(RETURNED_ERROR_SELF_HEAL_MARKER) + returned_error_cache_write = _marker_offset( + RETURNED_ERROR_SELF_HEAL_CACHE_WRITE_MARKER + ) + raised_exception = _marker_offset(RAISED_EXCEPTION_SELF_HEAL_MARKER) + raised_exception_cache_write = _marker_offset( + RAISED_EXCEPTION_SELF_HEAL_CACHE_WRITE_MARKER + ) + assert ( + initial + < initial_cache_write + < returned_error + < returned_error_cache_write + < raised_exception + < raised_exception_cache_write + ), ( + "streaming.py lifecycle regions are out of order; these anchors no " + "longer describe the send path" + ) + + bounds = { + "initial send": (initial, initial_cache_write), + "returned-error self-heal": (returned_error, returned_error_cache_write), + "raised-exception self-heal": ( + raised_exception, + raised_exception_cache_write, + ), + } + by_region: dict[str, tuple[int, str]] = {} + for region, (start, end) in bounds.items(): + found = [(offset, call) for offset, call in calls if start < offset < end] + assert len(found) == 1, ( + "expected exactly one _compute_agent_cache_signature() call in the " + f"{region} region of streaming.py -- between its lifecycle anchor " + f"and the cache write that stores `_agent_sig` -- found " + f"{len(found)}" + ) + by_region[region] = found[0] + return by_region + + def test_streaming_agent_reads_agent_max_turns_from_config(): assert "_agent_cfg_for_iterations" in STREAMING_PY assert "_agent_cfg_for_iterations.get('max_turns')" in STREAMING_PY @@ -25,6 +172,19 @@ def test_streaming_agent_passes_max_iterations_to_aiagent(): def test_streaming_agent_cache_signature_includes_max_iterations(): - sig_start = STREAMING_PY.index("_sig_blob = _json.dumps") - sig_block = STREAMING_PY[sig_start:STREAMING_PY.index("], sort_keys=True)", sig_start)] - assert "_max_iterations_cfg or ''" in sig_block + assert "max_iterations_cfg or ''" in _signature_blob(), ( + "_compute_agent_cache_signature() must hash max_iterations_cfg, or a " + "max_turns change reuses the cached agent built on the old budget." + ) + + calls = _production_signature_calls() + assert len(calls) == 3, ( + "expected exactly three _compute_agent_cache_signature() call sites " + "(initial send plus both self-heal retries), found " + f"{len(calls)}" + ) + for region, (_offset, call) in _signature_calls_by_region(calls).items(): + assert "max_iterations_cfg=_max_iterations_cfg" in call, ( + f"the {region} signature call must pass the resolved " + "max_iterations budget:\n" + call + ) diff --git a/tests/test_issue1013_handoff_dock.py b/tests/test_issue1013_handoff_dock.py index f8ca81a0..07af6646 100644 --- a/tests/test_issue1013_handoff_dock.py +++ b/tests/test_issue1013_handoff_dock.py @@ -620,6 +620,25 @@ def test_handoff_summary_codex_output_cap_matches_provider_compatibility( import api.models as models import api.routes as routes + if provider.startswith("custom:"): + # A named ``custom:`` route is identity-owned: the endpoint comes + # from the record that names the slug, and a route no record owns fails + # closed (#1806). ``resolve_model_provider`` only ever returns this + # provider/base_url pair BECAUSE such a row exists, so the fixture must + # carry it — otherwise the handler is handed a connection with no + # authority behind it and falls back before reaching the Codex path. + monkeypatch.setitem( + cfg.cfg, + "custom_providers", + [ + { + "name": provider.split(":", 1)[1], + "base_url": base_url, + "api_key": "handoff-summary-key", + } + ], + ) + if expects_normalized_base_url: real_urlsplit = routes.urlsplit diff --git a/tests/test_issue1806_named_custom_provider_resolution.py b/tests/test_issue1806_named_custom_provider_resolution.py index 8328c33b..d8a7777e 100644 --- a/tests/test_issue1806_named_custom_provider_resolution.py +++ b/tests/test_issue1806_named_custom_provider_resolution.py @@ -8,7 +8,10 @@ model selection and send runtime auth down an impossible env-var path. from __future__ import annotations +import contextlib +import copy import json +import os import sys import types @@ -117,3 +120,5121 @@ def test_available_models_drops_base_url_derived_custom_slug(monkeypatch): named_models = [m["id"] for m in groups_by_id["custom:ollama-local"]["models"]] assert "carnice-9b:latest" in named_models + + +def _with_multi_custom_provider_config(): + """Active custom provider PLUS a second, non-active named custom provider. + + Mirrors the config.yaml shape that has no ``providers:`` map at all: every + endpoint lives in ``custom_providers:``, and only one of them is the active + ``model.provider``. + """ + old_cfg = dict(config.cfg) + old_mtime = config._cfg_mtime + old_path = getattr(config, "_cfg_path", None) + config.cfg.clear() + config.cfg.update( + { + "model": { + "default": "active/model", + "provider": "custom:active", + "base_url": "https://active.example/v1", + "api_key": "active-key", + }, + "custom_providers": [ + { + "name": "active", + "base_url": "https://active.example/v1", + "api_key": "active-key", + }, + { + "name": "omni", + "base_url": "https://omni.example/v1", + "api_key": "omni-key", + }, + ], + } + ) + try: + config._cfg_mtime = config.Path(config._get_config_path()).stat().st_mtime + except Exception: + config._cfg_mtime = 0.0 + config._cfg_path = config._get_config_path() + + def restore(): + config.cfg.clear() + config.cfg.update(old_cfg) + config._cfg_mtime = old_mtime + config._cfg_path = old_path + config.invalidate_models_cache() + + return restore + + +def test_provider_qualified_id_uses_nonactive_custom_provider_base_url(): + """``@custom:omni:`` must route to omni's base_url, not the default. + + ``_get_provider_base_url`` only reads ``providers:`` and the ACTIVE + ``model.base_url``, so a named custom provider that lives solely in + ``custom_providers:`` used to resolve to base_url=None. The WebUI then sent + the bare model to the active endpoint and got back HTTP 400 "Invalid model + format or no credentials for provider: ". + """ + restore = _with_multi_custom_provider_config() + try: + model, provider, base_url = config.resolve_model_provider( + "@custom:omni:antigravity/gemini-3.7-flash-tiered" + ) + finally: + restore() + + assert model == "antigravity/gemini-3.7-flash-tiered" + assert provider == "custom:omni" + assert base_url == "https://omni.example/v1" + + +def test_provider_qualified_unknown_custom_slug_keeps_base_url_none(): + """An UNKNOWN ``custom:`` slug must stay base_url=None -- never a guess. + + Slugs derived from a base-url authority (``custom:local-(127.0.0.1:11434)``) + or from a provider that is simply not in ``custom_providers:`` have no + endpoint of their own. Guessing one (e.g. "there is only one custom provider, + use it" or "fall back to the active ``model.base_url``") would persist a + stale endpoint for that slug, which is the #4728 regression. Preserve the + prior behaviour: no unique matching entry -> no base_url. + """ + restore = _with_multi_custom_provider_config() + try: + model, provider, base_url = config.resolve_model_provider( + "@custom:not-configured:qwen/qwen-1.5b" + ) + finally: + restore() + + assert model == "qwen/qwen-1.5b" + assert provider == "custom:not-configured" + assert base_url is None + + +def test_provider_qualified_active_custom_slug_still_resolves(): + """The ACTIVE custom provider keeps resolving to its own endpoint.""" + restore = _with_multi_custom_provider_config() + try: + model, provider, base_url = config.resolve_model_provider( + "@custom:active:antigravity/gemini-3.7-flash-tiered" + ) + finally: + restore() + + assert model == "antigravity/gemini-3.7-flash-tiered" + assert provider == "custom:active" + assert base_url == "https://active.example/v1" + + +def test_provider_qualified_non_custom_provider_is_unaffected(): + """A non-``custom:`` @provider hint must not pick up a custom endpoint. + + The custom_providers lookup is gated on the ``custom:`` prefix, so an + @openrouter route still resolves through _get_provider_base_url() -- None + here, since openrouter is neither the active provider nor in ``providers:``. + """ + restore = _with_multi_custom_provider_config() + try: + model, provider, base_url = config.resolve_model_provider( + "@openrouter:anthropic/claude-sonnet-4.6" + ) + finally: + restore() + + assert model == "anthropic/claude-sonnet-4.6" + assert provider == "openrouter" + assert base_url is None + + +def _with_keyed_and_list_provider_config(): + """Same slug present BOTH as a ``providers:`` key and a ``custom_providers`` entry. + + Deployments that started on the legacy ``custom_providers:`` list and later + gained a keyed ``providers:`` map can carry two records for one slug, each + with its own ``base_url``. The other fixtures in this file deliberately model + the list-only shape, so this one pins which record wins. + """ + old_cfg = dict(config.cfg) + old_mtime = config._cfg_mtime + old_path = getattr(config, "_cfg_path", None) + config.cfg.clear() + config.cfg.update( + { + "model": { + "default": "active/model", + "provider": "custom:active", + "base_url": "https://active.example/v1", + "api_key": "active-key", + }, + "providers": { + "custom:omni": { + "base_url": "https://omni-keyed.example/v1", + "api_key": "omni-keyed-key", + }, + }, + "custom_providers": [ + { + "name": "active", + "base_url": "https://active.example/v1", + "api_key": "active-key", + }, + { + "name": "omni", + "base_url": "https://omni-list.example/v1", + "api_key": "omni-list-key", + }, + ], + } + ) + try: + config._cfg_mtime = config.Path(config._get_config_path()).stat().st_mtime + except Exception: + config._cfg_mtime = 0.0 + config._cfg_path = config._get_config_path() + + def restore(): + config.cfg.clear() + config.cfg.update(old_cfg) + config._cfg_mtime = old_mtime + config._cfg_path = old_path + config.invalidate_models_cache() + + return restore + + +def test_list_entry_wins_over_keyed_providers_entry_for_same_slug(): + """``custom_providers[]`` outranks a same-slug ``providers:`` key. + + The named entry is the record the picker's ``custom:`` id is MINTED + from (``_custom_provider_slug_from_name`` reads ``custom_providers[].name``), + and it is the one credential resolution scans, so the endpoint must come from + that same entry -- otherwise a stale keyed ``providers:`` leftover could pair + entry A's URL with entry B's API key. Guards the precedence of the + ``custom_base_url if custom_base_url is not None else _get_provider_base_url()`` + ordering: both lookups return a URL here, so a flipped order would silently + route to ``omni-keyed`` instead. + """ + restore = _with_keyed_and_list_provider_config() + try: + # Sanity: the keyed entry really is resolvable, so this test would fail + # (not merely pass vacuously on a None fallback) if precedence flipped. + keyed_base_url = config._get_provider_base_url("custom:omni") + model, provider, base_url = config.resolve_model_provider( + "@custom:omni:antigravity/gemini-3.7-flash-tiered" + ) + conn_api_key, conn_base_url = config.resolve_custom_provider_connection("custom:omni") + finally: + restore() + + assert keyed_base_url == "https://omni-keyed.example/v1" + assert model == "antigravity/gemini-3.7-flash-tiered" + assert provider == "custom:omni" + assert base_url == "https://omni-list.example/v1" + assert conn_api_key == "omni-list-key" + assert conn_base_url == "https://omni-list.example/v1" + assert (base_url, conn_api_key) == ("https://omni-list.example/v1", "omni-list-key") + + +def _with_keyed_and_blank_list_provider_config(): + """Same slug present in ``providers:`` and as a ``custom_providers`` entry with blank base_url. + + The list entry exists and has a distinct API key, but its ``base_url`` is + empty. The keyed ``providers:`` entry has both a valid ``base_url`` and its + own distinct API key. + """ + old_cfg = dict(config.cfg) + old_mtime = config._cfg_mtime + old_path = getattr(config, "_cfg_path", None) + config.cfg.clear() + config.cfg.update( + { + "model": { + "default": "active/model", + "provider": "custom:active", + "base_url": "https://active.example/v1", + "api_key": "active-key", + }, + "providers": { + "custom:omni": { + "base_url": "https://omni-keyed.example/v1", + "api_key": "omni-keyed-key", + }, + }, + "custom_providers": [ + { + "name": "active", + "base_url": "https://active.example/v1", + "api_key": "active-key", + }, + { + "name": "omni", + "base_url": "", + "api_key": "omni-list-key", + }, + ], + } + ) + try: + config._cfg_mtime = config.Path(config._get_config_path()).stat().st_mtime + except Exception: + config._cfg_mtime = 0.0 + config._cfg_path = config._get_config_path() + + def restore(): + config.cfg.clear() + config.cfg.update(old_cfg) + config._cfg_mtime = old_mtime + config._cfg_path = old_path + config.invalidate_models_cache() + + return restore + + +def test_blank_list_entry_does_not_fall_through_to_keyed_providers_endpoint(): + """A blank ``custom_providers[].base_url`` must not fall through to a keyed endpoint. + + When a slug exists both in ``custom_providers[]`` (with a blank or empty + base_url and API key K_list) and in ``providers:`` (with a populated base_url + and API key K_keyed), resolution must treat the list entry as authoritative. + Falling through to ``_get_provider_base_url()`` on empty base_url would pair + the keyed record's endpoint with the list record's API key, violating the + same-entry invariant between resolve_model_provider() and + resolve_custom_provider_connection(). + """ + restore = _with_keyed_and_blank_list_provider_config() + try: + keyed_base_url = config._get_provider_base_url("custom:omni") + model, provider, base_url = config.resolve_model_provider( + "@custom:omni:antigravity/gemini-3.7-flash-tiered" + ) + conn_api_key, conn_base_url = config.resolve_custom_provider_connection("custom:omni") + finally: + restore() + + assert keyed_base_url == "https://omni-keyed.example/v1" + assert model == "antigravity/gemini-3.7-flash-tiered" + assert provider == "custom:omni" + assert base_url is None + assert conn_base_url is None + assert conn_api_key == "omni-list-key" + assert (base_url, conn_api_key) != ("https://omni-keyed.example/v1", "omni-list-key") + + +def _setup_production_composed_runtime( + monkeypatch, + cfg_dict, + runtime_dict, + session_id="test-session-1806", + fail_first=None, + heal_mutate=None, +): + """Compose the production streaming send path around a capturing agent. + + ``fail_first`` drives the two 401 self-heal retry paths that rebuild the + runtime bundle a second time: + + * ``"returned_error"`` -- the first ``run_conversation`` RETURNS an auth + error without raising (``api/streaming.py`` returned-error retry). + * ``"raised"`` -- the first ``run_conversation`` RAISES it + (``api/streaming.py`` raised-exception retry). + + Both make ``_attempt_credential_self_heal`` hand back the same ambient + runtime dict production would re-resolve, so the retry sees the identical + truthy side-field sentinels the initial send did. + + ``heal_mutate`` is the hook for the state a retry guard actually defends + against: it runs INSIDE the stubbed self-heal, i.e. after the first agent + has already been constructed and has already failed with a 401, but BEFORE + the retry re-resolves its bundle. Mutating ``config.cfg`` there reproduces + the row being edited, unnamed or drained mid-turn -- the only way a route + that was routable at first resolution becomes terminal at retry + construction. + """ + from unittest import mock + import queue + import api.streaming as streaming + import api.oauth + + with config.SESSION_AGENT_CACHE_LOCK: + config.SESSION_AGENT_CACHE.clear() + + old_cfg = dict(config.cfg) + old_mtime = config._cfg_mtime + old_path = getattr(config, "_cfg_path", None) + config.cfg.clear() + config.cfg.update(cfg_dict) + try: + config._cfg_mtime = config.Path(config._get_config_path()).stat().st_mtime + except Exception: + config._cfg_mtime = 0.0 + config._cfg_path = config._get_config_path() + + class FakeSession: + def __init__(self): + self.session_id = session_id + self.title = "Test" + self.workspace = "/tmp" + self.model = "test-model" + self.messages = [] + self.personality = None + self.input_tokens = 0 + self.output_tokens = 0 + self.estimated_cost = None + self.tool_calls = [] + self.active_stream_id = None + self.pending_user_message = None + self.pending_attachments = [] + self.pending_started_at = None + self.pending_user_source = None + self.profile = None + + def save(self, touch_updated_at=True, skip_index=False): + self._saved = touch_updated_at + + def compact(self): + return {"session_id": self.session_id, "messages": self.messages} + + captured = {} + + class CapturingAgent: + # Every constructor-routing field must be a NAMED parameter. The + # streaming path gates each optional kwarg on + # ``inspect.signature(AIAgent.__init__)``, so a double that only accepts + # model/provider/base_url/api_key silently filters the runtime-owned + # fields out — hiding exactly the stale-authority defect these tests pin. + def __init__( + self, + model=None, + provider=None, + base_url=None, + api_key=None, + api_mode=None, + acp_command=None, + acp_args=None, + credential_pool=None, + **kwargs, + ): + captured["init_kwargs"] = { + "model": model, + "provider": provider, + "base_url": base_url, + "api_key": api_key, + "api_mode": api_mode, + "acp_command": acp_command, + "acp_args": acp_args, + "credential_pool": credential_pool, + **kwargs, + } + captured.setdefault("init_kwargs_history", []).append( + dict(captured["init_kwargs"]) + ) + captured.setdefault("instances", []).append(self) + # Mirror ``agent/agent_init.py:_init_openai_client()``: + # + # if api_key and base_url: + # client_kwargs = _explicit_client_kwargs(...) + # else: + # client_kwargs = _routed_client_kwargs(...) + # + # An incomplete connection pair is NOT a refusal at this boundary — + # it is the signal to resolve a provider all over again, through the + # centralized router and then the init-time fallback chain. Recording + # the branch here is what lets a test assert the real defect ("this + # send would have been re-routed") instead of the weaker proxy + # ("base_url came back None"), which an unroutable bundle satisfies + # while still reaching a provider the user never chose. + if api_key and base_url: + captured.setdefault("explicit_client_kwargs_calls", []).append( + {"api_key": api_key, "base_url": base_url} + ) + else: + captured.setdefault("routed_client_kwargs_calls", []).append( + {"provider": provider, "api_key": api_key, "base_url": base_url} + ) + self.session_id = kwargs.get("session_id") + self.context_compressor = None + self.session_prompt_tokens = 0 + self.session_completion_tokens = 0 + self.session_estimated_cost_usd = None + self.reasoning_config = None + self.ephemeral_system_prompt = None + self._last_error = None + + def run_conversation(self, **kwargs): + captured["run_kwargs"] = kwargs + captured["run_calls"] = captured.get("run_calls", 0) + 1 + if fail_first is not None and captured["run_calls"] == 1: + if fail_first == "raised": + raise RuntimeError("401 Unauthorized") + return {"messages": [], "error": "401 Unauthorized"} + return { + "messages": [ + {"role": "user", "content": kwargs.get("persist_user_message", "")}, + {"role": "assistant", "content": "ok"}, + ] + } + + def interrupt(self, _message): + captured["interrupted"] = _message + + fake_session = FakeSession() + fake_stream_id = f"stream-{session_id}" + # Both the returned-error retry branch AND the outer error emission live + # behind the stale-writeback guard, which only lets the OWNING worker + # persist and emit. /api/chat/start stamps ``active_stream_id`` before + # dispatching the worker, so model that here unconditionally — otherwise the + # guard returns early and a terminal route verdict reaches the queue as + # nothing at all, which a refusal test would read as "no controlled failure". + fake_session.active_stream_id = fake_stream_id + fake_queue = queue.Queue() + + fake_runtime_module = types.ModuleType("hermes_cli.runtime_provider") + fake_runtime_module.resolve_runtime_provider = mock.Mock(return_value=dict(runtime_dict)) + fake_hermes_cli = types.ModuleType("hermes_cli") + fake_hermes_cli.runtime_provider = fake_runtime_module + fake_hermes_state = types.ModuleType("hermes_state") + fake_hermes_state.SessionDB = mock.Mock(return_value=object()) + + monkeypatch.setitem(sys.modules, "hermes_cli", fake_hermes_cli) + monkeypatch.setitem(sys.modules, "hermes_cli.runtime_provider", fake_runtime_module) + monkeypatch.setitem(sys.modules, "hermes_state", fake_hermes_state) + + monkeypatch.setattr( + api.oauth, + "resolve_runtime_provider_with_anthropic_env_lock", + lambda resolver, **kwargs: resolver(**kwargs), + ) + monkeypatch.setattr(streaming, "get_session", lambda _session_id: fake_session) + monkeypatch.setattr(streaming, "_get_ai_agent", lambda: CapturingAgent) + monkeypatch.setattr("api.config.get_config", lambda: dict(config.cfg)) + monkeypatch.setattr("api.config._resolve_cli_toolsets", lambda *_args, **_kwargs: []) + if fail_first is not None: + # Production re-resolves the ambient runtime provider on a 401 heal, so + # the retry must see the same truthy side-field sentinels — that is + # exactly the state in which a partial rebuild leaks them through. + def _fake_self_heal(*_args, **_kwargs): + # Production re-reads provider state during the heal. Running the + # mutation here -- not before the send -- is what makes the FIRST + # resolution routable and only the RETRY resolution terminal. + if heal_mutate is not None: + heal_mutate() + return dict(runtime_dict) + + monkeypatch.setattr(streaming, "_attempt_credential_self_heal", _fake_self_heal) + + def restore(): + with config.SESSION_AGENT_CACHE_LOCK: + config.SESSION_AGENT_CACHE.clear() + config.cfg.clear() + config.cfg.update(old_cfg) + config._cfg_mtime = old_mtime + config._cfg_path = old_path + config.invalidate_models_cache() + + return fake_stream_id, fake_queue, captured, restore + + +def test_production_composed_nonblank_exact_list_row_yields_list_url_and_list_key(monkeypatch): + """(a) Nonblank exact list row plus distinct same-slug keyed row yields list URL/list key. + + In production, resolve_runtime_provider() scans providers: first and returns + the keyed URL and key. When an exact custom_providers[] entry matches, the + runtime send path must atomically replace both fields so that final AIAgent + construction receives list endpoint + list key, never keyed key. + """ + import api.streaming as streaming + + cfg_dict = { + "model": {"default": "active/model", "provider": "custom:active"}, + "providers": { + "custom:omni": { + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", + }, + }, + "custom_providers": [ + { + "name": "omni", + "base_url": "https://list-url-sentinel.example/v1", + "api_key": "list-key-sentinel-xyz", + }, + ], + } + runtime_dict = { + "provider": "custom:omni", + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", + } + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, cfg_dict, runtime_dict, session_id="session-1806-nonblank" + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id="session-1806-nonblank", + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + init_kwargs = captured["init_kwargs"] + assert init_kwargs["base_url"] == "https://list-url-sentinel.example/v1" + assert init_kwargs["api_key"] == "list-key-sentinel-xyz" + assert init_kwargs["provider"] == "custom" + assert init_kwargs["base_url"] != "https://keyed-url-sentinel.example/v1" + assert init_kwargs["api_key"] != "keyed-key-sentinel-abc" + + +def test_production_composed_blank_exact_list_row_cannot_repopulate_from_keyed_row(monkeypatch): + """(b) A blank exact list row is terminal — it cannot repopulate from the keyed row. + + When an exact ``custom_providers[]`` entry exists with an empty base_url, + that row is authoritative and the route has no endpoint. Handing AIAgent + ``base_url=None`` with the list key would NOT refuse: ``_init_openai_client()`` + honours an explicit pair only when both fields are truthy, so it would call + ``_routed_client_kwargs()`` and re-resolve a provider — reaching the keyed + row's endpoint by another door. The send must stop before construction. + """ + cfg_dict = { + "model": {"default": "active/model", "provider": "custom:active"}, + "providers": { + "custom:omni": { + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", + }, + }, + "custom_providers": [ + { + "name": "omni", + "base_url": "", + "api_key": "list-key-sentinel-xyz", + }, + ], + } + runtime_dict = { + "provider": "custom:omni", + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", + } + + captured, apperrors = _run_composed_send_expecting_refusal( + monkeypatch, + cfg_dict, + runtime_dict, + "session-1806-blank", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + ) + + payload = apperrors[-1] + assert "resolved no endpoint" in payload["message"], payload + assert "base_url" in payload["hint"], ( + f"the refusal must name the endpoint setting to fix: {payload}" + ) + # The refusal must not hand the user the keyed row it declined to fall + # through to — naming it would read as "this endpoint was used". + assert "keyed-url-sentinel" not in str(payload), payload + assert not captured.get("explicit_client_kwargs_calls"), ( + "a client was configured for a route with no endpoint" + ) + + +def test_production_composed_keyed_only_still_yields_keyed_url_and_key(monkeypatch): + """(c) Keyed-only/no-list still yields the keyed URL/key. + + When no matching row exists in custom_providers[], the keyed providers: + record is authoritative and supplies both base_url and api_key. + """ + import api.streaming as streaming + + cfg_dict = { + "model": {"default": "active/model", "provider": "custom:active"}, + "providers": { + "custom:omni": { + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", + }, + }, + "custom_providers": [ + { + "name": "active", + "base_url": "https://active.example/v1", + "api_key": "active-key", + }, + ], + } + runtime_dict = { + "provider": "custom:omni", + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", + } + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, cfg_dict, runtime_dict, session_id="session-1806-keyed" + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id="session-1806-keyed", + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + init_kwargs = captured["init_kwargs"] + assert init_kwargs["base_url"] == "https://keyed-url-sentinel.example/v1" + assert init_kwargs["api_key"] == "keyed-key-sentinel-abc" + assert init_kwargs["provider"] == "custom" + + +# ───────────────────────────────────────────────────────────────────────────── +# Full-constructor bundle authority (streaming initial send + both retry paths) +# +# The connection fields are only half the constructor contract. AIAgent also +# takes ``credential_pool``, ``api_mode``, ``acp_command`` and ``acp_args`` from +# the resolved runtime provider, and the ambient provider legitimately reports +# all four (it is what the process is otherwise authenticated as). A send that +# replaces provider/base_url/api_key from an exact ``custom_providers[]`` row but +# keeps those four builds an agent whose credential source, wire protocol and +# transport still point at the previous authority — the custom HTTP endpoint gets +# Anthropic credential pooling and a Claude ACP subprocess command. +# +# ``_AMBIENT_SIDE_FIELD_RUNTIME`` seeds every one of them truthy so a +# pass-through is a hard assertion failure rather than a vacuous ``None == None``. +# ───────────────────────────────────────────────────────────────────────────── + +_KEYED_VS_LIST_CFG = { + "model": {"default": "active/model", "provider": "custom:active"}, + "providers": { + "custom:omni": { + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", + }, + }, + "custom_providers": [ + { + "name": "omni", + "base_url": "https://list-url-sentinel.example/v1", + "api_key": "list-key-sentinel-xyz", + }, + ], +} + +_AMBIENT_SIDE_FIELD_RUNTIME = { + "provider": "custom:omni", + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", + # Runtime-owned constructor fields belonging to the AMBIENT provider. + "credential_pool": ["ambient-pool-sentinel-1", "ambient-pool-sentinel-2"], + "api_mode": "anthropic_messages", + "command": "claude-code-acp-sentinel", + "args": ["--acp-arg-sentinel"], +} + +_LIST_ROW_URL = "https://list-url-sentinel.example/v1" +_LIST_ROW_KEY = "list-key-sentinel-xyz" + + +# Case (d): every side field in ``_AMBIENT_SIDE_FIELD_RUNTIME`` is genuinely +# FOREIGN to ``_KEYED_VS_LIST_CFG`` -- the list row owns a different endpoint and +# declares none of these for itself, so provenance proves they belong to the +# ambient provider and every one must be cleared. This is an ownership verdict, +# not a blanket "custom routes never carry side fields" rule: the cases below +# pin records that DO own them, and there the same merge must keep them. +_FOREIGN_AMBIENT_SIDE_FIELDS = { + "api_mode": None, + "acp_command": None, + "acp_args": None, + "credential_pool": None, +} + + +def _assert_side_fields(init_kwargs, expected, label): + """Assert each constructor side field equals the authority that OWNS it.""" + for field, value in expected.items(): + actual = init_kwargs[field] + if callable(value): + assert actual is value, f"{label}: {field} lost its owner's value" + else: + assert actual == value, f"{label}: {field} is {actual!r}, expected {value!r}" + + +def _assert_exact_list_row_bundle(init_kwargs, label, side_fields=None): + """Assert the whole constructor bundle is target-owned, not a mixed one. + + ``side_fields`` names what the exact list row's authority resolves each side + field to; it defaults to case (d), the all-foreign ambient runtime. + """ + assert init_kwargs["base_url"] == _LIST_ROW_URL, label + assert init_kwargs["api_key"] == _LIST_ROW_KEY, label + assert init_kwargs["provider"] == "custom", label + # None of the keyed/ambient authority may survive anywhere in the bundle. + assert init_kwargs["base_url"] != "https://keyed-url-sentinel.example/v1", label + assert init_kwargs["api_key"] != "keyed-key-sentinel-abc", label + _assert_side_fields( + init_kwargs, + _FOREIGN_AMBIENT_SIDE_FIELDS if side_fields is None else side_fields, + label, + ) + + +def test_capturing_agent_exposes_every_runtime_constructor_field(monkeypatch): + """Guard the guard: the double must not signature-filter the side fields. + + ``_run_agent_streaming`` gates each optional kwarg on + ``inspect.signature(AIAgent.__init__).parameters``. A double that swallowed + ``credential_pool`` / ``api_mode`` / ``acp_command`` / ``acp_args`` into + ``**kwargs`` would make every assertion below pass vacuously, because the + stale values would never be passed at all. + """ + import inspect + + import api.streaming as streaming + + _stream_id, _q, _captured, restore = _setup_production_composed_runtime( + monkeypatch, dict(_KEYED_VS_LIST_CFG), {} + ) + try: + params = set(inspect.signature(streaming._get_ai_agent().__init__).parameters) + finally: + restore() + + for field in ("api_mode", "acp_command", "acp_args", "credential_pool"): + assert field in params, f"{field} would be signature-filtered out" + + +def test_production_composed_initial_send_replaces_full_runtime_bundle(monkeypatch): + """Initial send: exact list row owns the WHOLE constructor bundle.""" + import api.streaming as streaming + + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, + dict(_KEYED_VS_LIST_CFG), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + session_id="session-1806-bundle-initial", + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id="session-1806-bundle-initial", + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + _assert_exact_list_row_bundle(captured["init_kwargs"], "initial send") + + +def test_production_composed_returned_error_retry_replaces_full_runtime_bundle(monkeypatch): + """Returned-error 401 retry: the rebuilt bundle is target-owned too. + + The retry path rebuilds agent kwargs from the self-healed runtime dict. When + it only refreshed provider/key/base_url (plus ``credential_pool`` from the + heal result), the retry agent was constructed with the ambient provider's + pool/api_mode/ACP fields even though the endpoint is the custom list row. + """ + import api.streaming as streaming + + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, + dict(_KEYED_VS_LIST_CFG), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + session_id="session-1806-bundle-returned", + fail_first="returned_error", + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id="session-1806-bundle-returned", + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + history = captured["init_kwargs_history"] + assert len(history) >= 2, "returned-error retry did not construct a second agent" + for index, init_kwargs in enumerate(history): + _assert_exact_list_row_bundle(init_kwargs, f"returned-error construction #{index}") + + +def test_production_composed_raised_exception_retry_replaces_full_runtime_bundle(monkeypatch): + """Raised-exception 401 retry: same complete-bundle contract.""" + import api.streaming as streaming + + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, + dict(_KEYED_VS_LIST_CFG), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + session_id="session-1806-bundle-raised", + fail_first="raised", + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id="session-1806-bundle-raised", + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + history = captured["init_kwargs_history"] + assert len(history) >= 2, "raised-exception retry did not construct a second agent" + for index, init_kwargs in enumerate(history): + _assert_exact_list_row_bundle(init_kwargs, f"raised-exception construction #{index}") + + +def test_agent_cache_signature_tracks_the_resolved_bundle(monkeypatch): + """The cached-agent signature must be derived from the FINAL bundle. + + If ``_sig_blob`` still read ``api_mode`` / ACP / pool off the raw runtime + provider, two sends whose resolved bundles differ only in a cleared side + field would hash identically — so the second send would reuse an agent built + on the previous authority instead of minting a new one. + """ + import api.streaming as streaming + + def _run(runtime_dict, session_id): + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, dict(_KEYED_VS_LIST_CFG), runtime_dict, session_id=session_id + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id=session_id, + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + with config.SESSION_AGENT_CACHE_LOCK: + return config.SESSION_AGENT_CACHE[session_id][1] + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + # Same resolved bundle, wildly different ambient side fields: because the + # custom row clears them all, the signature must be identical. + plain_runtime = { + "provider": "custom:omni", + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", + } + sig_plain = _run(plain_runtime, "session-1806-sig-plain") + sig_ambient = _run(dict(_AMBIENT_SIDE_FIELD_RUNTIME), "session-1806-sig-ambient") + assert sig_plain == sig_ambient, ( + "signature still varies with runtime fields the bundle cleared" + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Ownership-aware side fields +# +# "Clear the runtime-owned side fields whenever a custom record supplies the +# connection" over-corrects. ``api_mode``, ``credential_pool``, ``acp_command`` +# and ``acp_args`` are not intrinsically ambient: a ``custom_providers[]`` row is +# free to declare ``api_mode: anthropic_messages``, and a keyed +# ``providers['custom:']`` record is free to declare a pool and an ACP +# transport. Blanket-clearing them downgrades an Anthropic-protocol row to +# chat-completions and drops a keyed record's own pool/transport. +# +# The rule these tests pin is provenance, not field name: +# +# * the selected record declares the field -> the record's value wins; +# * the runtime resolved the SAME endpoint -> its value is same-authority, keep; +# * the runtime resolved a DIFFERENT one -> proven foreign, clear. +# +# The same distinction gates ``dummy-key``: it is a statement that the endpoint +# is UNAUTHENTICATED, so it may only be substituted once the record's whole +# credential ladder (pool, api_key, key_env, CUSTOM__API_KEY, key_cmd, +# host-gated env) has come up empty. Substituting it over a ``key_cmd`` or a +# pooled credential turns a working endpoint into a 401. +# ───────────────────────────────────────────────────────────────────────────── + + +def _run_composed_send(monkeypatch, cfg_dict, runtime_dict, session_id, before_send=None): + """Drive ONE production-composed streaming send; return the constructor kwargs. + + ``before_send`` runs after the fake ``hermes_cli.runtime_provider`` module is + installed, so a test can hang extra runtime helpers (the credential-pool + lookup) off it before resolution happens. + """ + import api.streaming as streaming + + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, cfg_dict, runtime_dict, session_id=session_id + ) + try: + if before_send is not None: + before_send() + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id=session_id, + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + return captured["init_kwargs"] + + +# ───────────────────────────────────────────────────────────────────────────── +# Terminal route verdicts: the send must STOP, not fail closed and continue +# +# "Fail closed" was the wrong shape for an unresolvable named route. Clearing +# ``base_url``/``api_key`` looks terminal in a bundle assertion, but at the +# constructor it is the opposite: ``_init_openai_client()`` honours an explicit +# pair only when BOTH fields are truthy and otherwise calls +# ``_routed_client_kwargs()``, which re-resolves a provider through the +# centralized router and the init-time fallback chain. So the very bundles the +# earlier tests asserted were "safe" are the ones that route the user's prompt — +# and whatever credential init finds — to a provider they never picked. +# +# The helpers below assert the real property: for both terminal shapes (a slug +# nothing owns, and an owned row whose declared credential resolved to nothing) +# NO agent is constructed at all, so ``_routed_client_kwargs()`` is never +# reached, the agent cache is never written, and the turn ends on a controlled +# ``provider_unroutable`` apperror instead. +# ───────────────────────────────────────────────────────────────────────────── + + +def _drain_apperrors(fake_queue): + """Return every ``apperror`` payload the worker queued.""" + import queue as _queue + + payloads = [] + while True: + try: + item = fake_queue.get_nowait() + except _queue.Empty: + break + if item and item[0] == "apperror": + payloads.append(item[1]) + return payloads + + +def _assert_route_refused(captured, apperrors, label, *, expected_reason=None): + """Assert the send stopped at the route verdict, before any provider routing.""" + routed = captured.get("routed_client_kwargs_calls", []) + assert not routed, ( + f"{label}: AIAgent was constructed with an incomplete connection pair " + f"{routed}, so _init_openai_client() fell through to " + f"_routed_client_kwargs() and re-resolved a provider" + ) + assert not captured.get("init_kwargs_history"), ( + f"{label}: an agent was constructed for an unroutable route" + ) + assert not captured.get("run_calls"), f"{label}: the turn was actually sent" + + assert apperrors, f"{label}: no controlled failure was emitted" + payload = apperrors[-1] + assert payload["type"] == "provider_unroutable", ( + f"{label}: emitted {payload['type']!r} instead of a provider-route failure" + ) + assert payload.get("hint"), f"{label}: the failure named no fix" + if expected_reason is not None: + assert expected_reason in payload.get("message", "") or expected_reason in payload.get( + "hint", "" + ), f"{label}: the failure did not name {expected_reason!r}: {payload}" + + with config.SESSION_AGENT_CACHE_LOCK: + assert not config.SESSION_AGENT_CACHE, ( + f"{label}: the agent cache was poisoned with an unroutable bundle, so " + f"every later turn in this session would reuse it" + ) + return payload + + +def _run_composed_send_expecting_refusal( + monkeypatch, cfg_dict, runtime_dict, session_id, *, model +): + """Drive one composed send whose route is terminal at the FIRST resolution. + + Returns ``(captured, apperrors)``. Deliberately takes no ``fail_first``: the + verdict lands before any agent exists, so there is no 401 for the self-heal + retries to act on and threading one through would only produce cases that + re-run identical code. The retry regions have their own harness, + :func:`_run_composed_retry_expecting_abandoned_heal`. + """ + import api.streaming as streaming + + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, cfg_dict, runtime_dict, session_id=session_id + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id=session_id, + msg_text="hello", + model=model, + workspace="/tmp", + stream_id=stream_id, + ) + apperrors = _drain_apperrors(q) + # Read the cache BEFORE restore() clears it. + _assert_route_refused(captured, apperrors, session_id) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + return captured, apperrors + + +def _exact_list_row_cfg(*, drop=(), **row_fields): + """``_KEYED_VS_LIST_CFG`` with the exact ``custom_providers[]`` row extended.""" + cfg_dict = copy.deepcopy(_KEYED_VS_LIST_CFG) + for field in drop: + cfg_dict["custom_providers"][0].pop(field, None) + cfg_dict["custom_providers"][0].update(row_fields) + return cfg_dict + + +def _ambient_runtime(**overrides): + runtime = copy.deepcopy(_AMBIENT_SIDE_FIELD_RUNTIME) + runtime.update(overrides) + return runtime + + +# ── (a) exact-list-owned Anthropic mode ────────────────────────────────────── + + +@pytest.mark.parametrize( + "row_fields,session_suffix", + [ + ({"api_mode": "anthropic_messages"}, "api-mode"), + # ``transport:`` is the v12-migration spelling and ``anthropic`` an + # accepted alias; a hand-edited config using either still owns the mode. + ({"transport": "anthropic"}, "transport-alias"), + ], +) +def test_exact_list_row_keeps_its_own_anthropic_api_mode( + monkeypatch, row_fields, session_suffix +): + """(a) The row declares its wire protocol, so neither clearing nor the ambient wins. + + The ambient runtime reports ``chat_completions`` for a DIFFERENT endpoint. + Both failure modes are visible here: passing the ambient value through gives + ``chat_completions``, blanket-clearing gives ``None``, and only reading the + row's own declaration gives ``anthropic_messages`` — which is what decides + whether the send speaks /v1/messages or /v1/chat/completions. + """ + init_kwargs = _run_composed_send( + monkeypatch, + _exact_list_row_cfg(**row_fields), + _ambient_runtime(api_mode="chat_completions"), + f"session-1806-owned-{session_suffix}", + ) + + _assert_exact_list_row_bundle( + init_kwargs, + f"exact list row owns api_mode ({session_suffix})", + side_fields={**_FOREIGN_AMBIENT_SIDE_FIELDS, "api_mode": "anthropic_messages"}, + ) + + +# ── (b) exact-list key_cmd / pool credential ───────────────────────────────── + + +def _fake_command_token_source(monkeypatch, build): + fake_module = types.ModuleType("agent.command_token_source") + fake_module.build_command_token_provider = build + monkeypatch.setitem(sys.modules, "agent.command_token_source", fake_module) + + +def test_exact_list_row_key_cmd_is_not_replaced_by_dummy_key(monkeypatch): + """(b) A ``key_cmd`` row mints a real per-request bearer, so it is NOT keyless. + + ``key_cmd`` names a command that prints a short-lived bearer; both wire + clients accept a callable api_key and mint one per request. Handing the + endpoint ``dummy-key`` instead — because the row carries no literal + ``api_key`` — is a guaranteed 401 against an endpoint that does want auth. + """ + built = {} + + def _token_provider(): + return "minted-bearer-sentinel" + + def _build(key_cmd, name): + built["key_cmd"] = key_cmd + built["name"] = name + return _token_provider + + _fake_command_token_source(monkeypatch, _build) + + init_kwargs = _run_composed_send( + monkeypatch, + _exact_list_row_cfg(drop=("api_key",), key_cmd="print-omni-bearer"), + _ambient_runtime(), + "session-1806-owned-key-cmd", + ) + + assert built["key_cmd"] == "print-omni-bearer", "key_cmd was never consulted" + assert built["name"] == "omni" + assert init_kwargs["api_key"] is _token_provider, "row's key_cmd token source lost" + assert init_kwargs["api_key"]() == "minted-bearer-sentinel" + assert init_kwargs["api_key"] != config.KEYLESS_CUSTOM_API_KEY + assert init_kwargs["api_key"] != "keyed-key-sentinel-abc" + assert init_kwargs["base_url"] == _LIST_ROW_URL + assert init_kwargs["provider"] == "custom" + _assert_side_fields(init_kwargs, _FOREIGN_AMBIENT_SIDE_FIELDS, "exact list row key_cmd") + + +def test_exact_list_row_unbuildable_key_cmd_still_refuses_dummy_key(monkeypatch): + """(b) ``dummy-key`` asserts "this endpoint is unauthenticated" — never a guess. + + When the token provider cannot be built (older agent build, broken command + spec) the endpoint is still an authenticated one whose credential is missing. + The placeholder would report that as an opaque 401, and sending with NO key + is no safer: an agent built with ``api_key=None`` never reaches an explicit + client, so ``_routed_client_kwargs()`` re-resolves a provider and the turn + leaves for whatever credential init finds next. The route is terminal. + """ + + def _build(_key_cmd, _name): + raise RuntimeError("command token source unavailable") + + _fake_command_token_source(monkeypatch, _build) + + captured, apperrors = _run_composed_send_expecting_refusal( + monkeypatch, + _exact_list_row_cfg(drop=("api_key",), key_cmd="print-omni-bearer"), + _ambient_runtime(), + "session-1806-owned-key-cmd-broken", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + ) + + payload = apperrors[-1] + assert "produced no API key" in payload["message"], payload + assert "key_cmd" in payload["hint"], ( + f"the refusal must name the credential setting to fix: {payload}" + ) + # Neither the placeholder nor the ambient keyed credential may appear + # anywhere on the refusal path. + assert config.KEYLESS_CUSTOM_API_KEY not in str(payload), payload + assert "keyed-key-sentinel-abc" not in str(payload), payload + assert not captured.get("explicit_client_kwargs_calls"), ( + "a client was configured for a route whose declared credential failed" + ) + + +def test_exact_list_row_pool_credential_and_pool_object_both_survive(monkeypatch): + """(b) A pooled row keeps the pool's key AND the pool object — from ITS endpoint. + + The credential and the ``credential_pool`` the agent rotates it with come + from one lookup keyed on the ROW's base_url. Clearing the pool (while keeping + its key) leaves the agent unable to rotate; keeping the ambient pool points + rotation at the previous authority; falling back to ``dummy-key`` drops the + credential entirely. + """ + pool_sentinel = ["list-row-pool-sentinel"] + seen = {} + + def _before_send(): + runtime_module = sys.modules["hermes_cli.runtime_provider"] + + def _try_resolve_from_custom_pool( + base_url, provider_label, api_mode_override=None, provider_name=None + ): + seen["base_url"] = base_url + seen["provider_name"] = provider_name + if base_url != _LIST_ROW_URL: + return None + return {"api_key": "pool-key-sentinel", "credential_pool": pool_sentinel} + + runtime_module._try_resolve_from_custom_pool = _try_resolve_from_custom_pool + + init_kwargs = _run_composed_send( + monkeypatch, + _exact_list_row_cfg(drop=("api_key",)), + _ambient_runtime(), + "session-1806-owned-pool", + before_send=_before_send, + ) + + # The pool was looked up for the ROW's endpoint, not the ambient one. + assert seen["base_url"] == _LIST_ROW_URL + assert seen["provider_name"] == "omni" + assert init_kwargs["api_key"] == "pool-key-sentinel" + assert init_kwargs["api_key"] != config.KEYLESS_CUSTOM_API_KEY + assert init_kwargs["api_key"] != "keyed-key-sentinel-abc" + _assert_side_fields( + init_kwargs, + {**_FOREIGN_AMBIENT_SIDE_FIELDS, "credential_pool": pool_sentinel}, + "exact list row pool", + ) + assert init_kwargs["credential_pool"] is pool_sentinel + assert init_kwargs["credential_pool"] != _AMBIENT_SIDE_FIELD_RUNTIME["credential_pool"] + + +# ── (c) keyed-only record's own side fields ────────────────────────────────── + + +_KEYED_ONLY_OWNED_CFG = { + "model": {"default": "active/model", "provider": "custom:active"}, + "providers": { + "custom:omni": { + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", + # Side fields the KEYED record declares for itself. + "api_mode": "anthropic_messages", + "credential_pool": ["keyed-pool-sentinel"], + "acp_command": "keyed-acp-sentinel", + "acp_args": ["--keyed-arg-sentinel"], + }, + }, + "custom_providers": [ + { + "name": "active", + "base_url": "https://active.example/v1", + "api_key": "active-key", + }, + ], +} + + +def test_keyed_only_record_keeps_the_side_fields_it_owns(monkeypatch): + """(c) No list row: the keyed record is the authority for ITS side fields too. + + ``providers['custom:omni']`` supplies the endpoint and the credential, so it + also owns the pool, wire protocol and ACP transport it declares. Clearing + them because the route is ``custom:`` throws away the record's own + configuration; taking the ambient values (all four differ here) routes the + send through the previous authority. + """ + init_kwargs = _run_composed_send( + monkeypatch, + copy.deepcopy(_KEYED_ONLY_OWNED_CFG), + _ambient_runtime( + api_mode="chat_completions", + command="ambient-acp-sentinel", + args=["--ambient-arg-sentinel"], + credential_pool=["ambient-pool-sentinel"], + ), + "session-1806-keyed-owned", + ) + + assert init_kwargs["base_url"] == "https://keyed-url-sentinel.example/v1" + assert init_kwargs["api_key"] == "keyed-key-sentinel-abc" + assert init_kwargs["provider"] == "custom" + _assert_side_fields( + init_kwargs, + { + "api_mode": "anthropic_messages", + "credential_pool": ["keyed-pool-sentinel"], + "acp_command": "keyed-acp-sentinel", + "acp_args": ["--keyed-arg-sentinel"], + }, + "keyed-only owned side fields", + ) + + +def test_keyed_only_record_without_side_fields_keeps_same_endpoint_runtime(monkeypatch): + """(c) The clear is provenance-driven, not slug-driven. + + Here the keyed record declares no side fields and the runtime resolved the + SAME endpoint the record owns, so the runtime's values are same-authority. + Clearing them would strip a legitimately-pooled Anthropic-protocol endpoint + of its pool and protocol just because the route is spelled ``custom:``. + """ + cfg_dict = copy.deepcopy(_KEYED_ONLY_OWNED_CFG) + for field in ("api_mode", "credential_pool", "acp_command", "acp_args"): + cfg_dict["providers"]["custom:omni"].pop(field) + + init_kwargs = _run_composed_send( + monkeypatch, + cfg_dict, + _ambient_runtime(), + "session-1806-keyed-same-authority", + ) + + assert init_kwargs["base_url"] == "https://keyed-url-sentinel.example/v1" + _assert_side_fields( + init_kwargs, + { + "api_mode": _AMBIENT_SIDE_FIELD_RUNTIME["api_mode"], + "credential_pool": _AMBIENT_SIDE_FIELD_RUNTIME["credential_pool"], + "acp_command": _AMBIENT_SIDE_FIELD_RUNTIME["command"], + "acp_args": _AMBIENT_SIDE_FIELD_RUNTIME["args"], + }, + "keyed-only same-authority runtime", + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Non-streaming route consumers +# +# Four WebUI consumers build their own AIAgent outside the streaming path. +# Each resolved the endpoint deterministically (``resolve_model_provider`` now +# returns the exact ``custom_providers[]`` row's URL) and then applied the +# custom-provider connection with a FILL-ONLY pattern: +# +# if not api_key and _cp_key: api_key = _cp_key +# +# Because ``resolve_runtime_provider`` had already supplied a truthy key from the +# same-slug keyed ``providers:`` record, the guard never fired — so the final +# constructor received the LIST row's URL paired with the KEYED row's API key. +# Each test below asserts on the FINAL constructor kwargs, so it fails on that +# mixed bundle and passes only once the row is applied atomically. +# ───────────────────────────────────────────────────────────────────────────── + + +class _RouteAgentCaptured(Exception): + """Sentinel raised from the double's __init__ once the bundle is captured. + + All four consumers construct the agent and immediately use it (compress, + run_conversation, text completion). Stopping at construction keeps each test + scoped to the routing contract instead of to downstream persistence. + """ + + +# The runtime dict the route consumers see by default: production shape, where +# ``resolve_runtime_provider`` scans ``providers:`` first and so hands back the +# KEYED endpoint and the KEYED key for this slug. +_ROUTE_KEYED_RUNTIME = { + "provider": "custom:omni", + "base_url": "https://keyed-url-sentinel.example/v1", + "api_key": "keyed-key-sentinel-abc", +} + + +def _setup_route_consumer_runtime( + monkeypatch, session_messages=None, cfg_dict=None, runtime_dict=None +): + """Compose the keyed-vs-list config around a capturing route agent. + + Returns ``(captured, fake_session)``. ``captured["init_kwargs"]`` holds the + FINAL constructor bundle the consumer under test built. + + ``cfg_dict``/``runtime_dict`` override the defaults so a test can hand the + consumers an exact row that OWNS side fields (``api_mode``, + ``credential_pool``, ACP transport) and an ambient runtime that reports + different ones -- the probe for whether the complete bundle, or just its + three connection fields, reaches the constructor. + """ + import types as _types + from unittest import mock + + import api.config as _config + import api.oauth + import api.routes as routes + + cfg_dict = copy.deepcopy(_KEYED_VS_LIST_CFG if cfg_dict is None else cfg_dict) + monkeypatch.setattr(_config, "cfg", dict(cfg_dict), raising=False) + monkeypatch.setattr(_config, "get_config", lambda: copy.deepcopy(cfg_dict)) + + runtime_dict = copy.deepcopy( + _ROUTE_KEYED_RUNTIME if runtime_dict is None else runtime_dict + ) + fake_runtime_module = _types.ModuleType("hermes_cli.runtime_provider") + fake_runtime_module.resolve_runtime_provider = mock.Mock( + return_value=runtime_dict + ) + fake_hermes_cli = _types.ModuleType("hermes_cli") + fake_hermes_cli.__path__ = [] + fake_hermes_cli.runtime_provider = fake_runtime_module + monkeypatch.setitem(sys.modules, "hermes_cli", fake_hermes_cli) + monkeypatch.setitem(sys.modules, "hermes_cli.runtime_provider", fake_runtime_module) + monkeypatch.setattr( + api.oauth, + "resolve_runtime_provider_with_anthropic_env_lock", + lambda resolver, **kwargs: resolver(**kwargs), + ) + + captured = {} + + class CapturingRouteAgent: + # Every constructor-routing field must be a NAMED parameter, for the + # same reason as the streaming double: the route consumers gate each + # optional kwarg on ``inspect.signature(AIAgent.__init__)``, so a double + # that swallowed them into ``**kwargs`` would filter out exactly the + # fields these tests pin and pass vacuously. + def __init__( + self, + model=None, + provider=None, + base_url=None, + api_key=None, + api_mode=None, + acp_command=None, + acp_args=None, + credential_pool=None, + **kwargs, + ): + captured["init_kwargs"] = { + "model": model, + "provider": provider, + "base_url": base_url, + "api_key": api_key, + "api_mode": api_mode, + "acp_command": acp_command, + "acp_args": acp_args, + "credential_pool": credential_pool, + **kwargs, + } + raise _RouteAgentCaptured("captured") + + monkeypatch.setattr(routes, "require_ai_agent_class", lambda: CapturingRouteAgent) + monkeypatch.setattr(routes, "ensure_agent_runtime_current", lambda *_a, **_k: None) + monkeypatch.setattr(routes, "_resolve_cli_toolsets", lambda *_a, **_k: []) + monkeypatch.setattr(routes, "_session_is_subagent_view_only", lambda *_a, **_k: False) + + class _FakeSession: + def __init__(self): + self.session_id = "session-1806-route" + self.title = "Test" + self.workspace = "/tmp" + self.model = "antigravity/gemini-3.7-flash-tiered" + self.model_provider = "custom:omni" + self.messages = list(session_messages or []) + self.context_messages = list(session_messages or []) + self.active_stream_id = None + self.pending_user_message = None + self.pending_attachments = [] + self.pending_started_at = None + self.pending_user_source = None + self.profile = None + + def save(self, *_a, **_k): + return None + + return captured, _FakeSession() + + +def _assert_list_row_not_keyed(init_kwargs, label): + """The endpoint AND the credential must both come from the list row.""" + assert init_kwargs["base_url"] == _LIST_ROW_URL, label + assert init_kwargs["api_key"] == _LIST_ROW_KEY, label + assert init_kwargs["base_url"] != "https://keyed-url-sentinel.example/v1", label + assert init_kwargs["api_key"] != "keyed-key-sentinel-abc", label + assert init_kwargs["provider"] == "custom", label + + +def _four_route_messages(): + return [ + {"role": "user", "content": "one", "timestamp": 1.0, "_ts": 1.0}, + {"role": "assistant", "content": "two", "timestamp": 2.0, "_ts": 2.0}, + {"role": "user", "content": "three", "timestamp": 3.0, "_ts": 3.0}, + {"role": "assistant", "content": "four", "timestamp": 4.0, "_ts": 4.0}, + ] + + +# Each driver composes the harness the way its consumer needs it and returns the +# FINAL constructor kwargs. Sharing them keeps the two contracts below -- +# "the endpoint and credential are one record's" and "the record's side fields +# reach the constructor too" -- asserted over the SAME five consumers, so a new +# consumer cannot be added to one list and forgotten in the other. + + +def _drive_sync_chat_route(monkeypatch, cfg_dict=None, runtime_dict=None): + """Consumer 1/5: POST /api/chat.""" + import api.routes as routes + + captured, fake_session = _setup_route_consumer_runtime( + monkeypatch, + session_messages=_four_route_messages(), + cfg_dict=cfg_dict, + runtime_dict=runtime_dict, + ) + monkeypatch.setattr(routes, "get_session", lambda _sid: fake_session) + monkeypatch.setattr(routes, "_agent_runtime_barrier_response", lambda **_k: None) + monkeypatch.setattr(routes, "resolve_trusted_workspace", lambda ws: "/tmp") + monkeypatch.setattr( + routes, "_get_session_agent_lock", lambda _sid: contextlib.nullcontext() + ) + monkeypatch.setattr( + routes, "_read_profile_model_config", lambda *_a, **_k: (None, None, None) + ) + monkeypatch.setattr( + routes, + "_resolve_compatible_session_model_state", + lambda model, provider, **_k: (model, provider), + ) + monkeypatch.setattr(routes, "j", lambda _handler, payload, **_k: payload) + monkeypatch.setattr(routes, "bad", lambda _handler, message, *_a, **_k: {"error": message}) + + with pytest.raises(_RouteAgentCaptured): + routes._handle_chat_sync( + object(), {"session_id": "session-1806-route", "message": "hi"} + ) + return captured["init_kwargs"] + + +def _drive_manual_compression_route(monkeypatch, cfg_dict=None, runtime_dict=None): + """Consumer 2/5: POST /api/session/{sid}/compress.""" + import api.config as _config + import api.routes as routes + + captured, fake_session = _setup_route_consumer_runtime( + monkeypatch, + session_messages=_four_route_messages(), + cfg_dict=cfg_dict, + runtime_dict=runtime_dict, + ) + monkeypatch.setattr(routes, "get_session", lambda _sid: fake_session) + monkeypatch.setattr( + _config, "_get_session_agent_lock", lambda _sid: contextlib.nullcontext() + ) + monkeypatch.setattr(routes, "j", lambda _handler, payload, **_k: payload) + monkeypatch.setattr(routes, "bad", lambda _handler, message, *_a, **_k: {"error": message}) + + routes._handle_session_compress(object(), {"session_id": "session-1806-route"}) + return captured["init_kwargs"] + + +def _decline_auxiliary_client(monkeypatch, recorded_main_runtime=None): + """Force the auxiliary-model shortcut to decline. + + The consumers that have one would otherwise return before constructing the + main-model agent -- and that constructor is the bundle under test. + """ + import types as _types + + fake_aux = _types.ModuleType("agent.auxiliary_client") + + def _get_text_auxiliary_client(_task, main_runtime=None): + if recorded_main_runtime is not None: + recorded_main_runtime.update(main_runtime or {}) + return (None, None) + + fake_aux.get_text_auxiliary_client = _get_text_auxiliary_client + monkeypatch.setitem(sys.modules, "agent.auxiliary_client", fake_aux) + + +def _drive_commit_message_route( + monkeypatch, cfg_dict=None, runtime_dict=None, recorded_main_runtime=None +): + """Consumer 3/5: LLM git commit-message generation.""" + import api.routes as routes + + captured, fake_session = _setup_route_consumer_runtime( + monkeypatch, cfg_dict=cfg_dict, runtime_dict=runtime_dict + ) + _decline_auxiliary_client(monkeypatch, recorded_main_runtime) + + with pytest.raises(_RouteAgentCaptured): + routes._llm_git_commit_message("sys", "user", session=fake_session) + return captured["init_kwargs"] + + +def _drive_handoff_summary_route(monkeypatch, cfg_dict=None, runtime_dict=None): + """Consumer 4/5: on-demand handoff summary.""" + import api.models as models + import api.routes as routes + + captured, fake_session = _setup_route_consumer_runtime( + monkeypatch, + session_messages=_four_route_messages(), + cfg_dict=cfg_dict, + runtime_dict=runtime_dict, + ) + monkeypatch.setattr(models, "get_session", lambda _sid: fake_session) + monkeypatch.setattr( + models, + "count_conversation_rounds", + lambda _sid, since=None: models.CONVERSATION_ROUND_THRESHOLD + 1, + ) + monkeypatch.setattr( + models, "get_cli_session_messages", lambda _sid: _four_route_messages() + ) + monkeypatch.setattr(routes, "j", lambda _handler, payload, **_k: payload) + monkeypatch.setattr(routes, "bad", lambda _handler, message, *_a, **_k: {"error": message}) + + routes._handle_handoff_summary(object(), {"session_id": "session-1806-route"}) + return captured["init_kwargs"] + + +def _drive_update_summary_route( + monkeypatch, cfg_dict=None, runtime_dict=None, recorded_main_runtime=None +): + """Consumer 5/5: update summary (``_llm_update_summary``). + + This one resolves ``get_effective_default_model()`` rather than the session's + model, so the default has to name the slug under test. + """ + import api.routes as routes + + cfg_dict = copy.deepcopy(_KEYED_VS_LIST_CFG if cfg_dict is None else cfg_dict) + cfg_dict["model"] = { + "default": "@custom:omni:antigravity/gemini-3.7-flash-tiered", + "provider": "custom:omni", + } + captured, _fake_session = _setup_route_consumer_runtime( + monkeypatch, cfg_dict=cfg_dict, runtime_dict=runtime_dict + ) + _decline_auxiliary_client(monkeypatch, recorded_main_runtime) + + with pytest.raises(_RouteAgentCaptured): + routes._llm_update_summary("sys", "user", active_profile=None) + return captured["init_kwargs"] + + +_ROUTE_CONSUMER_DRIVERS = [ + (_drive_sync_chat_route, "sync chat (/api/chat)"), + (_drive_manual_compression_route, "manual compression (/compress)"), + (_drive_commit_message_route, "git commit message"), + (_drive_handoff_summary_route, "handoff summary"), + (_drive_update_summary_route, "update summary"), +] + + +@pytest.mark.parametrize( + "driver,label", _ROUTE_CONSUMER_DRIVERS, ids=[d[1] for d in _ROUTE_CONSUMER_DRIVERS] +) +def test_route_consumers_apply_exact_list_row_atomically(monkeypatch, driver, label): + """Every non-streaming consumer applies the exact row's URL *and* its key.""" + _assert_list_row_not_keyed(driver(monkeypatch), label) + + +# The two consumers whose auxiliary-client shortcut can answer the request +# outright -- for them ``main_runtime`` is the only carrier of the resolved +# authority, because AIAgent is never built. +_AUXILIARY_ROUTE_DRIVERS = [ + (_drive_commit_message_route, "git commit message"), + (_drive_update_summary_route, "update summary"), +] + + +@pytest.mark.parametrize( + "driver,label", + _AUXILIARY_ROUTE_DRIVERS, + ids=[d[1] for d in _AUXILIARY_ROUTE_DRIVERS], +) +def test_auxiliary_routes_hand_the_row_connection_to_the_auxiliary_client( + monkeypatch, driver, label +): + """The aux-client shortcut must see the row's connection, not the keyed one. + + It runs BEFORE the main-model constructor, so a consumer that resolved the + bundle only on the fallback path would still send this request to the keyed + endpoint. + """ + recorded_main_runtime = {} + driver(monkeypatch, recorded_main_runtime=recorded_main_runtime) + + assert recorded_main_runtime.get("base_url") == _LIST_ROW_URL, label + assert recorded_main_runtime.get("api_key") == _LIST_ROW_KEY, label + + +# ── The complete bundle, not just its three connection fields ──────────────── +# +# ``apply_custom_provider_connection_authority`` returns ``(provider, api_key, +# base_url)``. Every consumer above used to apply exactly that and construct the +# agent from it, so an exact row's ``api_mode`` and ``credential_pool`` -- the +# wire protocol the send speaks and the credential source it rotates -- were +# truncated away before AIAgent ever saw them, while the streaming path carried +# them through. The row below OWNS both, and the ambient runtime reports +# different values for all four side fields, so a truncating consumer fails on +# ``None`` and a pass-through consumer fails on the ambient sentinel. + + +_ROUTE_ROW_POOL_SENTINEL = ["list-row-pool-sentinel"] + +_ROUTE_OWNED_SIDE_FIELD_CFG = _exact_list_row_cfg( + api_mode="anthropic_messages", + credential_pool=_ROUTE_ROW_POOL_SENTINEL, +) + +# The ambient runtime disagrees on every side field it reports. +_ROUTE_AMBIENT_RUNTIME = { + **_ROUTE_KEYED_RUNTIME, + "api_mode": "chat_completions", + "credential_pool": ["ambient-pool-sentinel"], + "command": "ambient-acp-sentinel", + "args": ["--ambient-arg-sentinel"], +} + + +@pytest.mark.parametrize( + "driver,label", _ROUTE_CONSUMER_DRIVERS, ids=[d[1] for d in _ROUTE_CONSUMER_DRIVERS] +) +def test_route_consumers_pass_the_exact_rows_side_fields_to_the_constructor( + monkeypatch, driver, label +): + """The exact row's ``api_mode``/``credential_pool`` reach the FINAL constructor.""" + init_kwargs = driver( + monkeypatch, + cfg_dict=copy.deepcopy(_ROUTE_OWNED_SIDE_FIELD_CFG), + runtime_dict=copy.deepcopy(_ROUTE_AMBIENT_RUNTIME), + ) + + _assert_list_row_not_keyed(init_kwargs, label) + _assert_side_fields( + init_kwargs, + { + "api_mode": "anthropic_messages", + "credential_pool": _ROUTE_ROW_POOL_SENTINEL, + # The row declares no ACP transport and owns a different endpoint + # than the runtime, so the ambient subprocess is provably foreign. + "acp_command": None, + "acp_args": None, + }, + f"{label}: exact row side fields", + ) + assert init_kwargs["api_mode"] != _ROUTE_AMBIENT_RUNTIME["api_mode"], ( + f"{label}: the ambient wire protocol reached the constructor" + ) + assert init_kwargs["credential_pool"] != _ROUTE_AMBIENT_RUNTIME["credential_pool"], ( + f"{label}: the ambient credential pool reached the constructor" + ) + + +@pytest.mark.parametrize( + "driver,label", _ROUTE_CONSUMER_DRIVERS, ids=[d[1] for d in _ROUTE_CONSUMER_DRIVERS] +) +def test_route_consumers_clear_foreign_ambient_side_fields(monkeypatch, driver, label): + """A row that owns NO side fields still strips the ambient provider's. + + Complement of the test above: "carry the complete bundle" must not degrade + into "pass the runtime's side fields through". The row here declares none of + them and owns a different endpoint, so all four are provably foreign. + """ + init_kwargs = driver( + monkeypatch, runtime_dict=copy.deepcopy(_ROUTE_AMBIENT_RUNTIME) + ) + + _assert_list_row_not_keyed(init_kwargs, label) + _assert_side_fields(init_kwargs, _FOREIGN_AMBIENT_SIDE_FIELDS, f"{label}: foreign ambient") + + +@pytest.mark.parametrize( + "driver,label", + _AUXILIARY_ROUTE_DRIVERS, + ids=[d[1] for d in _AUXILIARY_ROUTE_DRIVERS], +) +def test_auxiliary_routes_hand_the_complete_bundle_to_the_auxiliary_client( + monkeypatch, driver, label +): + """``main_runtime`` carries the WHOLE bundle, not its three connection fields. + + When the auxiliary client answers, AIAgent is bypassed entirely, so the + side fields that never entered ``main_runtime`` were simply lost: the exact + row's ``api_mode: anthropic_messages`` silently degraded to the aux client's + default wire protocol, and the credential pool/ACP transport the row owns + never reached the send. The aux dict must therefore agree with the fallback + constructor field for field -- same authority, whichever path answers. + """ + import api.routes as routes + + recorded_main_runtime = {} + init_kwargs = driver( + monkeypatch, + cfg_dict=copy.deepcopy(_ROUTE_OWNED_SIDE_FIELD_CFG), + runtime_dict=copy.deepcopy(_ROUTE_AMBIENT_RUNTIME), + recorded_main_runtime=recorded_main_runtime, + ) + + _assert_list_row_not_keyed(recorded_main_runtime, f"{label}: aux main_runtime") + _assert_side_fields( + recorded_main_runtime, + { + "api_mode": "anthropic_messages", + "credential_pool": _ROUTE_ROW_POOL_SENTINEL, + # The row declares no ACP transport and owns a different endpoint + # than the runtime, so the ambient subprocess is provably foreign. + "acp_command": None, + "acp_args": None, + }, + f"{label}: aux main_runtime side fields", + ) + assert recorded_main_runtime["api_mode"] != _ROUTE_AMBIENT_RUNTIME["api_mode"], ( + f"{label}: the ambient wire protocol reached the auxiliary client" + ) + assert ( + recorded_main_runtime["credential_pool"] + != _ROUTE_AMBIENT_RUNTIME["credential_pool"] + ), f"{label}: the ambient credential pool reached the auxiliary client" + + for field in ("provider", "model", "base_url", "api_key") + tuple( + routes._AGENT_BUNDLE_SIDE_FIELDS + ): + assert recorded_main_runtime.get(field) == init_kwargs[field], ( + f"{label}: aux {field} disagrees with the fallback constructor, so " + "which path answers decides the authority" + ) + assert recorded_main_runtime["model"], f"{label}: aux main_runtime lost the model" + + +@pytest.mark.parametrize( + "driver,label", + _AUXILIARY_ROUTE_DRIVERS, + ids=[d[1] for d in _AUXILIARY_ROUTE_DRIVERS], +) +def test_auxiliary_routes_clear_foreign_ambient_side_fields(monkeypatch, driver, label): + """A row owning no side fields strips the ambient provider's from the aux dict too. + + Complement of the test above: "carry the complete bundle into + ``main_runtime``" must not degrade into "pass the runtime's side fields + through" on the path where nothing downstream re-resolves them. + """ + recorded_main_runtime = {} + driver( + monkeypatch, + runtime_dict=copy.deepcopy(_ROUTE_AMBIENT_RUNTIME), + recorded_main_runtime=recorded_main_runtime, + ) + + _assert_list_row_not_keyed(recorded_main_runtime, f"{label}: aux main_runtime") + _assert_side_fields( + recorded_main_runtime, + _FOREIGN_AMBIENT_SIDE_FIELDS, + f"{label}: aux foreign ambient", + ) + + +def test_capturing_route_agent_exposes_every_runtime_constructor_field(monkeypatch): + """Guard the guard, route edition. + + The consumers gate each optional kwarg on + ``inspect.signature(AIAgent.__init__).parameters``. A double that swallowed + ``api_mode`` / ``credential_pool`` / ``acp_command`` / ``acp_args`` into + ``**kwargs`` would make every assertion above pass vacuously, because the + fields would never be passed at all. + """ + import inspect + + import api.routes as routes + + _captured, _fake_session = _setup_route_consumer_runtime(monkeypatch) + agent_cls = routes.require_ai_agent_class() + params = set(inspect.signature(agent_cls.__init__).parameters) + + for field in routes._AGENT_BUNDLE_SIDE_FIELDS: + assert field in params, ( + f"the route double hides {field} behind **kwargs, so the " + "signature gate would filter it out and the assertions would be vacuous" + ) + + +def test_production_composed_retry_caches_agent_under_recomputed_signature(monkeypatch): + """Retry agents are cached under the healed bundle's signature, not the stale initial one.""" + import api.streaming as streaming + + computed_signatures = [] + real_compute_sig = streaming._compute_agent_cache_signature + + def tracking_compute_sig(*args, **kwargs): + sig = real_compute_sig(*args, **kwargs) + computed_signatures.append(sig) + return sig + + monkeypatch.setattr(streaming, "_compute_agent_cache_signature", tracking_compute_sig) + + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, + dict(_KEYED_VS_LIST_CFG), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + session_id="session-1806-retry-cache-sig", + fail_first="returned_error", + ) + # Give self-heal a distinct provider/endpoint so initial and healed signatures differ + healed_rt = dict(_AMBIENT_SIDE_FIELD_RUNTIME) + healed_rt["base_url"] = "https://healed-endpoint.example/v1" + monkeypatch.setattr( + streaming, + "_attempt_credential_self_heal", + lambda *_args, **_kwargs: dict(healed_rt), + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id="session-1806-retry-cache-sig", + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + assert len(computed_signatures) >= 2, "signature was not recomputed on retry" + with config.SESSION_AGENT_CACHE_LOCK: + cached = config.SESSION_AGENT_CACHE.get("session-1806-retry-cache-sig") + assert cached is not None, "retry agent was not cached" + cached_agent, cached_sig = cached + assert cached_agent is not None + assert cached_sig == computed_signatures[-1] + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + +# ───────────────────────────────────────────────────────────────────────────── +# Identity-owned selection, and the two ways a route can be "keyless" +# +# Selection for a named ``custom:`` must be owned by that identity: an +# exact normalized ``custom_providers[]`` row, an exact keyed record, or an +# explicitly matching ``model:``/bare-``custom`` authority. "The list happens to +# hold exactly one row, so use it" is NOT ownership — it resolved ``custom:ghost`` +# to the endpoint AND credential of a sole unrelated row named ``omni``, sending +# the prompt and that credential to a provider the user never named. +# +# The second half of the same contract is ``dummy-key``. It asserts "this +# endpoint is UNAUTHENTICATED", so it may only appear when the record declares no +# credential source at all. A DECLARED credential that failed to resolve (unset +# ``${ENV}``, ``key_env`` naming a missing variable, a pool that yielded nothing) +# is a misconfiguration to surface, not an unauthenticated endpoint: substituting +# the placeholder there reports it as an opaque 401 from the endpoint instead. +# ───────────────────────────────────────────────────────────────────────────── + + +_SOLE_UNRELATED_ROW_CFG = { + "model": {"default": "active/model", "provider": "custom:omni"}, + "custom_providers": [ + { + "name": "omni", + "base_url": "https://omni.example/v1", + "api_key": "omni-key", + }, + ], +} + + +def _with_direct_config(monkeypatch, cfg_dict): + """Point both ``config.cfg`` and ``get_config()`` at ``cfg_dict``.""" + monkeypatch.setattr(config, "get_config", lambda: copy.deepcopy(cfg_dict)) + monkeypatch.setitem(config.cfg, "custom_providers", cfg_dict.get("custom_providers", [])) + monkeypatch.setitem(config.cfg, "model", cfg_dict.get("model", {})) + monkeypatch.setitem(config.cfg, "providers", cfg_dict.get("providers", {})) + + +def test_unknown_named_slug_does_not_select_a_sole_unrelated_list_row(monkeypatch): + """The reviewer's probe: ``custom:ghost`` must not inherit sole row ``omni``. + + ``_select_custom_provider_record`` used to append ``custom_providers[0]`` + whenever the list held exactly one row, then accept it merely because it had + a key or a URL. Neither test is ownership: ``ghost`` and ``omni`` are + different identities, so the pair belongs to ``omni`` alone. + """ + _with_direct_config(monkeypatch, _SOLE_UNRELATED_ROW_CFG) + + assert config.resolve_custom_provider_connection("custom:ghost") == (None, None) + # The row is still authoritative for its OWN slug. + assert config.resolve_custom_provider_connection("custom:omni") == ( + "omni-key", + "https://omni.example/v1", + ) + + +def test_unknown_named_slug_reports_an_explicit_missing_selection(monkeypatch): + """The bundle carries the missing verdict instead of an ambiguous ``None``. + + ``keyless`` must be False on it: "nothing owns this route" is not a claim + that the route is unauthenticated, and it is ``keyless`` alone that gates + ``dummy-key``. + """ + _with_direct_config(monkeypatch, _SOLE_UNRELATED_ROW_CFG) + + ghost = config.resolve_custom_provider_bundle("custom:ghost") + assert ghost is not None, "a named custom route must report its selection outcome" + assert ghost["status"] == config.CUSTOM_SELECTION_MISSING + assert ghost["record"] is None + assert ghost["base_url"] is None + assert ghost["api_key"] is None + assert ghost["keyless"] is False, "a missing route must not be reported keyless" + assert ghost["owned"] == {} + + owned = config.resolve_custom_provider_bundle("custom:omni") + assert owned["status"] == config.CUSTOM_SELECTION_EXACT + assert owned["base_url"] == "https://omni.example/v1" + + +def test_unknown_named_slug_keeps_no_ambient_connection(monkeypatch): + """The merge refuses the ambient provider's URL/key/pool for an unowned slug.""" + _with_direct_config(monkeypatch, _SOLE_UNRELATED_ROW_CFG) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:ghost", + "ambient-key-sentinel", + "https://ambient.example/v1", + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + lookup_provider="custom:ghost", + ) + + assert bundle["base_url"] is None + assert bundle["api_key"] is None + assert bundle["api_key"] != config.KEYLESS_CUSTOM_API_KEY + # The provider must NOT be rewritten to generic ``custom``: that would + # present an unresolvable route as a resolved one. + assert bundle["provider"] == "custom:ghost" + _assert_side_fields(bundle, _FOREIGN_AMBIENT_SIDE_FIELDS, "unowned named slug") + + +# A provider LABEL is not proof of ownership. These two cases are how an unowned +# slug could still walk away with the ambient connection: +# +# 1. the runtime dict names ITSELF ``custom:ghost`` -- a self-assigned string +# on a connection the process authenticated for some other reason; and +# 2. no runtime dict at all, the non-streaming consumers' shape, where +# ``_connection_identity`` falls back to ``resolved_provider`` -- the very +# slug being looked up -- so the route would match itself. +# +# Both must fail closed exactly as the differently-labelled ambient runtime does: +# ownership is decided by config records, and here there are none. + + +_SELF_LABELLED_GHOST_RUNTIME = { + **_AMBIENT_SIDE_FIELD_RUNTIME, + "provider": "custom:ghost", + "base_url": "https://ambient.example/v1", + "api_key": "ambient-key-sentinel", +} + + +def _assert_ghost_bundle_failed_closed(bundle, label): + """No endpoint, no credential, no placeholder, no ambient side fields.""" + assert bundle["base_url"] is None, f"{label}: an unowned slug kept an endpoint" + assert bundle["api_key"] is None, f"{label}: an unowned slug kept a credential" + assert bundle["api_key"] != config.KEYLESS_CUSTOM_API_KEY, ( + f"{label}: an unresolvable route was handed the keyless placeholder" + ) + # Still the NAMED slug: rewriting it to generic ``custom`` would present an + # unresolvable route as a resolved one. + assert bundle["provider"] == "custom:ghost", label + _assert_side_fields(bundle, _FOREIGN_AMBIENT_SIDE_FIELDS, label) + + +@pytest.mark.parametrize( + "runtime_dict,label", + [ + (_SELF_LABELLED_GHOST_RUNTIME, "runtime labels itself custom:ghost"), + (None, "no runtime dict (non-streaming caller shape)"), + ], + ids=["self-labelled-runtime", "no-runtime-dict"], +) +def test_unknown_named_slug_fails_closed_even_when_the_runtime_claims_the_slug( + monkeypatch, runtime_dict, label +): + """A matching provider label must not resurrect the ambient connection. + + Nothing in config owns ``custom:ghost``: not an exact ``custom_providers[]`` + row, not a keyed ``providers:`` record, not a ``model:`` authority. The only + thing pointing at the slug is a string the caller supplied, so keeping the + URL, key, wire protocol and credential pool that came with it would send the + user's prompt and that credential to a provider they never configured. + """ + _with_direct_config(monkeypatch, _SOLE_UNRELATED_ROW_CFG) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:ghost", + "ambient-key-sentinel", + "https://ambient.example/v1", + copy.deepcopy(runtime_dict) if runtime_dict else runtime_dict, + lookup_provider="custom:ghost", + ) + + _assert_ghost_bundle_failed_closed(bundle, label) + assert bundle["base_url"] != "https://ambient.example/v1", label + assert bundle["api_key"] != "ambient-key-sentinel", label + + +def test_unknown_named_slug_connection_view_also_fails_closed(monkeypatch): + """The three-field view reports the same verdict, and ``custom_owned`` False. + + Callers that genuinely construct nothing else still take this view, so the + self-labelled runtime must not reach them with a connection either. + """ + _with_direct_config(monkeypatch, _SOLE_UNRELATED_ROW_CFG) + + provider, api_key, base_url, custom_owned = ( + config.apply_custom_provider_connection_authority( + "custom:ghost", + "ambient-key-sentinel", + "https://ambient.example/v1", + lookup_provider="custom:ghost", + runtime_provider=copy.deepcopy(_SELF_LABELLED_GHOST_RUNTIME), + ) + ) + + assert base_url is None + assert api_key is None + assert provider == "custom:ghost" + assert custom_owned is False, "nothing owns the slug, so the record cannot be reported as owning it" + + +def test_unowned_named_slug_fails_closed_at_the_first_streaming_resolution(monkeypatch): + """The initial resolution refuses the unrelated row — terminally. + + The ambient runtime seeds a truthy URL, key and pool (plus api_mode and an + ACP transport), and config holds exactly ONE custom row — named ``omni``, + with a truthy URL and key of its own. The send asks for ``custom:ghost``, + which nothing owns. + + This covers the FIRST of the three streaming regions that build an agent, + and only that one: the verdict is terminal here, so no agent is constructed, + no turn is sent, and no 401 exists for the two self-heal retries to act on. + Parametrizing ``fail_first`` over this case would re-run identical code and + prove nothing about those retry guards — they are exercised directly by + ``test_retry_abandons_the_heal_*`` below, which start from a ROUTABLE route + and break it mid-turn. + """ + label = "initial send" + session_id = "session-1806-ghost-initial" + + captured, apperrors = _run_composed_send_expecting_refusal( + monkeypatch, + copy.deepcopy(_SOLE_UNRELATED_ROW_CFG), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + session_id, + model="@custom:ghost:antigravity/gemini-3.7-flash-tiered", + ) + + payload = apperrors[-1] + assert "custom:ghost" in payload["message"], f"{label}: {payload}" + assert "is not configured" in payload["message"], f"{label}: {payload}" + + # Nothing that could have been routed to may appear on the refusal path: + # not the unrelated sole row's pair, not the ambient provider's, and never + # the keyless placeholder. + blob = str(payload) + str(captured) + for leaked in ( + "https://omni.example/v1", + "omni-key", + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + config.KEYLESS_CUSTOM_API_KEY, + ): + assert leaked not in blob, f"{label}: {leaked!r} leaked into the refused route" + + assert not captured.get("explicit_client_kwargs_calls"), ( + f"{label}: a client was configured for a slug nothing owns" + ) + + +# ── declared-but-unresolved credentials are NOT keyless ────────────────────── + + +_MISSING_ENV_VAR = "HERMES_TEST_1806_UNSET_CREDENTIAL" + + +def _clear_credential_env(monkeypatch): + """Remove every env var the credential ladder could still mint a key from.""" + monkeypatch.delenv(_MISSING_ENV_VAR, raising=False) + monkeypatch.delenv("CUSTOM_OMNI_API_KEY", raising=False) + + +_UNRESOLVED_CREDENTIAL_ROWS = [ + # A declared ``${ENV}`` reference whose variable is unset. + ({"api_key": "${" + _MISSING_ENV_VAR + "}"}, "env-reference"), + # A declared ``key_env`` naming a variable that does not exist. + ({"key_env": _MISSING_ENV_VAR}, "key-env"), + # A configured credential pool that yields nothing for this endpoint. + ({"credential_pool": ["configured-pool-sentinel"]}, "unavailable-pool"), +] + + +@pytest.mark.parametrize("row_fields,label", _UNRESOLVED_CREDENTIAL_ROWS) +def test_declared_but_unresolved_credential_is_not_keyless(monkeypatch, row_fields, label): + """``keyless`` means "declares no credential", not "resolved no credential". + + Each row here DECLARES a credential source that produces nothing. Reporting + that as keyless is what let ``dummy-key`` be substituted for a genuinely + missing credential, turning a fixable misconfiguration into an opaque 401. + """ + _clear_credential_env(monkeypatch) + cfg_dict = _exact_list_row_cfg(drop=("api_key",), **row_fields) + _with_direct_config(monkeypatch, cfg_dict) + + bundle = config.resolve_custom_provider_bundle("custom:omni") + + assert bundle["api_key"] is None, f"{label}: the credential must not resolve here" + assert bundle["keyless"] is False, ( + f"{label}: a declared credential source was reported as keyless" + ) + + +@pytest.mark.parametrize("row_fields,label", _UNRESOLVED_CREDENTIAL_ROWS) +def test_declared_but_unresolved_credential_never_gets_the_dummy_key( + monkeypatch, row_fields, label +): + """End to end: the send STOPS — neither the placeholder nor a keyless send. + + The endpoint resolves (the row owns it), but the declared credential source + produced nothing. Sending anyway with ``api_key=None`` is not the safe + middle ground it looks like: with only one of the pair truthy, AIAgent falls + through to ``_routed_client_kwargs()`` and re-resolves a provider, so the + turn — and whatever credential init then finds — leaves for an endpoint the + user never chose. The turn ends on the actionable cause instead. + """ + _clear_credential_env(monkeypatch) + + captured, apperrors = _run_composed_send_expecting_refusal( + monkeypatch, + _exact_list_row_cfg(drop=("api_key",), **row_fields), + _ambient_runtime(), + f"session-1806-unresolved-{label}", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + ) + + payload = apperrors[-1] + assert "produced no API key" in payload["message"], f"{label}: {payload}" + assert "custom:omni" in payload["message"], ( + f"{label}: the refusal did not name the provider that failed: {payload}" + ) + assert config.KEYLESS_CUSTOM_API_KEY not in str(payload), ( + f"{label}: keyless placeholder masked a declared-but-missing credential" + ) + assert "keyed-key-sentinel-abc" not in str(payload), ( + f"{label}: the ambient keyed record leaked into the refusal" + ) + # The row owns its endpoint, so nothing may have been dialled with it. + assert not captured.get("explicit_client_kwargs_calls"), f"{label}: a client was configured" + + +def test_row_declaring_no_credential_is_still_keyless(monkeypatch): + """The negative control: a genuinely unauthenticated endpoint keeps ``dummy-key``. + + Local OpenAI-compatible servers routinely run without auth, and tightening + the keyless rule must not take that away — otherwise every keyless setup + regresses into an agent built with no credential at all. + """ + _clear_credential_env(monkeypatch) + + init_kwargs = _run_composed_send( + monkeypatch, + _exact_list_row_cfg(drop=("api_key",)), + _ambient_runtime(), + "session-1806-genuinely-keyless", + ) + + assert init_kwargs["api_key"] == config.KEYLESS_CUSTOM_API_KEY + assert init_kwargs["base_url"] == _LIST_ROW_URL + assert init_kwargs["provider"] == "custom" + + +# ── the exact row's credential is never the same-endpoint keyed row's ─────── +# +# Every unresolved-credential case above puts the keyed record on a DIFFERENT +# endpoint, so the merge's provenance test already proves the ambient credential +# foreign. That leaves the collision untested: a keyed +# ``providers["custom:"]`` record may declare the SAME ``base_url`` as the +# exact ``custom_providers[]`` row, and then endpoint equality alone cannot tell +# the row's own resolution from the keyed row's. Inferring "same authority" from +# the URL there hands the keyed row's key to the exact row — the row's endpoint +# married to somebody else's credential, which is the split-authority merge the +# whole module exists to prevent. +# +# An exact row is a COMPLETE record: its credential comes from its own ladder or +# it does not come at all. + + +_SHARED_ENDPOINT_URL = "https://shared-url-sentinel.example/v1" +_SHARED_ENDPOINT_KEYED_KEY = "shared-endpoint-keyed-sentinel-def" + + +def _shared_endpoint_cfg(**row_fields): + """Exact list row and same-slug keyed record on the IDENTICAL ``base_url``.""" + return { + "model": {"default": "active/model", "provider": "custom:active"}, + "providers": { + "custom:omni": { + "base_url": _SHARED_ENDPOINT_URL, + "api_key": _SHARED_ENDPOINT_KEYED_KEY, + }, + }, + "custom_providers": [ + {"name": "omni", "base_url": _SHARED_ENDPOINT_URL, **row_fields}, + ], + } + + +def _shared_endpoint_runtime(): + """The ambient runtime that resolution of the KEYED record produces. + + Same endpoint as the exact row, carrying the keyed record's credential — the + state in which URL equality stops being evidence of provenance. + """ + return _ambient_runtime( + base_url=_SHARED_ENDPOINT_URL, api_key=_SHARED_ENDPOINT_KEYED_KEY + ) + + +@pytest.mark.parametrize("row_fields,label", _UNRESOLVED_CREDENTIAL_ROWS) +def test_exact_row_unresolved_credential_ignores_same_endpoint_keyed_key( + monkeypatch, row_fields, label +): + """The row DECLARED a credential that produced nothing: the route is terminal. + + The keyed record sharing the endpoint changes nothing about that. If the + merge accepts ``_rt["api_key"]`` because the URLs match, the bundle silently + authenticates the row's endpoint with the keyed row's secret and reports + itself routable, so the user never learns their ``key_env`` is unset. + """ + _clear_credential_env(monkeypatch) + _with_direct_config(monkeypatch, _shared_endpoint_cfg(**row_fields)) + + resolved = config.resolve_custom_provider_bundle("custom:omni") + assert resolved["is_exact"] is True, f"{label}: the exact row was not selected" + assert resolved["keyless"] is False, ( + f"{label}: a declared credential source was reported as keyless" + ) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _SHARED_ENDPOINT_KEYED_KEY, + _SHARED_ENDPOINT_URL, + _shared_endpoint_runtime(), + lookup_provider="custom:omni", + ) + + assert bundle["base_url"] == _SHARED_ENDPOINT_URL, f"{label}: the row lost its endpoint" + assert bundle["api_key"] is None, ( + f"{label}: the exact row borrowed the same-endpoint keyed credential: {bundle!r}" + ) + assert bundle["api_key"] != _SHARED_ENDPOINT_KEYED_KEY, f"{label}: {bundle!r}" + assert bundle["api_key"] != config.KEYLESS_CUSTOM_API_KEY, ( + f"{label}: the placeholder masked a declared-but-missing credential" + ) + verdict = config.custom_provider_route_error(bundle) + assert verdict, f"{label}: an unroutable bundle reported itself routable: {bundle!r}" + assert verdict["reason"] == config.CUSTOM_ROUTE_NO_CREDENTIAL, f"{label}: {verdict}" + + +@pytest.mark.parametrize("row_fields,label", _UNRESOLVED_CREDENTIAL_ROWS) +def test_composed_send_refuses_exact_row_sharing_the_keyed_endpoint( + monkeypatch, row_fields, label +): + """End to end: the turn stops instead of dialling with the keyed row's key. + + The production-composed path is where the borrowed credential would actually + be spent — the endpoint resolves, so nothing downstream would question a + truthy ``api_key``. + """ + _clear_credential_env(monkeypatch) + + captured, apperrors = _run_composed_send_expecting_refusal( + monkeypatch, + _shared_endpoint_cfg(**row_fields), + _shared_endpoint_runtime(), + f"session-1806-shared-endpoint-{label}", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + ) + + payload = apperrors[-1] + assert "produced no API key" in payload["message"], f"{label}: {payload}" + assert "custom:omni" in payload["message"], f"{label}: {payload}" + assert _SHARED_ENDPOINT_KEYED_KEY not in str(payload), ( + f"{label}: the same-endpoint keyed credential leaked into the refusal" + ) + assert config.KEYLESS_CUSTOM_API_KEY not in str(payload), f"{label}: {payload}" + assert not captured.get("explicit_client_kwargs_calls"), ( + f"{label}: a client was configured for a route whose credential failed" + ) + + +def test_exact_row_declaring_no_credential_ignores_the_shared_endpoint_key(monkeypatch): + """The other half of the rule: a keyless row does not borrow the key either. + + The row declares NO credential, which is its own statement that the endpoint + is unauthenticated. ``dummy-key`` follows from the row, not from the keyed + record that happens to point at the same URL — so the route stays routable + and the keyed secret still never leaves the keyed record. + """ + _clear_credential_env(monkeypatch) + _with_direct_config(monkeypatch, _shared_endpoint_cfg()) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _SHARED_ENDPOINT_KEYED_KEY, + _SHARED_ENDPOINT_URL, + _shared_endpoint_runtime(), + lookup_provider="custom:omni", + ) + + assert bundle["base_url"] == _SHARED_ENDPOINT_URL + assert bundle["api_key"] == config.KEYLESS_CUSTOM_API_KEY, ( + f"the keyless row did not keep its own credential verdict: {bundle!r}" + ) + assert bundle["api_key"] != _SHARED_ENDPOINT_KEYED_KEY, bundle + assert config.custom_provider_route_error(bundle) is None, bundle + + +# The exact row's ENDPOINT obeys the same positive-tie rule as every other +# record branch. "Complete record" decides whose endpoint and whose credential +# the route gets; it does not decide the SPELLING of an endpoint both sides +# already agree on. +_ROW_OWN_KEY = "row-own-key-sentinel-ghi" +_FOREIGN_AMBIENT_URL = "https://ambient-sentinel.example/v1" + + +@pytest.mark.parametrize( + "runtime_base_url,label", + [ + (_SHARED_ENDPOINT_URL, "same normalized URL from distinct keyed record"), + (_FOREIGN_AMBIENT_URL, "foreign ambient endpoint"), + ], + ids=["same-normalized-url", "foreign-endpoint"], +) +def test_exact_row_preserves_own_endpoint_beside_distinct_keyed_runtime( + monkeypatch, runtime_base_url, label +): + """An exact row preserves its own declared endpoint spelling beside a distinct record. + + Endpoint equality alone is NOT selected-record provenance. When the runtime + carries credentials from a same-slug keyed record (_SHARED_ENDPOINT_KEYED_KEY) + while the exact row declares _ROW_OWN_KEY, matching normalized URL spelling + does not tie the runtime to this row. The selected exact row's own endpoint + spelling stands in both cases. + """ + _clear_credential_env(monkeypatch) + _with_direct_config( + monkeypatch, + _shared_endpoint_cfg(base_url=_SHARED_ENDPOINT_URL + "/", api_key=_ROW_OWN_KEY), + ) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _SHARED_ENDPOINT_KEYED_KEY, + runtime_base_url, + _ambient_runtime(base_url=runtime_base_url, api_key=_SHARED_ENDPOINT_KEYED_KEY), + lookup_provider="custom:omni", + ) + + assert bundle["base_url"] == _SHARED_ENDPOINT_URL + "/", f"{label}: {bundle!r}" + assert bundle["base_url"] != _FOREIGN_AMBIENT_URL, ( + f"{label}: the row was pointed at the ambient provider's endpoint" + ) + assert bundle["api_key"] == _ROW_OWN_KEY, ( + f"{label}: the exact row did not supply its own credential: {bundle!r}" + ) + assert bundle["api_key"] != _SHARED_ENDPOINT_KEYED_KEY, f"{label}: {bundle!r}" + assert config.custom_provider_route_error(bundle) is None, f"{label}: {bundle!r}" + + +# ───────────────────────────────────────────────────────────────────────────── +# The two 401 self-heal RETRIES are route boundaries too +# +# Everything above stops a terminal route at the FIRST resolution, where no agent +# exists yet. The self-heal retries are a different boundary, and a strictly +# worse one: the route WAS routable when the turn started, an agent was built and +# written to ``SESSION_AGENT_CACHE`` under a valid bundle signature, the provider +# answered 401, and only THEN does the re-resolve run. Whatever that second +# resolution returns is what the retry agent gets constructed from. +# +# That window is real state, not a hypothetical: between the first send and the +# heal the owning ``custom_providers[]`` row can be edited, renamed or deleted in +# Settings, and its ``key_env`` / pool can stop yielding a credential. The +# refreshed bundle is then terminal, and building the retry agent from it hands +# ``_init_openai_client()`` an incomplete pair — so the retry, the send that +# actually carries the user's prompt, is the one that re-enters +# ``_routed_client_kwargs()``. It also overwrites a GOOD cache entry with the +# poisoned agent, so every later turn in the session reuses it. +# +# ``heal_mutate`` runs inside the stubbed ``_attempt_credential_self_heal`` — +# after the first agent has already failed with its 401, before the retry +# re-resolves its bundle — which is the only point at which that transition can +# be staged. Each test below asserts the first send really happened (one agent, +# one turn, one cache write), so a regression that makes the route terminal up +# front cannot pass these by the back door. +# ───────────────────────────────────────────────────────────────────────────── + + +# No ``providers:`` block and a ``model.provider`` naming an UNRELATED slug: the +# single list row is the ONLY authority for ``custom:omni``, so removing or +# renaming it below leaves the slug owned by nothing at all. A same-slug keyed +# record or a ``model:`` block naming ``custom:omni`` would each still be an +# authority and would make the "unowned" cases test something weaker. +_RETRY_OWNED_CFG = { + "model": {"default": "active/model", "provider": "custom:active"}, + "custom_providers": [ + { + "name": "omni", + "base_url": _LIST_ROW_URL, + "api_key": _LIST_ROW_KEY, + }, + ], +} + +_RETRY_MODEL = "@custom:omni:antigravity/gemini-3.7-flash-tiered" + + +def _retry_cfg_rows(**row_fields): + """The owning row with ``row_fields`` applied; ``None`` values drop the field.""" + row = dict(_RETRY_OWNED_CFG["custom_providers"][0]) + for field, value in row_fields.items(): + if value is None: + row.pop(field, None) + else: + row[field] = value + return [row] + + +def _drop_the_owning_row(): + """The row is deleted mid-turn — nothing owns ``custom:omni`` any more.""" + config.cfg["custom_providers"] = [] + + +def _rename_the_owning_row(): + """The row is renamed mid-turn, so it now owns a DIFFERENT slug. + + The pair is still sitting in config, which is exactly the shape that made + the sole-unrelated-row fallback look harmless: ``custom:omni`` must not + inherit it just because it is the only row left. + """ + config.cfg["custom_providers"] = _retry_cfg_rows(name="omni-renamed") + + +def _malform_the_owning_row(): + """The row loses its name, so it names no slug and therefore owns none. + + ``CUSTOM_SELECTION_MALFORMED`` proper describes a bare ``custom:`` lookup + with no slug behind it, which cannot arise mid-turn — the lookup is fixed at + the first resolve. A row that stops naming any identity is the reachable + malformed-record analogue, and it must fail closed the same way. + """ + config.cfg["custom_providers"] = _retry_cfg_rows(name=None) + + +_RETRY_UNOWNED_MUTATIONS = [ + (_drop_the_owning_row, "deleted", "row deleted"), + (_rename_the_owning_row, "renamed", "row renamed to another slug"), + (_malform_the_owning_row, "unnamed", "row lost its name"), +] + + +def _retry_endpoint_mutation(): + """Return a heal mutation that removes only the owning row's endpoint.""" + + def _mutate(): + # Keep the credential so this is the exact credential-only record shape: + # the row still owns the named route, but cannot supply an endpoint. + config.cfg["custom_providers"] = _retry_cfg_rows(base_url=None) + + return _mutate + + +def _retry_credential_mutation(row_fields): + """Return a heal mutation that keeps the endpoint but breaks the credential.""" + + def _mutate(): + # ``api_key`` first so a row that DECLARES one (the ``${ENV}`` shape) + # overrides the drop, while ``key_env`` / ``credential_pool`` rows keep + # it dropped. Either way the row still owns its endpoint. + fields = {"api_key": None, **row_fields} + config.cfg["custom_providers"] = _retry_cfg_rows(**fields) + + return _mutate + + +def _run_composed_retry_expecting_abandoned_heal( + monkeypatch, cfg_dict, runtime_dict, session_id, *, fail_first, heal_mutate +): + """Drive a send that starts routable and turns terminal at the 401 retry. + + Returns ``(captured, apperrors, cache_at_heal, cache_after)``. Both cache + views are read while the worker's entries are still live — ``restore()`` + clears ``SESSION_AGENT_CACHE`` — so the caller can compare what the initial + send cached against what survived the abandoned heal. + """ + import api.streaming as streaming + + cache_at_heal = {} + + def _stage_the_terminal_retry(): + # Snapshot the entry the FIRST (routable) agent was cached under, taken + # before the route is broken. The abandoned retry must leave it exactly + # as it is: replacing it is how one mid-turn config edit would re-route + # every remaining turn in the session. + with config.SESSION_AGENT_CACHE_LOCK: + cache_at_heal.update(config.SESSION_AGENT_CACHE) + heal_mutate() + + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, + copy.deepcopy(cfg_dict), + runtime_dict, + session_id=session_id, + fail_first=fail_first, + heal_mutate=_stage_the_terminal_retry, + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id=session_id, + msg_text="hello", + model=_RETRY_MODEL, + workspace="/tmp", + stream_id=stream_id, + ) + apperrors = _drain_apperrors(q) + with config.SESSION_AGENT_CACHE_LOCK: + cache_after = dict(config.SESSION_AGENT_CACHE) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + return captured, apperrors, cache_at_heal, cache_after + + +def _assert_retry_abandoned( + captured, + apperrors, + cache_at_heal, + cache_after, + session_id, + label, + *, + expected_cause, + expected_initial_url=_LIST_ROW_URL, + expected_initial_key=_LIST_ROW_KEY, +): + """Assert the retry stopped at the refreshed route verdict, not at the 401.""" + # (0) The case is only meaningful if the FIRST send was routable and really + # ran. Without this, a regression that made the route terminal at initial + # resolution would satisfy every assertion below for the wrong reason. + history = captured.get("init_kwargs_history", []) + assert history, f"{label}: the initial send never constructed an agent" + assert history[0]["base_url"] == expected_initial_url, ( + f"{label}: the initial send did not resolve the owning row, so no 401 " + f"retry path was ever reached: {history[0]}" + ) + assert history[0]["api_key"] == expected_initial_key, f"{label}: {history[0]}" + assert cache_at_heal.get(session_id), ( + f"{label}: the initial agent was never cached, so this case cannot show " + f"that the abandoned retry left a good cache entry alone" + ) + + # (1) No second agent — the retry construction is what the guard prevents. + assert len(history) == 1, ( + f"{label}: the retry constructed a second agent on a route that had " + f"become terminal ({len(history)} constructions: {history})" + ) + assert len(captured.get("instances", [])) == 1, label + + # (2) _routed_client_kwargs() is never reached. The one explicit call is the + # initial, complete pair; anything else means an incomplete pair reached + # _init_openai_client() and re-entered provider routing. + routed = captured.get("routed_client_kwargs_calls", []) + assert not routed, ( + f"{label}: the retry agent was built with an incomplete connection pair " + f"{routed}, so _init_openai_client() fell through to " + f"_routed_client_kwargs() and re-resolved a provider" + ) + explicit = captured.get("explicit_client_kwargs_calls", []) + assert explicit == [{"api_key": expected_initial_key, "base_url": expected_initial_url}], ( + f"{label}: expected only the initial send's explicit pair, got {explicit}" + ) + + # (3) The retry turn was never sent. + assert captured.get("run_calls") == 1, ( + f"{label}: run_conversation ran {captured.get('run_calls')!r} times — the " + f"retry turn was sent on a route that no longer resolves" + ) + + # (4) The cache still holds the FIRST agent under the FIRST bundle + # signature. A retry write here would be the durable half of the defect: + # later turns reuse the cached agent without re-resolving at all. + assert list(cache_after) == [session_id], ( + f"{label}: unexpected agent-cache contents {list(cache_after)}" + ) + assert cache_after[session_id][0] is cache_at_heal[session_id][0], ( + f"{label}: the agent cache was rewritten with a retry agent built on an " + f"unroutable bundle" + ) + assert cache_after[session_id][1] == cache_at_heal[session_id][1], ( + f"{label}: a cache entry was written under the invalid bundle's signature" + ) + assert cache_after[session_id][0] is captured["instances"][0], label + + # (5) The client is told the real cause, not the 401 that triggered the heal. + assert apperrors, f"{label}: no controlled failure was emitted" + payload = apperrors[-1] + assert payload["type"] == "provider_unroutable", ( + f"{label}: emitted {payload['type']!r} instead of a provider-route " + f"failure — the 401 is the symptom, the unroutable route is the cause" + ) + assert expected_cause in payload.get("message", ""), ( + f"{label}: the failure did not name {expected_cause!r}: {payload}" + ) + assert "custom:omni" in payload.get("message", ""), ( + f"{label}: the failure did not name the provider that failed: {payload}" + ) + assert payload.get("hint"), f"{label}: the failure named no fix" + + # Nothing the abandoned retry could have been re-routed to may appear on the + # refusal path, and the keyless placeholder may never stand in for it. + blob = str(payload) + for leaked in ( + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + config.KEYLESS_CUSTOM_API_KEY, + ): + assert leaked not in blob, f"{label}: {leaked!r} leaked into the refusal" + return payload + + +@pytest.mark.parametrize("fail_first", ["returned_error", "raised"]) +@pytest.mark.parametrize( + "heal_mutate,mutation_slug,mutation_label", + _RETRY_UNOWNED_MUTATIONS, + ids=[slug for _fn, slug, _label in _RETRY_UNOWNED_MUTATIONS], +) +def test_retry_abandons_the_heal_when_the_slug_stops_being_owned( + monkeypatch, fail_first, heal_mutate, mutation_slug, mutation_label +): + """Case A — the refreshed route is unowned, so the retry must not be built. + + The turn starts on a row that owns ``custom:omni`` outright, so an agent is + constructed with the row's exact pair and the turn is sent. The provider + answers 401. Before the heal re-resolves, the row stops owning the slug. + + Building the retry from that bundle is the whole defect: its key and URL are + empty, which ``_init_openai_client()`` reads as "resolve a provider yourself" + — and the ambient runtime dict the heal returns still carries a truthy + endpoint, credential and pool for a provider the user never named. + """ + label = f"{mutation_label} / {fail_first}" + session_id = f"session-1806-retry-unowned-{fail_first}-{mutation_slug}" + + captured, apperrors, cache_at_heal, cache_after = ( + _run_composed_retry_expecting_abandoned_heal( + monkeypatch, + _RETRY_OWNED_CFG, + _ambient_runtime(), + session_id, + fail_first=fail_first, + heal_mutate=heal_mutate, + ) + ) + + _assert_retry_abandoned( + captured, + apperrors, + cache_at_heal, + cache_after, + session_id, + label, + expected_cause="is not configured", + ) + + +@pytest.mark.parametrize("fail_first", ["returned_error", "raised"]) +@pytest.mark.parametrize("row_fields,credential_label", _UNRESOLVED_CREDENTIAL_ROWS) +def test_retry_abandons_the_heal_when_the_refreshed_credential_resolves_to_nothing( + monkeypatch, fail_first, row_fields, credential_label +): + """Case B — the row still owns the route, but its credential now yields nothing. + + This is the shape a 401 self-heal is most likely to meet in the wild: the + 401 happened BECAUSE the credential went away, so the re-resolve finds the + same row with a declared-but-unresolved source. ``keyless`` is False there, + so no ``dummy-key`` is substituted and the pair stays incomplete — and an + incomplete pair at the constructor is a re-route, not a refusal. + + The retry therefore has to stop, and the turn has to end naming the + credential setting rather than the 401 it produced. + """ + _clear_credential_env(monkeypatch) + label = f"{credential_label} / {fail_first}" + session_id = f"session-1806-retry-nocred-{fail_first}-{credential_label}" + + captured, apperrors, cache_at_heal, cache_after = ( + _run_composed_retry_expecting_abandoned_heal( + monkeypatch, + _RETRY_OWNED_CFG, + _ambient_runtime(), + session_id, + fail_first=fail_first, + heal_mutate=_retry_credential_mutation(row_fields), + ) + ) + + _assert_retry_abandoned( + captured, + apperrors, + cache_at_heal, + cache_after, + session_id, + label, + expected_cause="produced no API key", + ) + + +def test_retry_still_succeeds_when_the_refreshed_route_is_still_routable(monkeypatch): + """The negative control: a heal that re-resolves a GOOD route still retries. + + Without this, every assertion above would also pass if the guards simply + abandoned all self-heals — which would take the #1401 credential-refresh + retry away entirely. Here the mutation swaps the row's key for a NEW one + (the refresh a heal exists to pick up), so the retry must be constructed + with the new pair and the turn must be sent a second time. + """ + session_id = "session-1806-retry-still-routable" + + def _rotate_the_credential(): + config.cfg["custom_providers"] = _retry_cfg_rows(api_key="rotated-key-sentinel") + + import api.streaming as streaming + + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, + copy.deepcopy(_RETRY_OWNED_CFG), + _ambient_runtime(), + session_id=session_id, + fail_first="returned_error", + heal_mutate=_rotate_the_credential, + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id=session_id, + msg_text="hello", + model=_RETRY_MODEL, + workspace="/tmp", + stream_id=stream_id, + ) + apperrors = _drain_apperrors(q) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + history = captured["init_kwargs_history"] + assert len(history) == 2, f"the routable heal did not retry: {history}" + assert history[1]["api_key"] == "rotated-key-sentinel" + assert history[1]["base_url"] == _LIST_ROW_URL + assert captured["run_calls"] == 2, "the refreshed credential never sent a turn" + assert not apperrors, f"a routable retry emitted a failure: {apperrors}" + + +# ───────────────────────────────────────────────────────────────────────────── +# The generic bare-``custom`` authority owns NO named slug +# +# ``providers['custom']`` and a ``model:`` block whose provider is the bare +# string ``custom`` name no slug at all. While they were still eligible +# candidates for every ``custom:`` lookup, ``custom:ghost`` selected one of +# them and inherited its endpoint, credential, credential pool, ``api_mode`` and +# ACP transport — the same wrong-authority pairing the sole-unrelated-row rule +# above exists to prevent, only sourced from ``providers:``/``model:`` instead of +# ``custom_providers[]``. A named route is identity-owned: exact slug match, or +# nothing. +# ───────────────────────────────────────────────────────────────────────────── + + +_BARE_CUSTOM_PROVIDERS_RECORD = { + "base_url": "https://bare-providers-sentinel.example/v1", + "api_key": "bare-providers-key-sentinel", + "api_mode": "anthropic_messages", + "credential_pool": ["bare-pool-sentinel"], + "acp_command": "bare-acp-sentinel", + "acp_args": ["--bare-arg-sentinel"], +} + +_BARE_CUSTOM_MODEL_BLOCK = { + "provider": "custom", + "base_url": "https://bare-model-sentinel.example/v1", + "api_key": "bare-model-key-sentinel", +} + + +def _bare_custom_cfg(*, providers_record=True, model_block=True): + """Config carrying the generic bare-``custom`` authority, but no ``ghost``.""" + cfg_dict = { + "model": {"default": "ghostly/model"}, + "providers": {}, + "custom_providers": [ + { + "name": "omni", + "base_url": "https://omni.example/v1", + "api_key": "omni-key", + }, + ], + } + if providers_record: + cfg_dict["providers"]["custom"] = copy.deepcopy(_BARE_CUSTOM_PROVIDERS_RECORD) + if model_block: + cfg_dict["model"].update(copy.deepcopy(_BARE_CUSTOM_MODEL_BLOCK)) + return cfg_dict + + +def _clear_ghost_credential_env(monkeypatch): + """Keep the ``CUSTOM__API_KEY`` convention out of these verdicts.""" + monkeypatch.delenv("CUSTOM_GHOST_API_KEY", raising=False) + monkeypatch.delenv("CUSTOM_OMNI_API_KEY", raising=False) + + +_BARE_CUSTOM_SHAPES = [ + (True, False, "providers['custom'] only"), + (False, True, "model.provider: custom only"), + (True, True, "both bare-custom authorities"), +] + + +@pytest.mark.parametrize( + "providers_record,model_block,label", + _BARE_CUSTOM_SHAPES, + ids=["providers-custom", "model-provider-custom", "both"], +) +def test_unknown_named_slug_never_claims_the_bare_custom_authority( + monkeypatch, providers_record, model_block, label +): + """``custom:ghost`` must fail closed even with a generic ``custom`` record present. + + Neither authority names ``ghost``, so neither owns the route. Selecting one + would pair the user's prompt with an endpoint and a credential they never + pointed this slug at. + """ + _clear_ghost_credential_env(monkeypatch) + _with_direct_config( + monkeypatch, _bare_custom_cfg(providers_record=providers_record, model_block=model_block) + ) + + assert config.resolve_custom_provider_connection("custom:ghost") == (None, None), label + + ghost = config.resolve_custom_provider_bundle("custom:ghost") + assert ghost["status"] == config.CUSTOM_SELECTION_MISSING, label + assert ghost["source"] == "", label + assert ghost["record"] is None, f"{label}: a bare-custom record was selected for a named slug" + assert ghost["base_url"] is None, label + assert ghost["api_key"] is None, label + assert ghost["keyless"] is False, f"{label}: an unowned route was reported keyless" + assert ghost["owned"] == {}, f"{label}: an unowned route claimed side fields" + + # The named row is still authoritative for its OWN slug. + owned = config.resolve_custom_provider_bundle("custom:omni") + assert owned["status"] == config.CUSTOM_SELECTION_EXACT, label + assert owned["base_url"] == "https://omni.example/v1", label + + +@pytest.mark.parametrize( + "providers_record,model_block,label", + _BARE_CUSTOM_SHAPES, + ids=["providers-custom", "model-provider-custom", "both"], +) +def test_unknown_named_slug_inherits_no_bare_custom_connection_or_side_fields( + monkeypatch, providers_record, model_block, label +): + """The merge carries nothing from the bare-``custom`` record onto ``ghost``. + + Not the endpoint, not the credential, and — because a bare record's + ``api_mode``/pool/ACP transport are as unowned as its URL — none of the side + fields either. + """ + _clear_ghost_credential_env(monkeypatch) + _with_direct_config( + monkeypatch, _bare_custom_cfg(providers_record=providers_record, model_block=model_block) + ) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:ghost", + "ambient-key-sentinel", + "https://ambient.example/v1", + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + lookup_provider="custom:ghost", + ) + + _assert_ghost_bundle_failed_closed(bundle, label) + assert bundle["base_url"] != _BARE_CUSTOM_PROVIDERS_RECORD["base_url"], label + assert bundle["base_url"] != _BARE_CUSTOM_MODEL_BLOCK["base_url"], label + assert bundle["api_key"] != _BARE_CUSTOM_PROVIDERS_RECORD["api_key"], label + assert bundle["api_key"] != _BARE_CUSTOM_MODEL_BLOCK["api_key"], label + + +def test_unknown_named_slug_connection_view_ignores_the_bare_custom_record(monkeypatch): + """The three-field view reaches the same verdict, and reports ``custom_owned`` False.""" + _clear_ghost_credential_env(monkeypatch) + _with_direct_config(monkeypatch, _bare_custom_cfg()) + + provider, api_key, base_url, custom_owned = ( + config.apply_custom_provider_connection_authority( + "custom:ghost", + "ambient-key-sentinel", + "https://ambient.example/v1", + lookup_provider="custom:ghost", + runtime_provider=copy.deepcopy(_SELF_LABELLED_GHOST_RUNTIME), + ) + ) + + assert base_url is None + assert api_key is None + assert provider == "custom:ghost" + assert custom_owned is False + + +def test_bare_custom_route_still_uses_the_bare_custom_authority(monkeypatch): + """The carve-out: closing the generic path for NAMED slugs, not for bare ``custom``. + + ``providers['custom']`` is the bare route's own identity record, so it stays + eligible there — the tightening above is about a named slug borrowing an + authority that never named it. + """ + _clear_ghost_credential_env(monkeypatch) + _with_direct_config(monkeypatch, _bare_custom_cfg(model_block=False)) + + bare = config.resolve_custom_provider_bundle("custom:custom") + assert bare["status"] == config.CUSTOM_SELECTION_KEYED + assert bare["source"] == "providers" + assert bare["base_url"] == _BARE_CUSTOM_PROVIDERS_RECORD["base_url"] + assert bare["api_key"] == _BARE_CUSTOM_PROVIDERS_RECORD["api_key"] + assert bare["owned"]["api_mode"] == "anthropic_messages" + assert bare["owned"]["credential_pool"] == ["bare-pool-sentinel"] + assert bare["owned"]["acp_command"] == "bare-acp-sentinel" + + +def test_model_block_naming_the_slug_outright_still_owns_it(monkeypatch): + """Only the GENERIC ``custom`` spelling is refused, not an explicit slug. + + A ``model:`` block whose provider is ``custom:ghost`` names this identity, so + it remains the route's authority. + """ + _clear_ghost_credential_env(monkeypatch) + cfg_dict = _bare_custom_cfg(model_block=False) + cfg_dict["model"].update( + { + "provider": "custom:ghost", + "base_url": "https://named-model-sentinel.example/v1", + "api_key": "named-model-key-sentinel", + } + ) + _with_direct_config(monkeypatch, cfg_dict) + + ghost = config.resolve_custom_provider_bundle("custom:ghost") + assert ghost["status"] == config.CUSTOM_SELECTION_KEYED + assert ghost["source"] == "model" + assert ghost["base_url"] == "https://named-model-sentinel.example/v1" + assert ghost["api_key"] == "named-model-key-sentinel" + # And it beats the bare-custom record that is still sitting in providers:. + assert ghost["base_url"] != _BARE_CUSTOM_PROVIDERS_RECORD["base_url"] + + +# ───────────────────────────────────────────────────────────────────────────── +# An identity-keyed record owning ONLY side fields or a dynamic credential +# +# ``providers['custom:']`` names this provider outright, so whatever it +# declares, it owns. Judging it by "did a STATIC api_key resolve, or is there a +# base_url?" classified a perfectly valid ``key_cmd``-only / ``api_mode``-only / +# pool-only / ACP-only record as ``missing`` — and the merge then CLEARED the +# very fields that record exists to supply, reverting them to the ambient +# runtime's. +# ───────────────────────────────────────────────────────────────────────────── + + +def _keyed_side_field_cfg(record): + """Config whose only authority for ``omni`` is the keyed record ``record``.""" + return { + "model": {"default": "active/model", "provider": "custom:active"}, + "providers": {"custom:omni": copy.deepcopy(record)}, + "custom_providers": [ + { + "name": "active", + "base_url": "https://active.example/v1", + "api_key": "active-key", + }, + ], + } + + +_KEYED_SIDE_FIELD_ONLY_RECORDS = [ + ( + {"api_mode": "anthropic_messages"}, + {"api_mode": "anthropic_messages"}, + True, + "api_mode only", + ), + ( + {"transport": "anthropic"}, + {"api_mode": "anthropic_messages"}, + True, + "transport alias only", + ), + ( + {"credential_pool": ["keyed-pool-sentinel"]}, + {"credential_pool": ["keyed-pool-sentinel"]}, + # A configured pool is a declared credential source, so NOT keyless. + False, + "credential_pool only", + ), + ( + {"acp_command": "keyed-acp-sentinel", "acp_args": ["--keyed-arg-sentinel"]}, + {"acp_command": "keyed-acp-sentinel", "acp_args": ["--keyed-arg-sentinel"]}, + True, + "ACP transport only", + ), +] + + +@pytest.mark.parametrize( + "record,expected_owned,expected_keyless,label", + _KEYED_SIDE_FIELD_ONLY_RECORDS, + ids=["api-mode", "transport-alias", "credential-pool", "acp"], +) +def test_keyed_record_owning_only_side_fields_is_not_missing( + monkeypatch, record, expected_owned, expected_keyless, label +): + """A keyed record with no static URL/key still OWNS what it declares.""" + _clear_credential_env(monkeypatch) + _with_direct_config(monkeypatch, _keyed_side_field_cfg(record)) + + bundle = config.resolve_custom_provider_bundle("custom:omni") + assert bundle["status"] == config.CUSTOM_SELECTION_KEYED, ( + f"{label}: a keyed record that owns side fields was classified as missing" + ) + assert bundle["source"] == "providers", label + assert bundle["record"] is not None, label + assert bundle["owned"] == expected_owned, label + assert bundle["is_exact"] is False, label + # It declares no endpoint of its own; that is unowned, not invalid. + assert bundle["base_url"] is None, label + assert bundle["keyless"] is expected_keyless, label + + +def test_keyed_record_owning_only_key_cmd_is_not_missing(monkeypatch): + """``key_cmd`` is a real credential source, so a ``key_cmd``-only record owns the route. + + It resolves no STATIC key by design — the command mints a short-lived bearer + per request — so a static-key test reported the record as missing and the + route fell back to the ambient authority. + """ + _clear_credential_env(monkeypatch) + + def _token_provider(): + return "minted-bearer-sentinel" + + _fake_command_token_source(monkeypatch, lambda _key_cmd, _name: _token_provider) + _with_direct_config(monkeypatch, _keyed_side_field_cfg({"key_cmd": "print-omni-bearer"})) + + bundle = config.resolve_custom_provider_bundle("custom:omni") + assert bundle["status"] == config.CUSTOM_SELECTION_KEYED + assert bundle["source"] == "providers" + assert bundle["api_key"] is _token_provider, "the keyed record's key_cmd token source was lost" + assert bundle["keyless"] is False, "a declared key_cmd must never be reported keyless" + + +def test_keyed_record_owning_only_unresolved_env_credential_is_not_missing(monkeypatch): + """A declared-but-unset ``key_env`` still names this slug's credential source. + + Treating it as missing sends the route to the ambient endpoint instead of + surfacing the misconfiguration. + """ + _clear_credential_env(monkeypatch) + _with_direct_config(monkeypatch, _keyed_side_field_cfg({"key_env": _MISSING_ENV_VAR})) + + bundle = config.resolve_custom_provider_bundle("custom:omni") + assert bundle["status"] == config.CUSTOM_SELECTION_KEYED + assert bundle["api_key"] is None + assert bundle["keyless"] is False, "a declared-but-unresolved credential is not keyless" + + +_KEYED_SIDE_FIELDS_ONLY_CFG = { + "model": {"default": "active/model", "provider": "custom:active"}, + "providers": { + "custom:omni": { + # No base_url, no api_key: side fields are ALL this record declares. + "api_mode": "anthropic_messages", + "credential_pool": ["keyed-pool-sentinel"], + "acp_command": "keyed-acp-sentinel", + "acp_args": ["--keyed-arg-sentinel"], + }, + }, + "custom_providers": [ + { + "name": "active", + "base_url": "https://active.example/v1", + "api_key": "active-key", + }, + ], +} + + +def test_keyed_record_side_fields_survive_the_merge_without_a_static_pair(monkeypatch): + """The merge applies the record's owned fields instead of clearing them. + + The route itself is terminal — this record declares no endpoint, so it + borrows none — but "fail the route closed" must not degrade into "blank the + whole bundle". The fields the record declares for ITSELF are still its own + statement about itself, and only the ambient authority's values go with the + endpoint that was refused. + """ + _clear_credential_env(monkeypatch) + _with_direct_config(monkeypatch, copy.deepcopy(_KEYED_SIDE_FIELDS_ONLY_CFG)) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + "ambient-key-sentinel", + "https://ambient.example/v1", + _ambient_runtime( + api_mode="chat_completions", + command="ambient-acp-sentinel", + args=["--ambient-arg-sentinel"], + credential_pool=["ambient-pool-sentinel"], + ), + lookup_provider="custom:omni", + ) + + _assert_side_fields( + bundle, + { + "api_mode": "anthropic_messages", + "credential_pool": ["keyed-pool-sentinel"], + "acp_command": "keyed-acp-sentinel", + "acp_args": ["--keyed-arg-sentinel"], + }, + "keyed record owning only side fields", + ) + # The record owns no endpoint, so the route is terminal and keeps neither the + # ambient URL nor a credential to pair with it. + verdict = bundle[config.CUSTOM_ROUTE_ERROR_FIELD] + assert verdict is not None and verdict["reason"] == config.CUSTOM_ROUTE_NO_ENDPOINT + assert bundle["base_url"] is None + assert bundle["api_key"] is None + + +def test_keyed_record_side_fields_survive_the_runtime_bundle(monkeypatch): + """End to end: the production-composed send REFUSES, keeping no ambient pair. + + This record owns side fields and nothing else — no endpoint of its own. It + used to reach the constructor on the ambient provider's URL under the "keyed + fill-only" rule, which is exactly the defect: the record's own authority + decided the wire protocol, pool and transport while the ACTIVE provider + decided where the prompt went. Owning ``api_mode`` is not owning an endpoint. + + So the send stops at the terminal verdict instead, and the merge-level test + above keeps pinning that the record's owned fields still ride on the bundle + it hands back. ``custom:omni``'s own side fields therefore never arrive at a + constructor here, because no constructor runs at all. + """ + _clear_credential_env(monkeypatch) + + captured, apperrors = _run_composed_send_expecting_refusal( + monkeypatch, + copy.deepcopy(_KEYED_SIDE_FIELDS_ONLY_CFG), + _ambient_runtime( + api_mode="chat_completions", + command="ambient-acp-sentinel", + args=["--ambient-arg-sentinel"], + credential_pool=["ambient-pool-sentinel"], + ), + "session-1806-keyed-side-fields-only", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + ) + + payload = apperrors[-1] + assert config.CUSTOM_ROUTE_NO_ENDPOINT in str(payload) or "resolved no endpoint" in ( + payload.get("message", "") + ), payload + blob = str(payload) + str(captured) + for leaked in ( + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + "ambient-pool-sentinel", + "ambient-acp-sentinel", + ): + assert leaked not in blob, ( + f"the ambient provider's {leaked!r} survived a route with no endpoint" + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Non-streaming consumers: the verdict is terminal off the streaming path too +# +# ``api/streaming.py`` is not the only thing that builds an AIAgent from a +# merged bundle. POST /api/chat and the four auxiliary consumers all go through +# ``routes._resolve_agent_connection_bundle()``, which is the single chokepoint +# that must RAISE rather than hand back a bundle with a hole in it: an +# incomplete ``(api_key, base_url)`` pair is not a refusal at the constructor, +# it is the signal to re-resolve a provider through ``_routed_client_kwargs()``. +# These pin the raise, the 400 POST /api/chat turns it into, and the negative +# control that a routable bundle still comes back untouched. +# ───────────────────────────────────────────────────────────────────────────── + + +def _terminal_route_cfg(kind): + """Config producing each of the three terminal verdicts for one slug.""" + if kind == "unowned": + # Exactly one row, named something else: nothing owns ``custom:ghost``. + return copy.deepcopy(_SOLE_UNRELATED_ROW_CFG) + if kind == "no_endpoint": + # The exact row owns the slug and blanks the endpoint, so the keyed + # row's URL must not be reachable by falling through. + return _exact_list_row_cfg(base_url="") + # The exact row declares a credential source that resolves to nothing. + return _exact_list_row_cfg(drop=("api_key",), key_env=_MISSING_ENV_VAR) + + +_TERMINAL_ROUTE_CASES = [ + ("unowned", "custom:ghost", config.CUSTOM_ROUTE_UNOWNED, "is not configured", "Settings"), + ( + "no_endpoint", + "custom:omni", + config.CUSTOM_ROUTE_NO_ENDPOINT, + "resolved no endpoint", + "base_url", + ), + ( + "no_credential", + "custom:omni", + config.CUSTOM_ROUTE_NO_CREDENTIAL, + "produced no API key", + "key_env", + ), +] + + +@pytest.mark.parametrize( + "kind,slug,expected_reason,message_fragment,hint_fragment", _TERMINAL_ROUTE_CASES +) +def test_agent_connection_bundle_raises_on_terminal_route( + monkeypatch, kind, slug, expected_reason, message_fragment, hint_fragment +): + """The non-streaming chokepoint refuses instead of returning a holed bundle. + + Returning ``{"base_url": None, ...}`` here would look terminal to the caller + and be the opposite at the constructor, so all five consumers that share + this helper would each have had to remember to check. Raising makes the + refusal structural — and the exception subclasses ``ValueError``, which is + what the existing handlers at those call sites already catch. + """ + import api.routes as routes + + _clear_credential_env(monkeypatch) + _with_direct_config(monkeypatch, _terminal_route_cfg(kind)) + + # Mirror POST /api/chat's own sequence: resolve the model's provider, then + # let the ambient runtime fill the endpoint/credential it did not resolve. + _model, provider, base_url = config.resolve_model_provider( + f"@{slug}:antigravity/gemini-3.7-flash-tiered" + ) + runtime = copy.deepcopy(_ROUTE_KEYED_RUNTIME) + api_key = runtime["api_key"] + if not base_url: + base_url = runtime["base_url"] + + with pytest.raises(config.CustomProviderRouteError) as excinfo: + routes._resolve_agent_connection_bundle(provider, api_key, base_url, runtime) + + err = excinfo.value + assert isinstance(err, ValueError), ( + "the existing except-ValueError handlers at the five call sites must keep catching it" + ) + assert err.reason == expected_reason + assert err.provider == slug + assert message_fragment in err.message, err.message + assert hint_fragment in err.hint, err.hint + + +def test_agent_connection_bundle_returns_routable_bundle_unchanged(monkeypatch): + """The negative control: a routable named route still comes back, not raised. + + Without this, the three cases above are equally satisfied by a helper that + refuses everything — which would take every working custom provider down. + """ + import api.routes as routes + + _with_direct_config(monkeypatch, copy.deepcopy(_KEYED_VS_LIST_CFG)) + + bundle = routes._resolve_agent_connection_bundle( + "custom:omni", + "keyed-key-sentinel-abc", + "https://keyed-url-sentinel.example/v1", + copy.deepcopy(_ROUTE_KEYED_RUNTIME), + ) + + assert bundle["base_url"] == _LIST_ROW_URL + assert bundle["api_key"] == _LIST_ROW_KEY + assert bundle[config.CUSTOM_ROUTE_ERROR_FIELD] is None + + +def _drive_sync_chat_route_expecting_refusal(monkeypatch, cfg_dict, slug): + """Drive POST /api/chat to its refusal; return ``(payload, status, captured)``. + + The session carries the explicit ``@custom::`` form the picker + emits, which ``model_with_provider_context()`` passes through untouched. + That matters for the unowned case: a BARE model plus a stale + ``model_provider`` never mints an ``@custom:ghost:`` route in the first + place (the #7356 guard drops the hint), so the explicit form is the shape in + which an unowned slug actually reaches this route. + """ + import api.routes as routes + + captured, fake_session = _setup_route_consumer_runtime( + monkeypatch, + session_messages=_four_route_messages(), + cfg_dict=cfg_dict, + runtime_dict=copy.deepcopy(_ROUTE_KEYED_RUNTIME), + ) + fake_session.model = f"@{slug}:antigravity/gemini-3.7-flash-tiered" + fake_session.model_provider = slug + + responses = [] + monkeypatch.setattr(routes, "get_session", lambda _sid: fake_session) + monkeypatch.setattr(routes, "_agent_runtime_barrier_response", lambda **_k: None) + monkeypatch.setattr(routes, "resolve_trusted_workspace", lambda ws: "/tmp") + monkeypatch.setattr( + routes, "_get_session_agent_lock", lambda _sid: contextlib.nullcontext() + ) + monkeypatch.setattr( + routes, "_read_profile_model_config", lambda *_a, **_k: (None, None, None) + ) + monkeypatch.setattr( + routes, + "_resolve_compatible_session_model_state", + lambda model, provider, **_k: (model, provider), + ) + # Capture the status alongside the payload -- "which HTTP code" is half the + # contract here (400 user-fixable, not a 500 traceback). + def _capture_j(_handler, payload, **kwargs): + responses.append((payload, kwargs.get("status", 200))) + return payload + + monkeypatch.setattr(routes, "j", _capture_j) + monkeypatch.setattr(routes, "bad", lambda _handler, message, *_a, **_k: {"error": message}) + + routes._handle_chat_sync( + object(), {"session_id": "session-1806-route", "message": "hi"} + ) + + assert responses, "POST /api/chat returned no response at all" + payload, status = responses[-1] + return payload, status, captured + + +@pytest.mark.parametrize( + "kind,slug,expected_reason,message_fragment,hint_fragment", _TERMINAL_ROUTE_CASES +) +def test_sync_chat_route_answers_400_on_unroutable_custom_provider( + monkeypatch, kind, slug, expected_reason, message_fragment, hint_fragment +): + """POST /api/chat answers the actionable cause, and never builds the agent. + + 400, not 500: an unroutable ``custom:`` is a user-fixable provider + misconfiguration, exactly like the ambiguous-slug collision. The constructor + must not be reached at all — the capturing double raises on ``__init__``, so + a bundle that got that far would surface as a leaked + ``_RouteAgentCaptured`` rather than a quiet pass. + """ + _clear_credential_env(monkeypatch) + + payload, status, captured = _drive_sync_chat_route_expecting_refusal( + monkeypatch, _terminal_route_cfg(kind), slug + ) + + assert status == 400, payload + assert payload["type"] == "custom_provider_unroutable", payload + assert payload["reason"] == expected_reason, payload + assert message_fragment in payload["error"], payload + assert slug in payload["error"], payload + assert hint_fragment in payload["hint"], payload + + assert "init_kwargs" not in captured, ( + "an agent was constructed for an unroutable route on the non-streaming path" + ) + # The refusal must not hand back the endpoint or credential it declined to + # route through. + assert "keyed-url-sentinel" not in str(payload), payload + assert "keyed-key-sentinel-abc" not in str(payload), payload + + +# ───────────────────────────────────────────────────────────────────────────── +# The standard Hermes v12 config shape: a RAW ``providers:`` record +# +# Everything above this point describes a config that spells a named custom +# provider one of the two ways the WebUI itself writes: a ``custom_providers[]`` +# row, or a ``providers['custom:']`` key. Hermes v12 writes neither. Its +# config is +# +# model: +# provider: custom:omni +# providers: +# omni: +# base_url: https://omni.example/v1 +# key_env: OMNI_GATE_KEY +# +# and the installed Agent routes it fine: ``_match_new_style_provider()`` scans +# the ENABLED entries of the raw ``providers:`` mapping and takes the first whose +# alias set — minted from BOTH the display ``name`` and the config KEY — holds +# the requested identity. The WebUI looked only at the ``custom:``-prefixed +# spellings, so every send on a stock v12 config failed closed as +# ``unowned_custom_provider`` while the CLI on the same config sent happily. +# +# These regressions use ONLY that shape: no ``custom_providers[]`` list entry and +# no ``providers['custom:omni']`` key. :func:`_v12_raw_cfg` asserts the absence, +# so a future edit that quietly re-adds either spelling turns the whole section +# vacuous loudly rather than silently. +# ───────────────────────────────────────────────────────────────────────────── + + +_V12_URL = "https://v12-raw-url-sentinel.example/v1" +_V12_KEY = "v12-raw-key-sentinel" +_V12_KEY_ENV = "HERMES_TEST_1806_V12_OMNI_KEY" + + +def _v12_raw_cfg(record, *, key="omni", model_provider="custom:omni"): + """``providers: {: record}`` and a ``model:`` block — nothing else. + + The ``model:`` block deliberately declares NO connection field of its own. + ``model.provider: custom:omni`` makes it a candidate in + ``_select_custom_provider_record``, and a base_url there would let it own + the route — which would prove nothing about the raw record under test. + """ + cfg_dict = { + "model": { + "default": "antigravity/gemini-3.7-flash-tiered", + "provider": model_provider, + }, + "providers": {key: copy.deepcopy(record)}, + } + assert "custom_providers" not in cfg_dict, "the raw shape carries no list" + assert not any( + str(k).lower().startswith("custom") for k in cfg_dict["providers"] + ), "the raw shape carries no ``custom:``-prefixed key" + assert not (set(cfg_dict["model"]) & {"base_url", "api_key", "key_env"}), ( + "the model: block must declare no connection of its own" + ) + return cfg_dict + + +def _v12_env(monkeypatch): + """Seed the ``key_env``/``api_key_env`` credential and clear the ladder below it.""" + monkeypatch.setenv(_V12_KEY_ENV, _V12_KEY) + # ``CUSTOM__API_KEY`` is the next rung of the record's own credential + # ladder; leaving a stray one set would mask a record whose declared env var + # was never read at all. + monkeypatch.delenv("CUSTOM_OMNI_API_KEY", raising=False) + monkeypatch.delenv("CUSTOM_MY_OMNI_API_KEY", raising=False) + + +# Each variant is ONE raw record that must resolve to the same (URL, key) pair +# through a different pair of field spellings. ``api``/``url`` are the Agent's +# ``_entry_url`` precedence and ``api_key_env`` its ``key_env`` synonym; a config +# hand-written or migrated either way names the same endpoint. +_V12_RECORD_VARIANTS = [ + ({"base_url": _V12_URL, "key_env": _V12_KEY_ENV}, "base_url+key_env"), + ({"api": _V12_URL, "api_key_env": _V12_KEY_ENV}, "api+api_key_env"), + ({"url": _V12_URL, "key_env": _V12_KEY_ENV}, "url+key_env"), + ({"base_url": _V12_URL, "api_key": _V12_KEY}, "base_url+static_api_key"), + # Display name AND config key disagree: the key alone still names the slug. + ( + {"name": "My Omni", "base_url": _V12_URL, "key_env": _V12_KEY_ENV}, + "aliased-display-name", + ), +] + +_V12_VARIANT_IDS = [label for _record, label in _V12_RECORD_VARIANTS] + + +@pytest.mark.parametrize("record,label", _V12_RECORD_VARIANTS, ids=_V12_VARIANT_IDS) +def test_v12_raw_provider_record_owns_its_named_slug(monkeypatch, record, label): + """``providers: {omni: ...}`` is the authority for ``custom:omni``. + + The bug: none of these resolved at all. ``custom:omni`` matched no + ``custom_providers[]`` row and no ``providers['custom:omni']`` key, so the + route was reported unowned and the send refused. + """ + _v12_env(monkeypatch) + _with_direct_config(monkeypatch, _v12_raw_cfg(record)) + + api_key, base_url = config.resolve_custom_provider_connection("custom:omni") + assert base_url == _V12_URL, label + assert api_key == _V12_KEY, label + + bundle = config.resolve_custom_provider_bundle("custom:omni") + assert bundle["status"] == config.CUSTOM_SELECTION_KEYED, label + assert bundle["record"] is not None, label + assert bundle["base_url"] == _V12_URL, label + assert bundle["api_key"] == _V12_KEY, label + # A record that resolved a credential is not keyless, so the route can never + # be handed the placeholder that claims the endpoint needs no auth. + assert bundle["keyless"] is False, label + assert bundle["api_key"] != config.KEYLESS_CUSTOM_API_KEY, label + + +def test_v12_raw_record_is_reached_by_its_display_name_too(monkeypatch): + """One record, keyed ``omni`` and named ``My Omni``, answers to BOTH identities. + + The Agent mints its alias set from the display name and the config key + together, so ``custom:omni`` and ``custom:my-omni`` are the same provider to + it. A WebUI that honoured only one of them would refuse half the sends the + picker can emit for a single configured endpoint. + """ + _v12_env(monkeypatch) + record = {"name": "My Omni", "base_url": _V12_URL, "key_env": _V12_KEY_ENV} + + for slug in ("custom:omni", "custom:my-omni"): + _with_direct_config(monkeypatch, _v12_raw_cfg(record, model_provider=slug)) + assert config.resolve_custom_provider_connection(slug) == (_V12_KEY, _V12_URL), slug + + # …and an identity the record does NOT name is still refused, so the alias + # set widens the vocabulary without widening ownership. + _with_direct_config(monkeypatch, _v12_raw_cfg(record, model_provider="custom:omni")) + assert config.resolve_custom_provider_connection("custom:ghost") == (None, None) + ghost = config.resolve_custom_provider_bundle("custom:ghost") + assert ghost["status"] == config.CUSTOM_SELECTION_MISSING + + +# ── the non-active raw record: its endpoint, not the active provider's ─────── + + +def test_v12_raw_record_supplies_its_own_endpoint_beside_its_own_key(monkeypatch): + """A NON-ACTIVE raw record must not pair its key with the ambient endpoint. + + This is the #1806 split-authority failure mirrored onto the raw shape. The + ambient runtime has resolved the ACTIVE provider, so it seeds a truthy + ``base_url`` of its own. The merge then takes the credential from the raw + record (the runtime is a different authority, so its key is not this + provider's) — and, filling only what the runtime had left empty, kept the + ACTIVE provider's URL. The send went to the active endpoint carrying the + non-active provider's key: a guaranteed 401 at best, and the user's prompt + delivered to a provider they did not pick at worst. + + The record declares the endpoint, so the record's endpoint wins. + """ + _v12_env(monkeypatch) + _with_direct_config( + monkeypatch, _v12_raw_cfg({"base_url": _V12_URL, "key_env": _V12_KEY_ENV}) + ) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + copy.deepcopy(_AMBIENT_SIDE_FIELD_RUNTIME), + lookup_provider="custom:omni", + ) + + assert bundle["base_url"] == _V12_URL + assert bundle["api_key"] == _V12_KEY + assert bundle["base_url"] != _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], ( + "the record's key was paired with the ambient provider's endpoint" + ) + assert bundle["api_key"] != _AMBIENT_SIDE_FIELD_RUNTIME["api_key"] + assert bundle[config.CUSTOM_ROUTE_ERROR_FIELD] is None + # The runtime is a proven-foreign authority, so none of its side fields ride + # along to the custom HTTP endpoint either. + _assert_side_fields(bundle, _FOREIGN_AMBIENT_SIDE_FIELDS, "non-active raw record") + + +def test_v12_raw_record_keeps_the_runtimes_spelling_of_the_same_endpoint(monkeypatch): + """Negative control: same authority, so the runtime's normalized URL stands. + + Without this, the test above is equally satisfied by "always overwrite the + runtime's endpoint with the config spelling" — which would discard the + normalized form the runtime actually resolved (trailing-``/v1`` + de-duplication and friends) for an endpoint that is provably the same one. + """ + _v12_env(monkeypatch) + _with_direct_config( + monkeypatch, _v12_raw_cfg({"base_url": _V12_URL + "/", "key_env": _V12_KEY_ENV}) + ) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + None, + _V12_URL, + {"provider": "custom:omni", "base_url": _V12_URL, "api_key": _V12_KEY}, + lookup_provider="custom:omni", + ) + + assert bundle["base_url"] == _V12_URL, "a same-authority runtime spelling was discarded" + assert bundle["api_key"] == _V12_KEY + assert bundle[config.CUSTOM_ROUTE_ERROR_FIELD] is None + + +def test_matching_identity_metadata_with_conflicting_credential_rejects_tie(monkeypatch): + """Even when identity metadata matches, conflicting credentials strictly reject a tie.""" + _v12_env(monkeypatch) + _with_direct_config( + monkeypatch, _v12_raw_cfg({"base_url": _V12_URL + "/", "key_env": _V12_KEY_ENV}) + ) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + None, + _V12_URL, + { + "provider": "custom:omni", + "base_url": _V12_URL, + "record_id": "omni", + "api_key": "conflicting-foreign-key", + }, + lookup_provider="custom:omni", + ) + + assert bundle["base_url"] == _V12_URL + "/" + assert bundle["api_key"] == _V12_KEY + + +_CALLER_RESOLVED_URL = "https://caller-resolved-sentinel.example/v1" + + +def test_v12_raw_record_supplies_the_endpoint_nothing_is_positively_tied_to(monkeypatch): + """With NO runtime dict there is no TIE, so the record's own endpoint wins. + + The endpoint the caller arrived with was resolved before this slug was ever + looked up, so it says nothing about the record selected for it. Keeping it + only because no runtime dict arrived to contradict it reads silence as + provenance — and pairs the record's own credential with whatever endpoint + happened to be in flight, which is the split-authority send this module + exists to stop. + + So the rule is positive: an endpoint survives the merge only when the + SELECTED record supplied it, or when a runtime dict is positively tied to + that same record (:func:`_custom_runtime_endpoint_is_record_owned`, pinned by + the same-authority control above). An absent runtime dict is neither, so the + record's ``base_url`` displaces the caller's. The #2271 fill-only shape lives + on the injected-``connection_resolver`` path, where there IS no record to + take provenance from — see the control below. + """ + _v12_env(monkeypatch) + _with_direct_config( + monkeypatch, _v12_raw_cfg({"base_url": _V12_URL, "key_env": _V12_KEY_ENV}) + ) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + None, + _CALLER_RESOLVED_URL, + None, + lookup_provider="custom:omni", + ) + + assert bundle["base_url"] == _V12_URL, ( + "the record's credential was left beside an endpoint it never declared" + ) + assert bundle["base_url"] != _CALLER_RESOLVED_URL + assert bundle["api_key"] == _V12_KEY + assert bundle[config.CUSTOM_ROUTE_ERROR_FIELD] is None + + +def test_injected_connection_resolver_still_keeps_the_callers_base_url(monkeypatch): + """Negative control for #2271: no record, so the fill-only shape is unchanged. + + An injected ``connection_resolver`` reports a bare (key, URL) pair and IS the + whole authority on that path — there is no config record whose endpoint + provenance could displace anything. Applying the record rule here would + silently re-point every such caller at the resolver's spelling, which is + #2271's ``test_named_custom_provider_keeps_existing_runtime_base_url``. + """ + _v12_env(monkeypatch) + _with_direct_config( + monkeypatch, _v12_raw_cfg({"base_url": _V12_URL, "key_env": _V12_KEY_ENV}) + ) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + None, + _CALLER_RESOLVED_URL, + None, + lookup_provider="custom:omni", + connection_resolver=lambda _pid, **_kw: ( + "resolver-key-sentinel", + "https://resolver-config-sentinel.example/v1", + ), + ) + + assert bundle["base_url"] == _CALLER_RESOLVED_URL, ( + "the resolver path lost #2271's fill-only base_url handling" + ) + assert bundle["api_key"] == "resolver-key-sentinel" + assert bundle[config.CUSTOM_ROUTE_ERROR_FIELD] is None + + +# ── disabled rows are invisible to the Agent, so they own nothing here ─────── + + +_V12_DISABLED_FLAGS = [False, "false", "no", "off", "0"] + + +@pytest.mark.parametrize("flag", _V12_DISABLED_FLAGS, ids=[str(f) for f in _V12_DISABLED_FLAGS]) +def test_v12_disabled_raw_record_fails_closed_as_unowned(monkeypatch, flag): + """A switched-off row must not quietly keep routing. + + ``is_provider_enabled()`` hides a falsey ``enabled:`` entry from the Agent's + resolver, so honouring it here would make the WebUI send through an endpoint + the user disabled — and send it through the AMBIENT connection the moment the + row itself resolved nothing. Both halves are pinned: the verdict is + ``unowned_custom_provider``, and the bundle keeps no endpoint, credential, + placeholder or side field from anywhere. + """ + _v12_env(monkeypatch) + record = {"enabled": flag, "base_url": _V12_URL, "key_env": _V12_KEY_ENV} + _with_direct_config(monkeypatch, _v12_raw_cfg(record)) + + assert config.resolve_custom_provider_connection("custom:omni") == (None, None) + selection = config.resolve_custom_provider_bundle("custom:omni") + assert selection["status"] == config.CUSTOM_SELECTION_MISSING + assert selection["record"] is None + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + copy.deepcopy(_AMBIENT_SIDE_FIELD_RUNTIME), + lookup_provider="custom:omni", + ) + verdict = bundle[config.CUSTOM_ROUTE_ERROR_FIELD] + assert verdict is not None, "a disabled row left the route looking routable" + assert verdict["reason"] == config.CUSTOM_ROUTE_UNOWNED + assert verdict["provider"] == "custom:omni" + assert bundle["base_url"] is None + assert bundle["api_key"] is None + assert bundle["api_key"] != config.KEYLESS_CUSTOM_API_KEY + assert bundle["provider"] == "custom:omni" + _assert_side_fields(bundle, _FOREIGN_AMBIENT_SIDE_FIELDS, f"enabled: {flag!r}") + # The disabled row's own endpoint and credential must not leak either. + assert _V12_URL not in str(bundle) + assert _V12_KEY not in str(bundle) + + +@pytest.mark.parametrize("flag", [True, "true", "yes", "on", "1"], ids=lambda f: str(f)) +def test_v12_explicitly_enabled_raw_record_still_owns_its_slug(monkeypatch, flag): + """Negative control: only the FALSEY words hide a row. + + A mirror that read ``enabled`` as "present means off", or that treated any + string as truthy-by-presence, would take every explicitly-enabled v12 + provider offline — the same outage as the bug, from the opposite direction. + """ + _v12_env(monkeypatch) + record = {"enabled": flag, "base_url": _V12_URL, "key_env": _V12_KEY_ENV} + _with_direct_config(monkeypatch, _v12_raw_cfg(record)) + + assert config.resolve_custom_provider_connection("custom:omni") == (_V12_KEY, _V12_URL) + + +# ── two raw records claiming one identity split the authority ──────────────── + + +_V12_AMBIGUOUS_CASES = [ + ( + # Two config KEYS whose alias sets overlap: ``Omni`` normalizes onto the + # other record's key. + { + "omni": {"base_url": "https://omni-a.example/v1", "api_key": "omni-a-key"}, + "omni-two": { + "name": "Omni", + "base_url": "https://omni-b.example/v1", + "api_key": "omni-b-key", + }, + }, + "key-vs-display-name", + ), + ( + # Two display NAMES that normalize to the same slug: ``Omni`` and + # ``omni`` are one identity to the Agent's alias minting. + { + "gate-a": { + "name": "Omni", + "base_url": "https://omni-a.example/v1", + "key_env": _V12_KEY_ENV, + }, + "gate-b": { + "name": "omni", + "base_url": "https://omni-b.example/v1", + "api_key": "omni-b-key", + }, + }, + "display-name-vs-display-name", + ), + ( + # The legacy ``custom:``-carrying spelling of a display name is the same + # identity as the bare config key. + { + "omni": {"base_url": "https://omni-a.example/v1", "api_key": "omni-a-key"}, + "gate-b": { + "name": "custom:omni", + "base_url": "https://omni-b.example/v1", + "api_key": "omni-b-key", + }, + }, + "bare-key-vs-custom-prefixed-name", + ), +] + + +@pytest.mark.parametrize( + "providers_map,label", + [(m, lbl) for m, lbl in _V12_AMBIGUOUS_CASES], + ids=[lbl for _m, lbl in _V12_AMBIGUOUS_CASES], +) +def test_v12_ambiguous_raw_records_fail_closed(monkeypatch, providers_map, label): + """Two raw records naming one slug is a refusal, not a first-match race. + + Config order would decide which endpoint and which credential the send gets, + and nothing forces those two to come from the SAME record — the split + authority ``_unique_custom_provider_entry`` already refuses for + ``custom_providers[]``. Every entry point into the route must raise, because + each of them is the first thing some consumer calls. + """ + # The first case's ``omni`` key is ambiguous only because the OTHER record + # exists, so build the config by hand rather than via ``_v12_raw_cfg``. + cfg_dict = { + "model": {"default": "antigravity/gemini-3.7-flash-tiered", "provider": "custom:omni"}, + "providers": copy.deepcopy(providers_map), + } + _v12_env(monkeypatch) + _with_direct_config(monkeypatch, cfg_dict) + + import api.routes as routes + + entry_points = { + "resolve_custom_provider_connection": lambda: config.resolve_custom_provider_connection( + "custom:omni" + ), + "resolve_custom_provider_bundle": lambda: config.resolve_custom_provider_bundle( + "custom:omni" + ), + "merge_custom_provider_runtime_bundle": lambda: ( + config.merge_custom_provider_runtime_bundle( + "custom:omni", + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + copy.deepcopy(_AMBIENT_SIDE_FIELD_RUNTIME), + lookup_provider="custom:omni", + ) + ), + "_resolve_agent_connection_bundle": lambda: routes._resolve_agent_connection_bundle( + "custom:omni", + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + copy.deepcopy(_AMBIENT_SIDE_FIELD_RUNTIME), + ), + } + + for name, call in entry_points.items(): + with pytest.raises(config.AmbiguousCustomProviderError) as excinfo: + call() + # The refusal has to be actionable: it must name the colliding records + # and subclass ValueError, which is what the existing handlers at the + # consumer call sites already catch. + assert isinstance(excinfo.value, ValueError), f"{label}/{name}" + message = str(excinfo.value) + for key in providers_map: + assert key in message, f"{label}/{name}: {message}" + + # An identity only ONE of them names is unaffected: a collision on ``omni`` + # must not take an unrelated provider down with it. + unique_cfg = { + "model": cfg_dict["model"], + "providers": { + "solo": {"base_url": _V12_URL, "key_env": _V12_KEY_ENV}, + **copy.deepcopy(providers_map), + }, + } + _with_direct_config(monkeypatch, unique_cfg) + assert config.resolve_custom_provider_connection("custom:solo") == (_V12_KEY, _V12_URL) + + +def test_v12_record_naming_the_slug_without_a_connection_is_not_a_collision(monkeypatch): + """A record that declares nothing is not a competing authority. + + Only a record that could actually supply an endpoint or a credential can + split the authority, so a same-slug row holding e.g. a models allowlist and + nothing else must neither win the route nor block it. Refusing here would + fail closed on configs that are not ambiguous at all. + """ + _v12_env(monkeypatch) + cfg_dict = { + "model": {"default": "antigravity/gemini-3.7-flash-tiered", "provider": "custom:omni"}, + "providers": { + "omni": {"base_url": _V12_URL, "key_env": _V12_KEY_ENV}, + "omni-notes": {"name": "Omni", "models": ["antigravity/gemini-3.7-flash-tiered"]}, + }, + } + _with_direct_config(monkeypatch, cfg_dict) + + assert config.resolve_custom_provider_connection("custom:omni") == (_V12_KEY, _V12_URL) + + +# ── the three streaming regions that build an agent ────────────────────────── + + +def _assert_v12_bundle(init_kwargs, label, *, expected_key=_V12_KEY): + """The constructor bundle is wholly owned by the raw v12 record.""" + assert init_kwargs["base_url"] == _V12_URL, label + assert init_kwargs["api_key"] == expected_key, label + # ``custom``, not ``custom:omni``: the named provider has supplied a concrete + # endpoint, so Agent init must not synthesize ``CUSTOM:OMNI_API_KEY`` hints. + assert init_kwargs["provider"] == "custom", label + assert init_kwargs["base_url"] != _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], label + assert init_kwargs["api_key"] != _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], label + assert init_kwargs["api_key"] != config.KEYLESS_CUSTOM_API_KEY, label + _assert_side_fields(init_kwargs, _FOREIGN_AMBIENT_SIDE_FIELDS, label) + + +def _assert_explicit_client_only(captured, label): + """``_init_openai_client()`` took the explicit branch on EVERY construction. + + The weaker ``base_url is not None`` proxy is satisfied by a bundle that still + sends the turn somewhere else: an incomplete pair makes Agent init call + ``_routed_client_kwargs()`` and re-resolve a provider through the centralized + router. Asserting the branch is what pins "this send reached the endpoint the + user picked". + """ + routed = captured.get("routed_client_kwargs_calls", []) + assert not routed, f"{label}: _routed_client_kwargs() was reached: {routed}" + explicit = captured.get("explicit_client_kwargs_calls", []) + assert explicit, f"{label}: no client was configured at all" + for call in explicit: + assert call["base_url"] == _V12_URL, label + assert call["api_key"] == _V12_KEY, label + + +@pytest.mark.parametrize("record,label", _V12_RECORD_VARIANTS, ids=_V12_VARIANT_IDS) +def test_v12_raw_record_initial_streaming_send_builds_the_agent(monkeypatch, record, label): + """Initial send: the stock v12 config routes, and routes to its OWN endpoint. + + Before the fix this send did not merely mis-route — it never happened. The + route was refused as ``unowned_custom_provider`` on a config the CLI sends on + every day. + """ + import api.streaming as streaming + + _v12_env(monkeypatch) + session_id = f"session-1806-v12-initial-{label}" + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, + _v12_raw_cfg(record), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + session_id=session_id, + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id=session_id, + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + apperrors = _drain_apperrors(q) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + assert not apperrors, f"{label}: the send failed closed: {apperrors}" + assert captured.get("run_calls"), f"{label}: the turn was never sent" + _assert_v12_bundle(captured["init_kwargs"], f"{label}/initial send") + _assert_explicit_client_only(captured, f"{label}/initial send") + + +@pytest.mark.parametrize( + "fail_first,label", + [("returned_error", "returned-error heal"), ("raised", "raised-exception heal")], + ids=["returned-error", "raised-exception"], +) +def test_v12_raw_record_survives_both_credential_heal_paths(monkeypatch, fail_first, label): + """Both 401 self-heal retries rebuild the SAME raw-record bundle. + + The retry paths re-resolve the runtime provider from scratch, so they are + where a partial rebuild leaks the ambient authority back in: the heal hands + back the ambient dict, and a retry that refreshed only provider/key/base_url + would reconstruct the agent with the ambient pool, wire protocol and ACP + transport beside the custom endpoint. Every construction in the history is + checked, not just the last. + """ + import api.streaming as streaming + + _v12_env(monkeypatch) + session_id = f"session-1806-v12-{fail_first}" + stream_id, q, captured, restore = _setup_production_composed_runtime( + monkeypatch, + _v12_raw_cfg({"base_url": _V12_URL, "key_env": _V12_KEY_ENV}), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + session_id=session_id, + fail_first=fail_first, + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id=session_id, + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + history = captured["init_kwargs_history"] + assert len(history) >= 2, f"{label}: the retry constructed no second agent" + for index, init_kwargs in enumerate(history): + _assert_v12_bundle(init_kwargs, f"{label} construction #{index}") + _assert_explicit_client_only(captured, label) + + +def test_v12_disabled_record_refuses_the_streaming_send(monkeypatch): + """A disabled v12 row stops the turn instead of routing it anywhere. + + The complement of the two tests above at the same boundary: no agent is + constructed, so ``_routed_client_kwargs()`` is never reached, the agent cache + is never poisoned, and the turn ends on a controlled ``provider_unroutable`` + apperror naming the provider and a fix. + """ + _v12_env(monkeypatch) + captured, apperrors = _run_composed_send_expecting_refusal( + monkeypatch, + _v12_raw_cfg({"enabled": False, "base_url": _V12_URL, "key_env": _V12_KEY_ENV}), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + "session-1806-v12-disabled", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + ) + + payload = apperrors[-1] + assert "custom:omni" in payload["message"], payload + assert "is not configured" in payload["message"], payload + + blob = str(payload) + str(captured) + for leaked in ( + _V12_URL, + _V12_KEY, + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + config.KEYLESS_CUSTOM_API_KEY, + ): + assert leaked not in blob, f"{leaked!r} leaked into the refused route" + assert not captured.get("explicit_client_kwargs_calls") + + +# ── the shared non-streaming constructor boundary ──────────────────────────── + + +@pytest.mark.parametrize("record,label", _V12_RECORD_VARIANTS, ids=_V12_VARIANT_IDS) +def test_v12_raw_record_resolves_at_the_non_streaming_chokepoint(monkeypatch, record, label): + """``_resolve_agent_connection_bundle()`` returns the record's bundle, unraised. + + POST /api/chat and the four auxiliary consumers all build their agent from + here, so a v12 config that only worked on the streaming path would still take + compression, commit messages, handoffs and summaries down. The helper RAISES + on a terminal verdict, so a clean return is itself half the assertion. + """ + import api.routes as routes + + _v12_env(monkeypatch) + _with_direct_config(monkeypatch, _v12_raw_cfg(record)) + + # Mirror POST /api/chat's own sequence: resolve the model's provider, then + # let the ambient runtime fill the endpoint it did not resolve. + _model, provider, base_url = config.resolve_model_provider( + "@custom:omni:antigravity/gemini-3.7-flash-tiered" + ) + assert provider == "custom:omni", label + runtime = copy.deepcopy(_AMBIENT_SIDE_FIELD_RUNTIME) + bundle = routes._resolve_agent_connection_bundle( + provider, + runtime["api_key"], + base_url or runtime["base_url"], + runtime, + ) + + assert bundle[config.CUSTOM_ROUTE_ERROR_FIELD] is None, label + _assert_v12_bundle(bundle, f"{label}/non-streaming chokepoint") + + +def test_v12_disabled_record_raises_at_the_non_streaming_chokepoint(monkeypatch): + """The same boundary refuses a disabled row, with the actionable reason.""" + import api.routes as routes + + _v12_env(monkeypatch) + _with_direct_config( + monkeypatch, + _v12_raw_cfg({"enabled": "off", "base_url": _V12_URL, "key_env": _V12_KEY_ENV}), + ) + + with pytest.raises(config.CustomProviderRouteError) as excinfo: + routes._resolve_agent_connection_bundle( + "custom:omni", + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + copy.deepcopy(_AMBIENT_SIDE_FIELD_RUNTIME), + ) + + err = excinfo.value + assert isinstance(err, ValueError) + assert err.reason == config.CUSTOM_ROUTE_UNOWNED + assert err.provider == "custom:omni" + assert "is not configured" in err.message, err.message + assert err.hint, "the refusal named no fix" + + +# ── the agent cache signature is derived from the resolved bundle ──────────── + + +def _v12_send_signature(monkeypatch, cfg_dict, runtime_dict, session_id): + """Drive one composed send and return the signature its agent was cached under.""" + import api.streaming as streaming + + stream_id, q, _captured, restore = _setup_production_composed_runtime( + monkeypatch, cfg_dict, runtime_dict, session_id=session_id + ) + try: + streaming.STREAMS[stream_id] = q + streaming._run_agent_streaming( + session_id=session_id, + msg_text="hello", + model="@custom:omni:antigravity/gemini-3.7-flash-tiered", + workspace="/tmp", + stream_id=stream_id, + ) + with config.SESSION_AGENT_CACHE_LOCK: + assert session_id in config.SESSION_AGENT_CACHE, "the send cached no agent" + return config.SESSION_AGENT_CACHE[session_id][1] + finally: + streaming.STREAMS.pop(stream_id, None) + streaming.AGENT_INSTANCES.pop(stream_id, None) + restore() + + +def test_v12_cache_signature_is_bundle_derived_not_runtime_derived(monkeypatch): + """Two sends whose RESOLVED bundle is identical hash identically. + + The raw record clears every ambient side field, so a signature still taken + off the raw runtime provider would differ between these two sends even though + the agents they build are the same — churning a new agent per send. + """ + _v12_env(monkeypatch) + record = {"base_url": _V12_URL, "key_env": _V12_KEY_ENV} + + sig_plain = _v12_send_signature( + monkeypatch, + _v12_raw_cfg(record), + {"provider": "custom:omni", "base_url": "https://ambient.example/v1", "api_key": "amb"}, + "session-1806-v12-sig-plain", + ) + sig_ambient = _v12_send_signature( + monkeypatch, + _v12_raw_cfg(record), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + "session-1806-v12-sig-ambient", + ) + + assert sig_plain == sig_ambient, ( + "the signature still varies with runtime fields the bundle cleared" + ) + + +_V12_SIGNATURE_MUTATIONS = [ + ({"base_url": _V12_URL + "-moved", "key_env": _V12_KEY_ENV}, "endpoint"), + ({"base_url": _V12_URL, "api_key": _V12_KEY + "-rotated"}, "credential"), + ( + {"base_url": _V12_URL, "key_env": _V12_KEY_ENV, "api_mode": "anthropic_messages"}, + "api_mode", + ), + ( + { + "base_url": _V12_URL, + "key_env": _V12_KEY_ENV, + "command": "v12-acp-command-sentinel", + "args": ["--v12-acp-arg"], + }, + "acp_transport", + ), +] + + +@pytest.mark.parametrize( + "record,label", + _V12_SIGNATURE_MUTATIONS, + ids=[label for _r, label in _V12_SIGNATURE_MUTATIONS], +) +def test_v12_cache_signature_tracks_the_provider_configuration(monkeypatch, record, label): + """Editing the raw record mints a NEW agent instead of reusing the cached one. + + The cache is keyed by session, so a signature blind to any part of the + provider configuration would keep serving an agent built on the PREVIOUS + endpoint, credential, wire protocol or ACP transport for the rest of the + session — the user changes a setting and nothing happens. + """ + _v12_env(monkeypatch) + baseline = {"base_url": _V12_URL, "key_env": _V12_KEY_ENV} + + sig_before = _v12_send_signature( + monkeypatch, + _v12_raw_cfg(baseline), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + "session-1806-v12-sig-baseline", + ) + sig_after = _v12_send_signature( + monkeypatch, + _v12_raw_cfg(record), + dict(_AMBIENT_SIDE_FIELD_RUNTIME), + "session-1806-v12-sig-mutated", + ) + + assert sig_before != sig_after, ( + f"the signature ignores the record's {label}, so a cached agent built on " + f"the previous one would be reused for the rest of the session" + ) + + +# ── the WebUI and the installed Agent must select the SAME bundle ──────────── +# +# ``hermes_cli`` is not importable here, so the oracle below is written from the +# Agent's documented algorithm rather than by calling it — and deliberately NOT +# by calling ``api.config``'s mirrors of it, which would make the comparison a +# tautology. Two independent implementations agreeing on the same config is the +# only evidence available that a v12 send lands in the same place whichever of +# the two resolves it. That agreement is the entire point of the fix: the bug was +# precisely the WebUI and the Agent disagreeing about a stock v12 config. + + +def _oracle_agent_aliases(display_name, provider_key): + """``hermes_cli.providers.custom_provider_aliases()``, reimplemented.""" + aliases = set() + for value in (display_name, provider_key): + raw = str(value or "").strip().lower() + if not raw: + continue + dashed = raw.replace(" ", "-") + aliases.add(raw) + aliases.add(dashed) + aliases.add(dashed if dashed.startswith("custom:") else "custom:" + dashed) + if dashed.startswith("custom:"): + bare = dashed.split(":", 1)[1] + if bare: + aliases.add(bare) + aliases.add("custom:" + dashed) + aliases.discard("") + return aliases + + +def _oracle_agent_enabled(record): + """``hermes_cli.config_providers.is_provider_enabled()``, reimplemented.""" + flag = record.get("enabled", True) + if isinstance(flag, bool): + return flag + if isinstance(flag, str): + return flag.strip().lower() not in {"false", "0", "no", "off"} + return bool(flag) + + +def _oracle_agent_selection(cfg_dict, requested): + """``_match_new_style_provider()``: first ENABLED raw record whose aliases match. + + Returns the complete connection bundle the Agent would construct with, or + ``None`` when nothing in ``providers:`` names ``requested``. + """ + wanted = str(requested or "").strip().lower() + providers_cfg = cfg_dict.get("providers") or {} + for provider_key, record in providers_cfg.items(): + if not isinstance(record, dict) or not record: + continue + if not _oracle_agent_enabled(record): + continue + if wanted not in _oracle_agent_aliases(record.get("name") or provider_key, provider_key): + continue + url = None + for field in ("api", "url", "base_url"): # _entry_url precedence + value = record.get(field) + if isinstance(value, str) and value.strip(): + url = value.strip() + break + api_key = record.get("api_key") + if not api_key: + env_name = record.get("key_env") or record.get("api_key_env") + if env_name: + api_key = os.environ.get(str(env_name)) + return { + # The Agent routes a resolved named endpoint through the generic + # OpenAI-compatible custom client, exactly as the WebUI bundle does. + "provider": "custom", + "base_url": url, + "api_key": api_key or None, + "api_mode": record.get("api_mode") or record.get("transport") or None, + "acp_command": record.get("acp_command") or record.get("command") or None, + "acp_args": record.get("acp_args") or record.get("args") or None, + } + return None + + +_V12_ORACLE_RECORDS = [ + ({"base_url": _V12_URL, "key_env": _V12_KEY_ENV}, "base_url+key_env"), + ({"api": _V12_URL, "api_key_env": _V12_KEY_ENV}, "api+api_key_env"), + ({"url": _V12_URL, "api_key": _V12_KEY}, "url+static_api_key"), + ({"name": "My Omni", "base_url": _V12_URL, "key_env": _V12_KEY_ENV}, "aliased-name"), + ( + { + "base_url": _V12_URL, + "key_env": _V12_KEY_ENV, + "api_mode": "anthropic_messages", + "command": "v12-acp-command-sentinel", + "args": ["--v12-acp-arg"], + }, + "side-fields-owned", + ), + ({"enabled": False, "base_url": _V12_URL, "key_env": _V12_KEY_ENV}, "disabled"), +] + + +@pytest.mark.parametrize( + "record,label", + _V12_ORACLE_RECORDS, + ids=[label for _r, label in _V12_ORACLE_RECORDS], +) +def test_webui_and_agent_select_the_same_complete_bundle(monkeypatch, record, label): + """One config, two resolvers, one connection bundle — including the side fields. + + Not just the URL and the key: ``api_mode`` decides the wire protocol and + ``acp_command``/``acp_args`` the transport, so a WebUI that agreed on the + endpoint and disagreed on those would still talk to it differently than the + CLI does. The disabled case is in the same table on purpose — agreeing that + a row owns NOTHING is as much a part of the contract as agreeing where it + points. + """ + _v12_env(monkeypatch) + cfg_dict = _v12_raw_cfg(record) + _with_direct_config(monkeypatch, cfg_dict) + + expected = _oracle_agent_selection(cfg_dict, "custom:omni") + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + copy.deepcopy(_AMBIENT_SIDE_FIELD_RUNTIME), + lookup_provider="custom:omni", + ) + fields = ("provider", "base_url", "api_key", "api_mode", "acp_command", "acp_args") + + if expected is None: + # The Agent routes this nowhere, so neither may the WebUI — and it must + # say so terminally rather than hand back a holed bundle. + verdict = bundle[config.CUSTOM_ROUTE_ERROR_FIELD] + assert verdict is not None, label + assert verdict["reason"] == config.CUSTOM_ROUTE_UNOWNED, label + assert bundle["base_url"] is None and bundle["api_key"] is None, label + return + + assert bundle[config.CUSTOM_ROUTE_ERROR_FIELD] is None, label + actual = {field: bundle[field] for field in fields} + assert actual == expected, ( + f"{label}: the WebUI and the Agent would reach this provider differently" + ) + # ``credential_pool`` has no counterpart in the raw record, so the ambient + # provider's must not ride along with an endpoint it does not belong to. + assert bundle["credential_pool"] is None, label + + +def test_the_agent_oracle_disagrees_with_the_prefixed_only_reader(monkeypatch): + """Guard the oracle: it must actually resolve what the old reader could not. + + If ``_oracle_agent_selection`` quietly returned ``None`` for the raw shape, + every comparison above would collapse into the unowned branch and prove + nothing. Pin that it resolves the stock v12 record — and that it is reading + the RAW key, not a ``custom:``-prefixed one, by checking it also refuses a + record that names a different identity. + """ + _v12_env(monkeypatch) + cfg_dict = _v12_raw_cfg({"base_url": _V12_URL, "key_env": _V12_KEY_ENV}) + + selection = _oracle_agent_selection(cfg_dict, "custom:omni") + assert selection is not None, "the oracle cannot resolve the stock v12 shape" + assert selection["base_url"] == _V12_URL + assert selection["api_key"] == _V12_KEY + + assert _oracle_agent_selection(cfg_dict, "custom:ghost") is None + + +# ───────────────────────────────────────────────────────────────────────────── +# Reviewer probe: a DISABLED exact-keyed record must not route either +# +# ``providers['custom:']`` is the one candidate that names the slug by its +# own exact config key, and it was the one rung of the ladder that never ran the +# ``enabled`` check. A switched-off keyed record therefore kept handing out a +# complete, routable bundle — its endpoint AND its secret — while every other +# shape of the same disable failed closed. +# ───────────────────────────────────────────────────────────────────────────── + + +_EXACT_KEYED_URL = "https://exact-keyed-disabled-sentinel.example/v1" +_EXACT_KEYED_KEY = "exact-keyed-disabled-key-sentinel" + + +def _exact_keyed_disabled_cfg(flag): + """``providers['custom:omni']`` switched off, beside a live ambient provider. + + The generic ``custom`` record and the ``model:`` block are the less-specific + candidates the disabled key must NOT fall through to: honouring the disable + by routing the turn through the ACTIVE provider's endpoint and credential is + the same leak from the other direction. + """ + return { + "model": {"default": "active/model", "provider": "custom:active"}, + "providers": { + "custom:omni": { + "enabled": flag, + "base_url": _EXACT_KEYED_URL, + "api_key": _EXACT_KEYED_KEY, + }, + "custom": { + "base_url": _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + "api_key": _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + }, + }, + } + + +@pytest.mark.parametrize("flag", _V12_DISABLED_FLAGS, ids=[str(f) for f in _V12_DISABLED_FLAGS]) +def test_exact_keyed_disabled_provider_fails_closed(monkeypatch, flag): + """A disabled ``providers['custom:']`` owns nothing, at every boundary. + + Pinned across all three surfaces a send can enter through — the connection + view, the streaming bundle merge and the non-streaming chokepoint — because + the selection bug sat below all of them and a fix at one would leave the + other two routing a row the user switched off. + """ + import api.routes as routes + + _clear_credential_env(monkeypatch) + _with_direct_config(monkeypatch, _exact_keyed_disabled_cfg(flag)) + label = f"enabled: {flag!r}" + + # 1. Selection and the connection view: nothing owns the slug. + assert config.resolve_custom_provider_connection("custom:omni") == (None, None), label + selection = config.resolve_custom_provider_bundle("custom:omni") + assert selection["status"] == config.CUSTOM_SELECTION_MISSING, label + assert selection["record"] is None, label + assert selection["base_url"] is None and selection["api_key"] is None, label + + # 2. The streaming bundle merge: terminal, and stripped of every authority. + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + _ambient_runtime(provider="custom:active"), + lookup_provider="custom:omni", + ) + verdict = bundle[config.CUSTOM_ROUTE_ERROR_FIELD] + assert verdict is not None, f"{label}: a disabled keyed record still looked routable" + assert verdict["reason"] == config.CUSTOM_ROUTE_UNOWNED, label + assert verdict["provider"] == "custom:omni", label + assert bundle["base_url"] is None and bundle["api_key"] is None, label + assert bundle["api_key"] != config.KEYLESS_CUSTOM_API_KEY, label + _assert_side_fields(bundle, _FOREIGN_AMBIENT_SIDE_FIELDS, label) + + # Neither the disabled record's own pair nor the ambient one it must not + # inherit may survive anywhere in the bundle. + blob = str(bundle) + for leaked in ( + _EXACT_KEYED_URL, + _EXACT_KEYED_KEY, + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + ): + assert leaked not in blob, f"{label}: {leaked!r} leaked onto the disabled route" + + # 3. The non-streaming chokepoint raises rather than returning a holed bundle. + with pytest.raises(config.CustomProviderRouteError) as excinfo: + routes._resolve_agent_connection_bundle( + "custom:omni", + _AMBIENT_SIDE_FIELD_RUNTIME["api_key"], + _AMBIENT_SIDE_FIELD_RUNTIME["base_url"], + _ambient_runtime(provider="custom:active"), + ) + err = excinfo.value + assert err.reason == config.CUSTOM_ROUTE_UNOWNED, label + assert err.provider == "custom:omni", label + assert err.hint, f"{label}: the refusal named no fix" + + +def test_exact_keyed_enabled_provider_still_owns_its_slug(monkeypatch): + """Negative control: only a FALSEY ``enabled`` hides the keyed record. + + A guard that read ``enabled`` as "present means off" would take every + explicitly-enabled keyed provider offline — the same outage as the leak, from + the opposite direction. + """ + _clear_credential_env(monkeypatch) + _with_direct_config(monkeypatch, _exact_keyed_disabled_cfg(True)) + + assert config.resolve_custom_provider_connection("custom:omni") == ( + _EXACT_KEYED_KEY, + _EXACT_KEYED_URL, + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Reviewer probe: a shared endpoint is not shared credential authority +# +# ``_custom_bundle_endpoint_matches()`` identified the runtime as same-authority +# from the URL alone, so a record that merely declared the SAME base_url as the +# active provider was handed the active provider's key. Two providers sharing a +# host — a gateway fronting two accounts, a local proxy — is an ordinary setup, +# and there the record's URL rode out beside somebody else's secret. +# ───────────────────────────────────────────────────────────────────────────── + + +_SHARED_HOST_URL = "https://shared.example/v1" +_SHARED_HOST_AMBIENT_KEY = "ambient-secret-sentinel" +_SHARED_HOST_RECORD_KEY = "record-secret-sentinel" + + +def _shared_host_cfg(record): + """Raw provider ``omni`` on the shared host, while ``active-other`` is live.""" + return { + "model": {"default": "active/model", "provider": "active-other"}, + "providers": {"omni": copy.deepcopy(record)}, + } + + +def _shared_host_runtime(): + """The ambient runtime: a DIFFERENT provider that reached the same host.""" + return { + "provider": "active-other", + "base_url": _SHARED_HOST_URL, + "api_key": _SHARED_HOST_AMBIENT_KEY, + } + + +def test_shared_endpoint_different_provider_does_not_split_credential(monkeypatch): + """The record's OWN credential rides with the record's endpoint. + + Identical URLs plus different keys is exactly the case URL-equality cannot + tell apart: the ambient runtime names itself ``active-other``, so it is + provably a different authority no matter which host it reached. + """ + _clear_credential_env(monkeypatch) + record = {"base_url": _SHARED_HOST_URL, "api_key": _SHARED_HOST_RECORD_KEY} + _with_direct_config(monkeypatch, _shared_host_cfg(record)) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _SHARED_HOST_AMBIENT_KEY, + _SHARED_HOST_URL, + _shared_host_runtime(), + lookup_provider="custom:omni", + ) + + assert bundle[config.CUSTOM_ROUTE_ERROR_FIELD] is None + assert bundle["base_url"] == _SHARED_HOST_URL + assert bundle["api_key"] == _SHARED_HOST_RECORD_KEY, ( + "the non-active record inherited the ambient provider's secret across a " + "shared endpoint" + ) + assert bundle["api_key"] != _SHARED_HOST_AMBIENT_KEY + # The ambient provider is a foreign authority here, so its side fields go too. + assert bundle["credential_pool"] is None + + +def test_shared_endpoint_unresolved_key_fails_closed_not_borrow_ambient(monkeypatch): + """A declared-but-unresolved credential fails closed; it does not borrow. + + The dangerous shape: the record declares ``key_env`` and the variable is + unset, so its own ladder yields nothing. Filling that hole from the ambient + runtime just because both name one host is precisely the split-authority + send — the active provider's key against the record's route. + """ + _clear_credential_env(monkeypatch) + monkeypatch.delenv(_MISSING_ENV_VAR, raising=False) + record = {"base_url": _SHARED_HOST_URL, "key_env": _MISSING_ENV_VAR} + _with_direct_config(monkeypatch, _shared_host_cfg(record)) + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _SHARED_HOST_AMBIENT_KEY, + _SHARED_HOST_URL, + _shared_host_runtime(), + lookup_provider="custom:omni", + ) + + verdict = bundle[config.CUSTOM_ROUTE_ERROR_FIELD] + assert verdict is not None, "an unresolvable credential still looked routable" + assert verdict["reason"] == config.CUSTOM_ROUTE_NO_CREDENTIAL + assert verdict["provider"] == "custom:omni" + assert bundle["api_key"] is None + assert bundle["api_key"] != _SHARED_HOST_AMBIENT_KEY + # Not the keyless placeholder either: the record DOES declare a credential + # source, so an unauthenticated send would be a silent 401, not a keyless one. + assert bundle["api_key"] != config.KEYLESS_CUSTOM_API_KEY + assert _SHARED_HOST_AMBIENT_KEY not in str(bundle) + + +def test_same_provider_sharing_its_endpoint_still_keeps_the_runtime_fields(monkeypatch): + """Negative control: the identity test must not orphan the provider's OWN runtime. + + The ambient runtime dict normally names the very slug being resolved (that is + the active-custom-provider shape the WebUI writes). A provenance test that + rejected it would clear the side fields of every live custom provider. + """ + _clear_credential_env(monkeypatch) + record = {"base_url": _SHARED_HOST_URL, "api_key": _SHARED_HOST_RECORD_KEY} + _with_direct_config(monkeypatch, _shared_host_cfg(record)) + + runtime = _shared_host_runtime() + runtime["provider"] = "custom:omni" + runtime["api_mode"] = "chat_completions" + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _SHARED_HOST_AMBIENT_KEY, + _SHARED_HOST_URL, + runtime, + lookup_provider="custom:omni", + ) + + assert bundle[config.CUSTOM_ROUTE_ERROR_FIELD] is None + assert bundle["base_url"] == _SHARED_HOST_URL + # Still the record's own credential — same authority does not make the + # ambient key authoritative over one the record declares for itself. + assert bundle["api_key"] == _SHARED_HOST_RECORD_KEY + assert bundle["api_mode"] == "chat_completions", ( + "the provider's own runtime side field was cleared as foreign" + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# A credential-only record borrows NO endpoint (PR #7319) +# +# The shape that started this: ``providers: {omni: {api_key: ...}}`` — a raw v12 +# record that declares a CREDENTIAL and no ``base_url`` — while the process is +# already talking to a different, ACTIVE provider whose URL and key are sitting +# in the runtime dict. Selection legitimately picks the omni record (it owns the +# slug, and a declared-but-endpoint-less record is not "missing"), and the merge +# then had no positive statement about where that record's endpoint came from. +# The seeded ``base_url`` was simply still there, so the bundle handed the +# constructor the ACTIVE provider's URL beside the omni record's secret: a +# non-active provider's credential, and the user's prompt, delivered to an +# endpoint neither the record nor the user ever named. +# +# ``endpoint_owned`` is the positive half of that provenance, and it is False +# here. Absence of contradiction is not a tie: the caller's ``resolved_base_url`` +# and a missing runtime dict are both silence. So the route is terminal on its +# own name — :data:`config.CUSTOM_ROUTE_NO_ENDPOINT` — and keeps neither the +# ambient endpoint nor a credential to pair with it, because pairing them is the +# whole defect. +# +# These assert the property at the boundaries that can actually send: the +# production-composed streaming send (all three agent-constructing regions), the +# non-streaming chokepoint, and the auxiliary client — which bypasses AIAgent +# entirely, so ``main_runtime`` would be the only place the stolen pair reached +# the wire. The last two tests are the controls that keep the refusal narrow: a +# COMPLETE record still sends, and a genuinely keyless one still gets the +# placeholder. +# ───────────────────────────────────────────────────────────────────────────── + + +# The non-active record's own secret: the thing that must never leave with the +# active provider's URL. +_CREDENTIAL_ONLY_KEY = "credential-only-omni-key-sentinel" +# The endpoint the record declares in the two control cases below. +_OMNI_OWN_URL = "https://omni-own-endpoint-sentinel.example/v1" + +# The ACTIVE provider: truthy URL and key, plus truthy side fields, all resolved +# for somebody else entirely. Named ``custom:active-other`` so identity settles +# the provenance question without relying on the URLs differing. +_ACTIVE_OTHER_URL = "https://active-other-sentinel.example/v1" +_ACTIVE_OTHER_KEY = "active-other-key-sentinel" +_ACTIVE_OTHER_POOL = ["active-other-pool-sentinel"] +_ACTIVE_OTHER_ACP = "active-other-acp-sentinel" + +_ACTIVE_OTHER_RUNTIME = { + "provider": "custom:active-other", + "base_url": _ACTIVE_OTHER_URL, + "api_key": _ACTIVE_OTHER_KEY, + "api_mode": "chat_completions", + "command": _ACTIVE_OTHER_ACP, + "args": ["--active-other-arg-sentinel"], + "credential_pool": _ACTIVE_OTHER_POOL, +} + +# Everything the refused route could have walked away with. Asserted as a set +# over the whole captured harness, so a leak through ANY field — bundle, error +# payload, constructor kwargs, aux handoff — fails rather than only the fields a +# test remembered to name. +_ACTIVE_OTHER_SENTINELS = ( + _ACTIVE_OTHER_URL, + _ACTIVE_OTHER_KEY, + _ACTIVE_OTHER_POOL[0], + _ACTIVE_OTHER_ACP, +) + +_CREDENTIAL_ONLY_MODEL = "@custom:omni:antigravity/gemini-3.7-flash-tiered" + + +def _credential_only_cfg(): + """The raw v12 record that declares a credential and NO endpoint.""" + return _v12_raw_cfg({"api_key": _CREDENTIAL_ONLY_KEY}) + + +def _assert_no_borrowed_pair(blob, label): + """Neither the record's own secret nor the active provider's connection.""" + for leaked in _ACTIVE_OTHER_SENTINELS: + assert leaked not in blob, ( + f"{label}: the ACTIVE provider's {leaked!r} survived a route whose " + "record declares no endpoint" + ) + assert _CREDENTIAL_ONLY_KEY not in blob, ( + f"{label}: the non-active record's credential was handed on without an " + "endpoint of its own to send it to" + ) + assert config.KEYLESS_CUSTOM_API_KEY not in blob, ( + f"{label}: a record that DECLARES a credential was called keyless" + ) + + +def test_credential_only_record_refuses_the_active_providers_endpoint(monkeypatch): + """The merge verdict itself: terminal, and holding neither half of the pair.""" + _clear_credential_env(monkeypatch) + _with_direct_config(monkeypatch, _credential_only_cfg()) + + selection = config.resolve_custom_provider_bundle("custom:omni") + assert selection["record"] is not None, ( + "a credential-only record must still OWN its slug — treating it as " + "missing is a different bug with the same symptom" + ) + assert selection["endpoint_owned"] is False + assert selection["api_key"] == _CREDENTIAL_ONLY_KEY + assert selection["keyless"] is False + + bundle = config.merge_custom_provider_runtime_bundle( + "custom:omni", + _ACTIVE_OTHER_KEY, + _ACTIVE_OTHER_URL, + copy.deepcopy(_ACTIVE_OTHER_RUNTIME), + lookup_provider="custom:omni", + ) + + verdict = bundle[config.CUSTOM_ROUTE_ERROR_FIELD] + assert verdict is not None, "an endpoint-less record still looked routable" + assert verdict["reason"] == config.CUSTOM_ROUTE_NO_ENDPOINT + assert verdict["provider"] == "custom:omni" + assert verdict["hint"], "the refusal named no setting to fix" + assert bundle["base_url"] is None + assert bundle["api_key"] is None + # Still the NAMED slug: rewriting it to generic ``custom`` would present an + # unresolvable route as a resolved one. + assert bundle["provider"] == "custom:omni" + # The refused endpoint's side fields go with it; this record declares none. + _assert_side_fields(bundle, _FOREIGN_AMBIENT_SIDE_FIELDS, "credential-only record") + _assert_no_borrowed_pair(str(bundle), "merged bundle") + + +def test_credential_only_record_refuses_the_production_composed_send(monkeypatch): + """Refuse on initial send before any constructor, client, or cache write. + + A raw record declaring only a credential and no endpoint must fail closed + at initial resolution as ``CUSTOM_ROUTE_NO_ENDPOINT`` + (custom_provider_endpoint_unresolved), constructing no agent, making no + explicit client call, sending no turn, and writing no cache entry. + """ + _clear_credential_env(monkeypatch) + + captured, apperrors = _run_composed_send_expecting_refusal( + monkeypatch, + _credential_only_cfg(), + copy.deepcopy(_ACTIVE_OTHER_RUNTIME), + "session-1806-credential-only-omni", + model=_CREDENTIAL_ONLY_MODEL, + ) + + payload = apperrors[-1] + # The STRUCTURED verdict, not the prose: a generic failure whose message + # happens to read "resolved no endpoint" would pass a message-only check, + # and the message is free to be reworded. This is the same terminal reason + # the merge-level and auxiliary-client regressions assert. + # The literal is pinned alongside the constant because it is the wire value + # the client branches on; renaming it silently would be a breaking change. + assert payload["reason"] == config.CUSTOM_ROUTE_NO_ENDPOINT == ( + "custom_provider_endpoint_unresolved" + ), f"the failure did not carry {config.CUSTOM_ROUTE_NO_ENDPOINT}: {payload}" + assert "custom:omni" in payload["message"], payload + assert "resolved no endpoint" in payload["message"], ( + f"the failure did not name {config.CUSTOM_ROUTE_NO_ENDPOINT}: {payload}" + ) + assert "base_url" in payload["hint"], payload + + assert not captured.get("init_kwargs_history"), ( + "an agent was constructed for a record that declares no endpoint" + ) + assert not captured.get("explicit_client_kwargs_calls"), ( + "a client was configured with the active provider's borrowed connection" + ) + + _assert_no_borrowed_pair(str(payload) + str(captured), "composed send") + + +@pytest.mark.parametrize("fail_first", ["returned_error", "raised"]) +def test_credential_only_record_refuses_the_production_composed_retry( + monkeypatch, fail_first +): + """A standard-v12 record losing only its endpoint stops a 401 self-heal. + + The initial send is routable and cached with the complete raw record. During + the self-heal, only that record's ``base_url`` is removed; its credential and + the foreign ambient runtime bundle remain available. The refreshed terminal + route must therefore prevent every retry construction and cache write. + """ + _clear_credential_env(monkeypatch) + session_id = f"session-1806-credential-only-omni-{fail_first}" + label = f"credential-only record / {fail_first}" + cfg_dict = _v12_raw_cfg({"base_url": _V12_URL, "api_key": _V12_KEY}) + + def remove_only_the_endpoint(): + config.cfg["providers"]["omni"]["base_url"] = "" + + captured, apperrors, cache_at_heal, cache_after = ( + _run_composed_retry_expecting_abandoned_heal( + monkeypatch, + cfg_dict, + copy.deepcopy(_ACTIVE_OTHER_RUNTIME), + session_id, + fail_first=fail_first, + heal_mutate=remove_only_the_endpoint, + ) + ) + + _assert_retry_abandoned( + captured, + apperrors, + cache_at_heal, + cache_after, + session_id, + label, + expected_cause="resolved no endpoint", + expected_initial_url=_V12_URL, + expected_initial_key=_V12_KEY, + ) + + payload = apperrors[-1] + assert payload["reason"] == config.CUSTOM_ROUTE_NO_ENDPOINT, payload + assert "base_url" in payload["hint"], payload + assert _V12_KEY not in str(payload), f"{label}: leaked the record's own key into payload" + blob = str(captured) + str(apperrors) + for leaked in _ACTIVE_OTHER_SENTINELS: + assert leaked not in blob, f"{label}: {leaked!r} leaked into the retry path" + assert config.KEYLESS_CUSTOM_API_KEY not in blob + + +# The two consumers whose auxiliary client can answer outright. There AIAgent is +# never built, so ``main_runtime`` is the ONLY carrier of the resolved authority +# — and the only place a borrowed pair could still reach the wire after every +# constructor assertion above passes. +_CREDENTIAL_ONLY_AUX_DRIVERS = [ + ( + "git commit message", + lambda routes, session: routes._llm_git_commit_message("sys", "user", session=session), + ), + ( + "update summary", + lambda routes, _session: routes._llm_update_summary("sys", "user", active_profile=None), + ), +] + + +@pytest.mark.parametrize( + "label,invoke", + _CREDENTIAL_ONLY_AUX_DRIVERS, + ids=[d[0] for d in _CREDENTIAL_ONLY_AUX_DRIVERS], +) +def test_credential_only_record_reaches_no_auxiliary_client(monkeypatch, label, invoke): + """The refusal is terminal BEFORE ``main_runtime`` is built, so the aux client + is never handed the active provider's connection. + + ``_resolve_agent_connection_bundle`` raises at the chokepoint, which is + upstream of ``_auxiliary_main_runtime``. Returning the holed bundle instead + would let the aux path send with the borrowed pair while bypassing every + AIAgent-side guard entirely. + """ + import api.routes as routes + + _clear_credential_env(monkeypatch) + cfg_dict = _credential_only_cfg() + cfg_dict["model"] = {"default": _CREDENTIAL_ONLY_MODEL, "provider": "custom:omni"} + + captured, fake_session = _setup_route_consumer_runtime( + monkeypatch, + cfg_dict=cfg_dict, + runtime_dict=copy.deepcopy(_ACTIVE_OTHER_RUNTIME), + ) + recorded_main_runtime = {} + _decline_auxiliary_client(monkeypatch, recorded_main_runtime) + + with pytest.raises(config.CustomProviderRouteError) as excinfo: + invoke(routes, fake_session) + + assert excinfo.value.reason == config.CUSTOM_ROUTE_NO_ENDPOINT, label + assert excinfo.value.provider == "custom:omni", label + assert excinfo.value.hint, f"{label}: the refusal named no setting to fix" + + assert not recorded_main_runtime, ( + f"{label}: the auxiliary client was handed {recorded_main_runtime!r} for a " + "route with no endpoint of its own" + ) + assert "init_kwargs" not in captured, f"{label}: an agent was constructed anyway" + _assert_no_borrowed_pair( + str(excinfo.value) + str(recorded_main_runtime) + str(captured), label + ) + + +# ── controls: the refusal is about MISSING provenance, not about raw records ── + + +def test_complete_raw_record_still_sends_through_its_own_endpoint(monkeypatch): + """Control: the same record WITH a ``base_url`` routes exactly as before. + + Without this, "refuse when ``endpoint_owned`` is False" is equally satisfied + by refusing every raw ``providers:`` record — which would break the + standard v12 shape outright. + """ + _clear_credential_env(monkeypatch) + + init_kwargs = _run_composed_send( + monkeypatch, + _v12_raw_cfg({"base_url": _OMNI_OWN_URL, "api_key": _CREDENTIAL_ONLY_KEY}), + copy.deepcopy(_ACTIVE_OTHER_RUNTIME), + "session-1806-credential-only-control-complete", + ) + + assert init_kwargs["base_url"] == _OMNI_OWN_URL + assert init_kwargs["api_key"] == _CREDENTIAL_ONLY_KEY + assert init_kwargs["provider"] == "custom" + # The record owns an endpoint the active provider did not resolve, so the + # active provider's side fields are provably foreign and go. + _assert_side_fields(init_kwargs, _FOREIGN_AMBIENT_SIDE_FIELDS, "complete raw record") + for leaked in _ACTIVE_OTHER_SENTINELS: + assert leaked not in str(init_kwargs), ( + f"the active provider's {leaked!r} reached a record-owned route" + ) + + +def test_genuinely_keyless_raw_record_still_sends_with_the_placeholder(monkeypatch): + """Control: a record declaring an endpoint and NO credential is still keyless. + + The mirror of the case under test — endpoint but no credential, rather than + credential but no endpoint. This one is a complete statement about the route + ("that endpoint is unauthenticated"), so it must still route with + :data:`config.KEYLESS_CUSTOM_API_KEY` and must NOT borrow the active + provider's key on the way. + """ + _clear_credential_env(monkeypatch) + + init_kwargs = _run_composed_send( + monkeypatch, + _v12_raw_cfg({"base_url": _OMNI_OWN_URL}), + copy.deepcopy(_ACTIVE_OTHER_RUNTIME), + "session-1806-credential-only-control-keyless", + ) + + assert init_kwargs["base_url"] == _OMNI_OWN_URL + assert init_kwargs["api_key"] == config.KEYLESS_CUSTOM_API_KEY + assert init_kwargs["api_key"] != _ACTIVE_OTHER_KEY + assert init_kwargs["provider"] == "custom" diff --git a/tests/test_issue1897_profile_switch_agent_cache.py b/tests/test_issue1897_profile_switch_agent_cache.py index 7c6782f7..684c5a01 100644 --- a/tests/test_issue1897_profile_switch_agent_cache.py +++ b/tests/test_issue1897_profile_switch_agent_cache.py @@ -13,10 +13,151 @@ REPO = Path(__file__).resolve().parent.parent STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8") +def _compute_agent_cache_signature_source() -> str: + """Return the source of the `_compute_agent_cache_signature()` helper. + + The signature blob used to be inlined in the streaming send path; it now + lives in this helper so the initial send and both self-heal retry paths + derive the signature from the same final runtime bundle. + """ + start = STREAMING_PY.index("def _compute_agent_cache_signature(") + end = STREAMING_PY.index("\ndef ", start) + return STREAMING_PY[start:end] + + def _signature_block() -> str: - sig_start = STREAMING_PY.index("_sig_blob = _json.dumps") - sig_end = STREAMING_PY.index("], sort_keys=True)", sig_start) - return STREAMING_PY[sig_start:sig_end] + """Return the `_json.dumps([...])` field list the signature hashes.""" + helper = _compute_agent_cache_signature_source() + sig_start = helper.index("_sig_blob = _json.dumps") + sig_end = helper.index("], sort_keys=True)", sig_start) + return helper[sig_start:sig_end] + + +def _production_signature_calls() -> list[tuple[int, str]]: + """Return `(offset, source)` for every production signature call site. + + Paren-balanced so the whole multi-line keyword-argument list is captured, + and every live call site is returned so a retry path cannot silently drop a + field that the initial send still passes. Commented-out occurrences are + skipped -- a disabled retry call must read as missing, not as present. + """ + marker = "_agent_sig = _compute_agent_cache_signature(" + calls: list[tuple[int, str]] = [] + pos = STREAMING_PY.find(marker) + while pos != -1: + line_start = STREAMING_PY.rfind("\n", 0, pos) + 1 + if STREAMING_PY[line_start:pos].strip(): + # Commented-out (or otherwise non-statement) occurrence: it no + # longer runs, so it must not count as a live call site. + pos = STREAMING_PY.find(marker, pos + 1) + continue + depth = 0 + for idx in range(pos + len(marker) - 1, len(STREAMING_PY)): + char = STREAMING_PY[idx] + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + calls.append((pos, STREAMING_PY[pos:idx + 1])) + break + else: + raise AssertionError("unterminated _compute_agent_cache_signature( call") + pos = STREAMING_PY.find(marker, pos + 1) + return calls + + +# Lifecycle anchors for the streaming send path. Pinning one signature call to +# each region keeps the oracle honest: a retry call that silently disappears +# fails the region check instead of hiding behind the calls that remain. +INITIAL_SEND_MARKER = "# ── Agent cache: reuse across messages in the same session ──" +RETURNED_ERROR_SELF_HEAL_MARKER = ( + "logger.info('[webui] self-heal: retrying stream after credential refresh')" +) +RAISED_EXCEPTION_SELF_HEAL_MARKER = ( + "logger.info('[webui] self-heal (except path): " + "retrying stream after credential refresh')" +) + +# Each region ends at the cache write that stores the signature it just +# computed. Bounding the region at the write -- rather than at the next +# lifecycle anchor or at EOF -- is what proves the signature is recomputed +# *before* it is cached: a call moved below its own write lands outside the +# region and fails, instead of passing while the write consumes a stale +# `_agent_sig`. +INITIAL_SEND_CACHE_WRITE_MARKER = ( + "SESSION_AGENT_CACHE[session_id] = (agent, _agent_sig)" +) +RETURNED_ERROR_SELF_HEAL_CACHE_WRITE_MARKER = ( + "_SAC[session_id] = (agent, _agent_sig)" +) +RAISED_EXCEPTION_SELF_HEAL_CACHE_WRITE_MARKER = ( + "_SAC2[session_id] = (_heal_agent, _agent_sig)" +) + + +def _marker_offset(marker: str) -> int: + """Return the single offset of a lifecycle marker in streaming.py.""" + offset = STREAMING_PY.index(marker) + assert STREAMING_PY.find(marker, offset + 1) == -1, ( + "lifecycle marker is no longer unique in streaming.py:\n" + marker + ) + return offset + + +def _signature_calls_by_region( + calls: list[tuple[int, str]], +) -> dict[str, tuple[int, str]]: + """Map each lifecycle region to the single signature call inside it. + + The regions are the initial send, the returned-error self-heal retry and + the raised-exception self-heal retry; each runs from its lifecycle anchor + to the cache write that consumes `_agent_sig`. Requiring exactly one call + inside those bounds means neither a dropped retry call (masked by its + surviving siblings) nor a call recomputed after its own cache write (which + would leave the write storing a stale signature) can read as correct. + """ + initial = _marker_offset(INITIAL_SEND_MARKER) + initial_cache_write = _marker_offset(INITIAL_SEND_CACHE_WRITE_MARKER) + returned_error = _marker_offset(RETURNED_ERROR_SELF_HEAL_MARKER) + returned_error_cache_write = _marker_offset( + RETURNED_ERROR_SELF_HEAL_CACHE_WRITE_MARKER + ) + raised_exception = _marker_offset(RAISED_EXCEPTION_SELF_HEAL_MARKER) + raised_exception_cache_write = _marker_offset( + RAISED_EXCEPTION_SELF_HEAL_CACHE_WRITE_MARKER + ) + assert ( + initial + < initial_cache_write + < returned_error + < returned_error_cache_write + < raised_exception + < raised_exception_cache_write + ), ( + "streaming.py lifecycle regions are out of order; these anchors no " + "longer describe the send path" + ) + + bounds = { + "initial send": (initial, initial_cache_write), + "returned-error self-heal": (returned_error, returned_error_cache_write), + "raised-exception self-heal": ( + raised_exception, + raised_exception_cache_write, + ), + } + by_region: dict[str, tuple[int, str]] = {} + for region, (start, end) in bounds.items(): + found = [(offset, call) for offset, call in calls if start < offset < end] + assert len(found) == 1, ( + "expected exactly one _compute_agent_cache_signature() call in the " + f"{region} region of streaming.py -- between its lifecycle anchor " + f"and the cache write that stores `_agent_sig` -- found " + f"{len(found)}" + ) + by_region[region] = found[0] + return by_region def test_same_session_profile_switch_rebuilds_agent_under_new_soul_home(tmp_path, monkeypatch): @@ -225,22 +366,51 @@ def test_same_session_profile_switch_rebuilds_agent_under_new_soul_home(tmp_path def test_cache_signature_includes_profile_home(): block = _signature_block() - assert "_profile_home" in block, ( - "SESSION_AGENT_CACHE signature is missing `_profile_home`. Without this, " + assert "profile_home" in block, ( + "SESSION_AGENT_CACHE signature is missing `profile_home`. Without this, " "same-session profile switches reuse the cached agent built under the " "previous profile's HERMES_HOME, leaking the old SOUL.md into new turns." ) + calls = _production_signature_calls() + assert len(calls) == 3, ( + "expected exactly three _compute_agent_cache_signature() call sites " + "(initial send plus both self-heal retries), found " + f"{len(calls)}" + ) + for region, (_offset, call) in _signature_calls_by_region(calls).items(): + assert "profile_home=_profile_home" in call, ( + f"the {region} signature call site must pass the resolved profile home, " + "or a retry re-caches the agent under a signature that ignores the active profile:\n" + call + ) + def test_profile_home_resolved_before_cache_signature(): profile_home_assignment = STREAMING_PY.index("_profile_home = str(_profile_home_path)") - sig_start = STREAMING_PY.index("_sig_blob = _json.dumps") - assert profile_home_assignment < sig_start + + calls = _production_signature_calls() + assert len(calls) == 3, ( + "expected exactly three _compute_agent_cache_signature() call sites " + "(initial send plus both self-heal retries), found " + f"{len(calls)}" + ) + for region, (offset, call) in _signature_calls_by_region(calls).items(): + assert profile_home_assignment < offset, ( + "`_profile_home` must be resolved before the cache signature is " + f"computed, otherwise the {region} signature hashes a stale/unbound " + "home." + ) + assert "profile_home=_profile_home" in call, call + assert "max_iterations_cfg=_max_iterations_cfg" in call, call + assert "max_tokens_cfg=_max_tokens_cfg" in call, call def test_signature_uses_profile_home_with_fallback(): block = _signature_block() - assert "_profile_home or ''" in block, ( - "Signature should use `_profile_home or ''` so empty-home deployments get " + assert "profile_home or ''" in block, ( + "Signature should use `profile_home or ''` so empty-home deployments get " "a stable cache key rather than unnecessary cache churn." ) + + for _offset, call in _production_signature_calls(): + assert "profile_home=_profile_home" in call, call diff --git a/tests/test_sprint42.py b/tests/test_sprint42.py index b3bc27ee..a8b6d5ce 100644 --- a/tests/test_sprint42.py +++ b/tests/test_sprint42.py @@ -123,12 +123,17 @@ class TestRuntimeRouteInjection(unittest.TestCase): Since issue #772 these are passed defensively via inspect-guarded kwargs so the WebUI degrades gracefully against older hermes-agent builds. + + They are read from the resolved connection bundle rather than straight + off the runtime provider dict: a named custom provider owns none of them, + so the bundle clears them and forwarding ``_rt`` directly would send the + ambient provider's transport/protocol/pool to a custom endpoint. """ for snippet in ( - "_agent_kwargs['api_mode'] = _rt.get('api_mode')", - "_agent_kwargs['acp_command'] = _rt.get('command')", - "_agent_kwargs['acp_args'] = _rt.get('args')", - "_agent_kwargs['credential_pool'] = _rt.get('credential_pool')", + "_agent_kwargs['api_mode'] = _runtime_bundle['api_mode']", + "_agent_kwargs['acp_command'] = _runtime_bundle['acp_command']", + "_agent_kwargs['acp_args'] = _runtime_bundle['acp_args']", + "_agent_kwargs['credential_pool'] = _runtime_bundle['credential_pool']", ): self.assertIn( snippet, diff --git a/tests/test_streaming_max_tokens_quota.py b/tests/test_streaming_max_tokens_quota.py index 2e37734d..ed352290 100644 --- a/tests/test_streaming_max_tokens_quota.py +++ b/tests/test_streaming_max_tokens_quota.py @@ -16,6 +16,156 @@ def _src() -> str: return STREAMING.read_text(encoding="utf-8") +def _compute_agent_cache_signature_source() -> str: + """Return the source of the `_compute_agent_cache_signature()` helper. + + The signature blob used to be inlined in the streaming send path; it now + lives in this helper so the initial send and both self-heal retry paths + derive the signature from the same final runtime bundle. + """ + src = _src() + start = src.index("def _compute_agent_cache_signature(") + end = src.index("\ndef ", start) + return src[start:end] + + +def _signature_blob() -> str: + """Return the `_json.dumps([...])` field list the signature hashes.""" + helper = _compute_agent_cache_signature_source() + blob_start = helper.index("_sig_blob = _json.dumps") + blob_end = helper.index("], sort_keys=True)", blob_start) + return helper[blob_start:blob_end] + + +def _production_signature_calls() -> list[tuple[int, str]]: + """Return `(offset, source)` for every production signature call site. + + Paren-balanced so the whole multi-line keyword-argument list is captured, + and every live call site is returned so a retry path cannot silently drop a + field that the initial send still passes. Commented-out occurrences are + skipped -- a disabled retry call must read as missing, not as present. + """ + src = _src() + marker = "_agent_sig = _compute_agent_cache_signature(" + calls: list[tuple[int, str]] = [] + pos = src.find(marker) + while pos != -1: + line_start = src.rfind("\n", 0, pos) + 1 + if src[line_start:pos].strip(): + # Commented-out (or otherwise non-statement) occurrence: it no + # longer runs, so it must not count as a live call site. + pos = src.find(marker, pos + 1) + continue + depth = 0 + for idx in range(pos + len(marker) - 1, len(src)): + char = src[idx] + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + calls.append((pos, src[pos:idx + 1])) + break + else: + raise AssertionError("unterminated _compute_agent_cache_signature( call") + pos = src.find(marker, pos + 1) + return calls + + +# Lifecycle anchors for the streaming send path. Pinning one signature call to +# each region keeps the oracle honest: a retry call that silently disappears +# fails the region check instead of hiding behind the calls that remain. +INITIAL_SEND_MARKER = "# ── Agent cache: reuse across messages in the same session ──" +RETURNED_ERROR_SELF_HEAL_MARKER = ( + "logger.info('[webui] self-heal: retrying stream after credential refresh')" +) +RAISED_EXCEPTION_SELF_HEAL_MARKER = ( + "logger.info('[webui] self-heal (except path): " + "retrying stream after credential refresh')" +) + +# Each region ends at the cache write that stores the signature it just +# computed. Bounding the region at the write -- rather than at the next +# lifecycle anchor or at EOF -- is what proves the signature is recomputed +# *before* it is cached: a call moved below its own write lands outside the +# region and fails, instead of passing while the write consumes a stale +# `_agent_sig`. +INITIAL_SEND_CACHE_WRITE_MARKER = ( + "SESSION_AGENT_CACHE[session_id] = (agent, _agent_sig)" +) +RETURNED_ERROR_SELF_HEAL_CACHE_WRITE_MARKER = ( + "_SAC[session_id] = (agent, _agent_sig)" +) +RAISED_EXCEPTION_SELF_HEAL_CACHE_WRITE_MARKER = ( + "_SAC2[session_id] = (_heal_agent, _agent_sig)" +) + + +def _marker_offset(marker: str) -> int: + """Return the single offset of a lifecycle marker in streaming.py.""" + src = _src() + offset = src.index(marker) + assert src.find(marker, offset + 1) == -1, ( + "lifecycle marker is no longer unique in streaming.py:\n" + marker + ) + return offset + + +def _signature_calls_by_region( + calls: list[tuple[int, str]], +) -> dict[str, tuple[int, str]]: + """Map each lifecycle region to the single signature call inside it. + + The regions are the initial send, the returned-error self-heal retry and + the raised-exception self-heal retry; each runs from its lifecycle anchor + to the cache write that consumes `_agent_sig`. Requiring exactly one call + inside those bounds means neither a dropped retry call (masked by its + surviving siblings) nor a call recomputed after its own cache write (which + would leave the write storing a stale signature) can read as correct. + """ + initial = _marker_offset(INITIAL_SEND_MARKER) + initial_cache_write = _marker_offset(INITIAL_SEND_CACHE_WRITE_MARKER) + returned_error = _marker_offset(RETURNED_ERROR_SELF_HEAL_MARKER) + returned_error_cache_write = _marker_offset( + RETURNED_ERROR_SELF_HEAL_CACHE_WRITE_MARKER + ) + raised_exception = _marker_offset(RAISED_EXCEPTION_SELF_HEAL_MARKER) + raised_exception_cache_write = _marker_offset( + RAISED_EXCEPTION_SELF_HEAL_CACHE_WRITE_MARKER + ) + assert ( + initial + < initial_cache_write + < returned_error + < returned_error_cache_write + < raised_exception + < raised_exception_cache_write + ), ( + "streaming.py lifecycle regions are out of order; these anchors no " + "longer describe the send path" + ) + + bounds = { + "initial send": (initial, initial_cache_write), + "returned-error self-heal": (returned_error, returned_error_cache_write), + "raised-exception self-heal": ( + raised_exception, + raised_exception_cache_write, + ), + } + by_region: dict[str, tuple[int, str]] = {} + for region, (start, end) in bounds.items(): + found = [(offset, call) for offset, call in calls if start < offset < end] + assert len(found) == 1, ( + "expected exactly one _compute_agent_cache_signature() call in the " + f"{region} region of streaming.py -- between its lifecycle anchor " + f"and the cache write that stores `_agent_sig` -- found " + f"{len(found)}" + ) + by_region[region] = found[0] + return by_region + + def test_streaming_passes_configured_max_tokens_to_agent(): src = _src() assert "_raw_max_tokens = _cfg.get('max_tokens')" in src @@ -24,9 +174,31 @@ def test_streaming_passes_configured_max_tokens_to_agent(): def test_streaming_agent_cache_signature_includes_max_tokens_and_fallback(): - src = _src() - assert "_max_tokens_cfg or ''" in src - assert "_fallback_resolved or {}" in src + blob = _signature_blob() + assert "max_tokens_cfg or ''" in blob, ( + "_compute_agent_cache_signature() must hash max_tokens_cfg, or a " + "max_tokens change reuses the agent built on the old output ceiling." + ) + assert "fallback_resolved or {}" in blob, ( + "_compute_agent_cache_signature() must hash the resolved fallback " + "chain so a fallback edit mints a new agent." + ) + + calls = _production_signature_calls() + assert len(calls) == 3, ( + "expected exactly three _compute_agent_cache_signature() call sites " + "(initial send plus both self-heal retries), found " + f"{len(calls)}" + ) + for region, (_offset, call) in _signature_calls_by_region(calls).items(): + assert "max_tokens_cfg=_max_tokens_cfg" in call, ( + f"the {region} signature call must pass the resolved " + "max_tokens:\n" + call + ) + assert "fallback_resolved=_fallback_resolved" in call, ( + f"the {region} signature call must pass the resolved fallback " + "chain:\n" + call + ) def test_openrouter_more_credits_error_is_classified_as_quota():