fix(providers): reduce pool-exhaustion TTL for HTTP 402 from 1h to 2min (#6626)

* fix(providers): reduce pool-exhaustion TTL for HTTP 402 from 1h to 2min

Mirrors the agent-side fix: when a provider returns HTTP 402 (Payment
Required), the credential pool marks the key as exhausted with a 1-hour
TTL — even though recharging credits restores the key immediately.

Both copies of _entry_exhausted_ttl_seconds() now handle HTTP 402 with
a 2-minute TTL, matching the new EXHAUSTED_TTL_402_SECONDS constant in
agent/credential_pool.py.

* fix(providers): tie HTTP 402 TTL to installed runtime contract (#6626)

* test(providers): behavior-test 402 TTL 119/120s boundary on both helper copies (#6626)

Add executed boundary tests for the HTTP 402 pool-exhaustion TTL:

- Host helper: with the runtime TTL monkeypatched to 120, an exhausted
  entry is still unavailable at 119s and becomes available at 120s, for
  both the integer 402 and the persisted string "402" error codes.
- Embedded _ACCOUNT_USAGE_SUBPROCESS_CODE copy: execute the subprocess
  code under the existing quota-subprocess harness and assert the same
  TTL and 119/120s eligibility boundary against the embedded helper,
  instead of counting source-string occurrences.
- Keep the legacy 3600s fallback and last_error_reset_at precedence
  assertions.

* test(providers): make 402 TTL parity tests self-contained for standalone CI

The parity tests imported agent.credential_pool directly, but the WebUI
test suite runs standalone (Hermes Agent runtime is not installed in CI),
causing ModuleNotFoundError on every shard since 08-02. Simulate the
runtime contract with a fake agent.credential_pool module (same pattern
already used by the embedded-subprocess helper test) so both the 120s
cooldown and the legacy 1h fallback paths are covered.

---------

Co-authored-by: nesquena-hermes <nesquena+hermes@gmail.com>

api/providers.py

@@ -425,6 +425,19 @@ def _entry_exhausted_ttl_seconds(error_code):
     code = str(error_code or "").strip()
     if code == "401":
         return 5 * 60
+    if code == "402":
+        # #6626: keep WebUI's eligibility decision tied to the installed
+        # runtime contract. The runtime routes 402 via
+        # credential_pool._exhausted_ttl() (120s when the new
+        # EXHAUSTED_TTL_402_SECONDS is present, 1h fallback otherwise).
+        # Hard-coding 120s here would let display/probe code mark an entry
+        # usable before CredentialPool.select() is willing to lease it on
+        # mixed-version installations.
+        try:
+            from agent.credential_pool import _exhausted_ttl as _runtime_exhausted_ttl
+            return _runtime_exhausted_ttl(int(code))
+        except Exception:
+            return 60 * 60
     return 60 * 60
 
 
@@ -824,6 +837,19 @@ def _entry_exhausted_ttl_seconds(error_code):
     code = str(error_code or "").strip()
     if code == "401":
         return 5 * 60
+    if code == "402":
+        # #6626: keep WebUI's eligibility decision tied to the installed
+        # runtime contract. The runtime routes 402 via
+        # credential_pool._exhausted_ttl() (120s when the new
+        # EXHAUSTED_TTL_402_SECONDS is present, 1h fallback otherwise).
+        # Hard-coding 120s here would let display/probe code mark an entry
+        # usable before CredentialPool.select() is willing to lease it on
+        # mixed-version installations.
+        try:
+            from agent.credential_pool import _exhausted_ttl as _runtime_exhausted_ttl
+            return _runtime_exhausted_ttl(int(code))
+        except Exception:
+            return 60 * 60
     return 60 * 60
 
 

tests/test_issue6626_402_ttl_parity.py

@@ -0,0 +1,180 @@
+"""Tests for issue #6626: HTTP 402 pool-exhaustion TTL parity with the runtime.
+
+WebUI's copies of ``_entry_exhausted_ttl_seconds()`` must stay tied to the
+installed runtime contract (``agent.credential_pool._exhausted_ttl``): when the
+runtime exposes the 2-minute 402 cooldown, WebUI uses it; on mixed-version
+installs the one-hour fallback is preserved, so display/probe code never marks
+an entry usable earlier than ``CredentialPool.select()`` will.
+
+The WebUI test suite runs standalone (the Hermes Agent runtime is not
+installed in CI), so parity tests simulate the runtime contract with a fake
+``agent.credential_pool`` module instead of importing the real one.
+"""
+
+import sys
+import types
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+
+import pytest
+
+
+def _entry(status="exhausted", error_code=None, status_at=None, reset_at=None):
+    return SimpleNamespace(
+        last_status=status,
+        last_error_code=error_code,
+        last_status_at=status_at,
+        last_error_reset_at=reset_at,
+    )
+
+
+@pytest.fixture
+def providers():
+    import api.providers as p
+    return p
+
+
+@pytest.fixture
+def fake_runtime(monkeypatch):
+    """Install a fake ``agent.credential_pool`` module in ``sys.modules``.
+
+    Mirrors the standalone CI environment where the real Hermes Agent runtime
+    is not installed. Exposes ``_exhausted_ttl`` (default: 120s for 402, the
+    contract when ``EXHAUSTED_TTL_402_SECONDS`` is present in the runtime).
+    """
+
+    def _install(ttl_402):
+        agent_mod = types.ModuleType("agent")
+        agent_mod.__path__ = []
+        credential_pool_mod = types.ModuleType("agent.credential_pool")
+        credential_pool_mod._exhausted_ttl = (
+            lambda error_code: ttl_402 if error_code == 402 else 3600
+        )
+        monkeypatch.setitem(sys.modules, "agent", agent_mod)
+        monkeypatch.setitem(sys.modules, "agent.credential_pool", credential_pool_mod)
+        return credential_pool_mod
+
+    return _install
+
+
+def test_402_ttl_matches_installed_runtime(providers, fake_runtime):
+    """Parity: WebUI eligibility must equal the installed runtime contract."""
+    fake_runtime(ttl_402=120)
+
+    from agent.credential_pool import _exhausted_ttl as runtime_ttl
+
+    assert providers._entry_exhausted_ttl_seconds(402) == runtime_ttl(402)
+    # Persisted error codes are strings ("402") — same contract.
+    assert providers._entry_exhausted_ttl_seconds("402") == runtime_ttl(402)
+
+
+def test_402_boundary_not_one_hour(providers, fake_runtime):
+    """With a runtime that supports the 402 cooldown, WebUI is NOT 1h (and not 119s)."""
+    fake_runtime(ttl_402=120)
+
+    from agent.credential_pool import _exhausted_ttl as runtime_ttl
+
+    ttl = providers._entry_exhausted_ttl_seconds("402")
+    assert ttl == runtime_ttl(402)
+    assert ttl == 120
+    assert ttl != 119
+    assert ttl != 3600
+
+
+def test_host_helper_402_boundary_119_120(providers, fake_runtime):
+    """Host helper: with runtime TTL=120, an exhausted 402 entry stays unavailable
+    at 119s and becomes available at 120s (both int and persisted-string codes)."""
+    fake_runtime(ttl_402=120)
+
+    assert providers._entry_exhausted_ttl_seconds(402) == 120
+    assert providers._entry_exhausted_ttl_seconds("402") == 120
+
+    now = datetime.now(timezone.utc)
+    entry_119 = _entry(error_code="402", status_at=(now - timedelta(seconds=119)).timestamp())
+    entry_120 = _entry(error_code="402", status_at=(now - timedelta(seconds=120)).timestamp())
+    # 119s after the failure the pool is still exhausted (unavailable)...
+    assert providers._entry_is_pool_exhausted(entry_119) is True
+    # ...and exactly at the 120s TTL boundary it becomes available again.
+    assert providers._entry_is_pool_exhausted(entry_120) is False
+
+
+def test_401_and_default_unchanged(providers):
+    assert providers._entry_exhausted_ttl_seconds(401) == 5 * 60
+    assert providers._entry_exhausted_ttl_seconds("401") == 5 * 60
+    assert providers._entry_exhausted_ttl_seconds(None) == 60 * 60
+    assert providers._entry_exhausted_ttl_seconds("") == 60 * 60
+    assert providers._entry_exhausted_ttl_seconds("403") == 60 * 60
+
+
+def test_legacy_runtime_fallback_preserved(providers, fake_runtime):
+    """Older installed runtime (no 402 support) -> WebUI keeps the 1h fallback."""
+    fake_runtime(ttl_402=3600)
+
+    assert providers._entry_exhausted_ttl_seconds("402") == 60 * 60
+    assert providers._entry_exhausted_ttl_seconds(402) == 60 * 60
+
+
+def test_pool_exhausted_until_uses_402_ttl(providers, fake_runtime):
+    fake_runtime(ttl_402=120)
+
+    from agent.credential_pool import _exhausted_ttl as runtime_ttl
+
+    status_at = datetime(2026, 8, 2, 12, 0, 0, tzinfo=timezone.utc)
+    entry = _entry(error_code="402", status_at=status_at.timestamp())
+    exhausted_until = providers._entry_pool_exhausted_until(entry)
+    assert exhausted_until is not None
+    assert exhausted_until == status_at + timedelta(seconds=runtime_ttl(402))
+
+
+def test_last_error_reset_at_precedence(providers):
+    """A present last_error_reset_at wins over status_at + ttl."""
+    status_at = datetime(2026, 8, 2, 12, 0, 0, tzinfo=timezone.utc)
+    reset_at = status_at + timedelta(minutes=30)
+    entry = _entry(error_code="402", status_at=status_at.timestamp(), reset_at=reset_at.timestamp())
+    assert providers._entry_pool_exhausted_until(entry) == reset_at
+
+
+def test_embedded_subprocess_helper_402_boundary_119_120(monkeypatch):
+    """The helper copy embedded in _ACCOUNT_USAGE_SUBPROCESS_CODE must behave
+    identically: TTL 120 for int/string 402 and 119/120s eligibility boundary."""
+    import api.providers as providers
+
+    agent_mod = types.ModuleType("agent")
+    agent_mod.__path__ = []
+    account_usage_mod = types.ModuleType("agent.account_usage")
+    credential_pool_mod = types.ModuleType("agent.credential_pool")
+
+    def fake_fetch_account_usage(provider, *, base_url=None, api_key=None):
+        return None
+
+    class FakePool:
+        def entries(self):
+            return []
+
+    credential_pool_mod._exhausted_ttl = lambda error_code: 120
+    credential_pool_mod.load_pool = lambda provider: FakePool()
+    account_usage_mod.fetch_account_usage = fake_fetch_account_usage
+    monkeypatch.setitem(sys.modules, "agent", agent_mod)
+    monkeypatch.setitem(sys.modules, "agent.account_usage", account_usage_mod)
+    monkeypatch.setitem(sys.modules, "agent.credential_pool", credential_pool_mod)
+    monkeypatch.setattr(sys, "argv", ["quota-probe", "openai-codex", ""])
+
+    namespace = {"__name__": "__main__"}
+    exec(providers._ACCOUNT_USAGE_SUBPROCESS_CODE, namespace)
+
+    embedded_ttl = namespace["_entry_exhausted_ttl_seconds"]
+    embedded_is_pool_exhausted = namespace["_entry_is_pool_exhausted"]
+    embedded_pool_exhausted_until = namespace["_entry_pool_exhausted_until"]
+
+    assert embedded_ttl(402) == 120
+    assert embedded_ttl("402") == 120
+
+    now = datetime.now(timezone.utc)
+    entry_119 = _entry(error_code="402", status_at=(now - timedelta(seconds=119)).timestamp())
+    entry_120 = _entry(error_code="402", status_at=(now - timedelta(seconds=120)).timestamp())
+    assert embedded_is_pool_exhausted(entry_119) is True
+    assert embedded_is_pool_exhausted(entry_120) is False
+
+    status_at = datetime(2026, 8, 2, 12, 0, 0, tzinfo=timezone.utc)
+    entry = _entry(error_code="402", status_at=status_at.timestamp())
+    assert embedded_pool_exhausted_until(entry) == status_at + timedelta(seconds=120)