fix(#3929 partial): classify credential-pool exhaustion with an actionable hint

'All 0 credential(s) exhausted for <provider>' did not match _is_quota_error_text
so it fell through to a generic 'Error' with no hint (the exception path's else
branch discarded the hint entirely). Adds a distinct credential_pool_empty
classification (ordered before the quota check) + an explicit exception-path
branch + an actionable pool-specific hint, so the user sees why the turn failed
and what to do. Does not over-match (doesn't swallow real quota errors) and
doesn't block partial-harvest; downstream tolerates the new type.

NOTE: this is the CLASSIFICATION half of #3929 only — the data-loss half
(reasoning/partial-work restore on reload-reconcile) is a scoped follow-up that
needs the reporter's server logs.

Self-built (nesquena-hermes); nesquena independent review APPROVED.
This commit is contained in:
nesquena-hermes committed 2026-07-05 00:35:26 +00:00
1 parent 94e5778feb
commit 93f447c7af
2 files changed
+76 -1

No files matched your search

+24 -1
View File
@@ -1195,8 +1195,20 @@ def _classify_provider_error(err_str: str, exc=None, *, silent_failure: bool = F
'hint': 'The run stopped before a provider response completed. If you did not cancel it, try again.',
}
_is_quota = _is_quota_error_text(err_str)
# A credential-POOL exhaustion ("All 0 credential(s) exhausted for <provider>")
# is a distinct shape from account/plan quota: it means the profile's
# credential pool has no usable keys for that provider (a config problem),
# not that a funded account ran out of credits. It is NOT matched by
# _is_quota_error_text ('credential(s) exhausted' != 'credits exhausted'), so
# without this it fell through to the generic error label/hint. Classify it
# explicitly so the user gets a pool-specific, actionable hint. (#3929)
_is_credential_pool_empty = (
'credential(s) exhausted' in _err_lower
or 'credentials exhausted' in _err_lower
or ('credential' in _err_lower and 'exhausted' in _err_lower)
)
_is_auth = (
not _is_quota and (
not _is_quota and not _is_credential_pool_empty and (
_probe_status_code == 401
or
'401' in err_str
@@ -1228,6 +1240,12 @@ def _classify_provider_error(err_str: str, exc=None, *, silent_failure: bool = F
or ('context length exceeded' in _err_lower and 'cannot compress further' in _err_lower)
or ('context compression' in _err_lower and 'max compression attempts' in _err_lower)
)
if _is_credential_pool_empty:
return {
'label': 'No usable credentials',
'type': 'credential_pool_empty',
'hint': 'The credential pool for this provider has no usable keys left (all entries exhausted or unconfigured). Add or refresh a key for this provider in your Hermes config / credential pool, or switch providers via `hermes model`.',
}
if _is_quota:
return {
'label': 'Out of credits',
@@ -9529,6 +9547,7 @@ def _run_agent_streaming(
put('cancel', _cancel_event_payload('Cancelled by user'))
return
_exc_is_quota = _classification['type'] == 'quota_exhausted'
_exc_is_credential_pool_empty = _classification['type'] == 'credential_pool_empty'
# Exception quota text still includes: 'more credits' in _exc_lower, 'can only afford' in _exc_lower, 'fewer max_tokens' in _exc_lower.
# Rate-limit detection remains guarded as: (not _exc_is_quota).
_exc_is_rate_limit = (_classification['type'] == 'rate_limit') and (not _exc_is_quota)
@@ -9542,6 +9561,10 @@ def _run_agent_streaming(
_exc_label, _exc_type, _exc_hint = (
_classification['label'], _classification['type'], _classification['hint'],
)
elif _exc_is_credential_pool_empty:
_exc_label, _exc_type, _exc_hint = (
_classification['label'], _classification['type'], _classification['hint'],
)
elif _exc_is_rate_limit:
_exc_label, _exc_type, _exc_hint = (
_classification['label'], _classification['type'], _classification['hint'],
@@ -0,0 +1,52 @@
"""Regression test for #3929 — credential-pool exhaustion classification.
Opus-advise on #3929 flagged that the error text a zero-credential pool produces,
`"All 0 credential(s) exhausted for opencode_go"`, does NOT match
_is_quota_error_text ('credential(s) exhausted' != 'credits exhausted'), so it
fell through to the generic 'Error' label with an empty hint — and in the
exception path the else-branch discarded the classification hint entirely.
This does not by itself fix the partial-work loss (that needs the reload-reconcile
reasoning-restore work + reporter logs for the incident autopsy — tracked
separately on the issue), but it gives the user an accurate, actionable message
for the exact error they hit instead of a bare 'Error'.
"""
from api import streaming
CREDENTIAL_POOL_ERRORS = [
"All 0 credential(s) exhausted for opencode_go",
"All 3 credential(s) exhausted for openrouter",
"credentials exhausted",
]
def test_credential_pool_exhaustion_classified_distinctly():
"""The 'N credential(s) exhausted' pool shape gets its own classification —
not generic 'error', not mislabeled as account quota."""
for err in CREDENTIAL_POOL_ERRORS:
classified = streaming._classify_provider_error(err, Exception(err))
assert classified["type"] == "credential_pool_empty", err
# Must carry an actionable, pool-specific hint (not empty, not top-up-account).
assert classified["hint"], f"empty hint for {err!r}"
assert "credential pool" in classified["hint"].lower(), err
assert classified["label"] == "No usable credentials", err
def test_credential_pool_is_not_misclassified_as_account_quota():
"""A zero-credential POOL is a config problem, not account credit exhaustion:
it must NOT get the 'Out of credits / top up your account' quota hint."""
classified = streaming._classify_provider_error(
"All 0 credential(s) exhausted for opencode_go",
Exception("All 0 credential(s) exhausted for opencode_go"),
)
assert classified["type"] != "quota_exhausted"
assert "top up" not in classified["hint"].lower()
def test_real_account_quota_still_classifies_as_quota():
"""Guard against over-matching: genuine account-credit exhaustion phrases
must still classify as quota_exhausted, not credential_pool_empty."""
for err in ("insufficient credits", "credits exhausted", "you have exceeded your current quota"):
classified = streaming._classify_provider_error(err, Exception(err))
assert classified["type"] == "quota_exhausted", err