fix(#3929 partial): classify credential-pool exhaustion with an actionable hint
'All 0 credential(s) exhausted for <provider>' did not match _is_quota_error_text so it fell through to a generic 'Error' with no hint (the exception path's else branch discarded the hint entirely). Adds a distinct credential_pool_empty classification (ordered before the quota check) + an explicit exception-path branch + an actionable pool-specific hint, so the user sees why the turn failed and what to do. Does not over-match (doesn't swallow real quota errors) and doesn't block partial-harvest; downstream tolerates the new type. NOTE: this is the CLASSIFICATION half of #3929 only — the data-loss half (reasoning/partial-work restore on reload-reconcile) is a scoped follow-up that needs the reporter's server logs. Self-built (nesquena-hermes); nesquena independent review APPROVED.
This commit is contained in:
1 parent
94e5778feb
commit
93f447c7af
2 files changed
+76
-1
No files matched your search
+24
-1
@@ -1195,8 +1195,20 @@ def _classify_provider_error(err_str: str, exc=None, *, silent_failure: bool = F
|
||||
'hint': 'The run stopped before a provider response completed. If you did not cancel it, try again.',
|
||||
}
|
||||
_is_quota = _is_quota_error_text(err_str)
|
||||
# A credential-POOL exhaustion ("All 0 credential(s) exhausted for <provider>")
|
||||
# is a distinct shape from account/plan quota: it means the profile's
|
||||
# credential pool has no usable keys for that provider (a config problem),
|
||||
# not that a funded account ran out of credits. It is NOT matched by
|
||||
# _is_quota_error_text ('credential(s) exhausted' != 'credits exhausted'), so
|
||||
# without this it fell through to the generic error label/hint. Classify it
|
||||
# explicitly so the user gets a pool-specific, actionable hint. (#3929)
|
||||
_is_credential_pool_empty = (
|
||||
'credential(s) exhausted' in _err_lower
|
||||
or 'credentials exhausted' in _err_lower
|
||||
or ('credential' in _err_lower and 'exhausted' in _err_lower)
|
||||
)
|
||||
_is_auth = (
|
||||
not _is_quota and (
|
||||
not _is_quota and not _is_credential_pool_empty and (
|
||||
_probe_status_code == 401
|
||||
or
|
||||
'401' in err_str
|
||||
@@ -1228,6 +1240,12 @@ def _classify_provider_error(err_str: str, exc=None, *, silent_failure: bool = F
|
||||
or ('context length exceeded' in _err_lower and 'cannot compress further' in _err_lower)
|
||||
or ('context compression' in _err_lower and 'max compression attempts' in _err_lower)
|
||||
)
|
||||
if _is_credential_pool_empty:
|
||||
return {
|
||||
'label': 'No usable credentials',
|
||||
'type': 'credential_pool_empty',
|
||||
'hint': 'The credential pool for this provider has no usable keys left (all entries exhausted or unconfigured). Add or refresh a key for this provider in your Hermes config / credential pool, or switch providers via `hermes model`.',
|
||||
}
|
||||
if _is_quota:
|
||||
return {
|
||||
'label': 'Out of credits',
|
||||
@@ -9529,6 +9547,7 @@ def _run_agent_streaming(
|
||||
put('cancel', _cancel_event_payload('Cancelled by user'))
|
||||
return
|
||||
_exc_is_quota = _classification['type'] == 'quota_exhausted'
|
||||
_exc_is_credential_pool_empty = _classification['type'] == 'credential_pool_empty'
|
||||
# Exception quota text still includes: 'more credits' in _exc_lower, 'can only afford' in _exc_lower, 'fewer max_tokens' in _exc_lower.
|
||||
# Rate-limit detection remains guarded as: (not _exc_is_quota).
|
||||
_exc_is_rate_limit = (_classification['type'] == 'rate_limit') and (not _exc_is_quota)
|
||||
@@ -9542,6 +9561,10 @@ def _run_agent_streaming(
|
||||
_exc_label, _exc_type, _exc_hint = (
|
||||
_classification['label'], _classification['type'], _classification['hint'],
|
||||
)
|
||||
elif _exc_is_credential_pool_empty:
|
||||
_exc_label, _exc_type, _exc_hint = (
|
||||
_classification['label'], _classification['type'], _classification['hint'],
|
||||
)
|
||||
elif _exc_is_rate_limit:
|
||||
_exc_label, _exc_type, _exc_hint = (
|
||||
_classification['label'], _classification['type'], _classification['hint'],
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Regression test for #3929 — credential-pool exhaustion classification.
|
||||
|
||||
Opus-advise on #3929 flagged that the error text a zero-credential pool produces,
|
||||
`"All 0 credential(s) exhausted for opencode_go"`, does NOT match
|
||||
_is_quota_error_text ('credential(s) exhausted' != 'credits exhausted'), so it
|
||||
fell through to the generic 'Error' label with an empty hint — and in the
|
||||
exception path the else-branch discarded the classification hint entirely.
|
||||
|
||||
This does not by itself fix the partial-work loss (that needs the reload-reconcile
|
||||
reasoning-restore work + reporter logs for the incident autopsy — tracked
|
||||
separately on the issue), but it gives the user an accurate, actionable message
|
||||
for the exact error they hit instead of a bare 'Error'.
|
||||
"""
|
||||
from api import streaming
|
||||
|
||||
|
||||
CREDENTIAL_POOL_ERRORS = [
|
||||
"All 0 credential(s) exhausted for opencode_go",
|
||||
"All 3 credential(s) exhausted for openrouter",
|
||||
"credentials exhausted",
|
||||
]
|
||||
|
||||
|
||||
def test_credential_pool_exhaustion_classified_distinctly():
|
||||
"""The 'N credential(s) exhausted' pool shape gets its own classification —
|
||||
not generic 'error', not mislabeled as account quota."""
|
||||
for err in CREDENTIAL_POOL_ERRORS:
|
||||
classified = streaming._classify_provider_error(err, Exception(err))
|
||||
assert classified["type"] == "credential_pool_empty", err
|
||||
# Must carry an actionable, pool-specific hint (not empty, not top-up-account).
|
||||
assert classified["hint"], f"empty hint for {err!r}"
|
||||
assert "credential pool" in classified["hint"].lower(), err
|
||||
assert classified["label"] == "No usable credentials", err
|
||||
|
||||
|
||||
def test_credential_pool_is_not_misclassified_as_account_quota():
|
||||
"""A zero-credential POOL is a config problem, not account credit exhaustion:
|
||||
it must NOT get the 'Out of credits / top up your account' quota hint."""
|
||||
classified = streaming._classify_provider_error(
|
||||
"All 0 credential(s) exhausted for opencode_go",
|
||||
Exception("All 0 credential(s) exhausted for opencode_go"),
|
||||
)
|
||||
assert classified["type"] != "quota_exhausted"
|
||||
assert "top up" not in classified["hint"].lower()
|
||||
|
||||
|
||||
def test_real_account_quota_still_classifies_as_quota():
|
||||
"""Guard against over-matching: genuine account-credit exhaustion phrases
|
||||
must still classify as quota_exhausted, not credential_pool_empty."""
|
||||
for err in ("insufficient credits", "credits exhausted", "you have exceeded your current quota"):
|
||||
classified = streaming._classify_provider_error(err, Exception(err))
|
||||
assert classified["type"] == "quota_exhausted", err
|
||||
Reference in new issue
Block a user