From 93f447c7af49d3c1d3da466cb9ca645772126c13 Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Sun, 5 Jul 2026 00:13:23 +0000 Subject: [PATCH] fix(#3929 partial): classify credential-pool exhaustion with an actionable hint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'All 0 credential(s) exhausted for ' did not match _is_quota_error_text so it fell through to a generic 'Error' with no hint (the exception path's else branch discarded the hint entirely). Adds a distinct credential_pool_empty classification (ordered before the quota check) + an explicit exception-path branch + an actionable pool-specific hint, so the user sees why the turn failed and what to do. Does not over-match (doesn't swallow real quota errors) and doesn't block partial-harvest; downstream tolerates the new type. NOTE: this is the CLASSIFICATION half of #3929 only — the data-loss half (reasoning/partial-work restore on reload-reconcile) is a scoped follow-up that needs the reporter's server logs. Self-built (nesquena-hermes); nesquena independent review APPROVED. --- api/streaming.py | 25 ++++++++- ...ssue3929_credential_pool_classification.py | 52 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 tests/test_issue3929_credential_pool_classification.py diff --git a/api/streaming.py b/api/streaming.py index 2903e80f..697e258b 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -1195,8 +1195,20 @@ def _classify_provider_error(err_str: str, exc=None, *, silent_failure: bool = F 'hint': 'The run stopped before a provider response completed. If you did not cancel it, try again.', } _is_quota = _is_quota_error_text(err_str) + # A credential-POOL exhaustion ("All 0 credential(s) exhausted for ") + # is a distinct shape from account/plan quota: it means the profile's + # credential pool has no usable keys for that provider (a config problem), + # not that a funded account ran out of credits. It is NOT matched by + # _is_quota_error_text ('credential(s) exhausted' != 'credits exhausted'), so + # without this it fell through to the generic error label/hint. Classify it + # explicitly so the user gets a pool-specific, actionable hint. (#3929) + _is_credential_pool_empty = ( + 'credential(s) exhausted' in _err_lower + or 'credentials exhausted' in _err_lower + or ('credential' in _err_lower and 'exhausted' in _err_lower) + ) _is_auth = ( - not _is_quota and ( + not _is_quota and not _is_credential_pool_empty and ( _probe_status_code == 401 or '401' in err_str @@ -1228,6 +1240,12 @@ def _classify_provider_error(err_str: str, exc=None, *, silent_failure: bool = F or ('context length exceeded' in _err_lower and 'cannot compress further' in _err_lower) or ('context compression' in _err_lower and 'max compression attempts' in _err_lower) ) + if _is_credential_pool_empty: + return { + 'label': 'No usable credentials', + 'type': 'credential_pool_empty', + 'hint': 'The credential pool for this provider has no usable keys left (all entries exhausted or unconfigured). Add or refresh a key for this provider in your Hermes config / credential pool, or switch providers via `hermes model`.', + } if _is_quota: return { 'label': 'Out of credits', @@ -9529,6 +9547,7 @@ def _run_agent_streaming( put('cancel', _cancel_event_payload('Cancelled by user')) return _exc_is_quota = _classification['type'] == 'quota_exhausted' + _exc_is_credential_pool_empty = _classification['type'] == 'credential_pool_empty' # Exception quota text still includes: 'more credits' in _exc_lower, 'can only afford' in _exc_lower, 'fewer max_tokens' in _exc_lower. # Rate-limit detection remains guarded as: (not _exc_is_quota). _exc_is_rate_limit = (_classification['type'] == 'rate_limit') and (not _exc_is_quota) @@ -9542,6 +9561,10 @@ def _run_agent_streaming( _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], ) + elif _exc_is_credential_pool_empty: + _exc_label, _exc_type, _exc_hint = ( + _classification['label'], _classification['type'], _classification['hint'], + ) elif _exc_is_rate_limit: _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], diff --git a/tests/test_issue3929_credential_pool_classification.py b/tests/test_issue3929_credential_pool_classification.py new file mode 100644 index 00000000..dfd746ca --- /dev/null +++ b/tests/test_issue3929_credential_pool_classification.py @@ -0,0 +1,52 @@ +"""Regression test for #3929 — credential-pool exhaustion classification. + +Opus-advise on #3929 flagged that the error text a zero-credential pool produces, +`"All 0 credential(s) exhausted for opencode_go"`, does NOT match +_is_quota_error_text ('credential(s) exhausted' != 'credits exhausted'), so it +fell through to the generic 'Error' label with an empty hint — and in the +exception path the else-branch discarded the classification hint entirely. + +This does not by itself fix the partial-work loss (that needs the reload-reconcile +reasoning-restore work + reporter logs for the incident autopsy — tracked +separately on the issue), but it gives the user an accurate, actionable message +for the exact error they hit instead of a bare 'Error'. +""" +from api import streaming + + +CREDENTIAL_POOL_ERRORS = [ + "All 0 credential(s) exhausted for opencode_go", + "All 3 credential(s) exhausted for openrouter", + "credentials exhausted", +] + + +def test_credential_pool_exhaustion_classified_distinctly(): + """The 'N credential(s) exhausted' pool shape gets its own classification — + not generic 'error', not mislabeled as account quota.""" + for err in CREDENTIAL_POOL_ERRORS: + classified = streaming._classify_provider_error(err, Exception(err)) + assert classified["type"] == "credential_pool_empty", err + # Must carry an actionable, pool-specific hint (not empty, not top-up-account). + assert classified["hint"], f"empty hint for {err!r}" + assert "credential pool" in classified["hint"].lower(), err + assert classified["label"] == "No usable credentials", err + + +def test_credential_pool_is_not_misclassified_as_account_quota(): + """A zero-credential POOL is a config problem, not account credit exhaustion: + it must NOT get the 'Out of credits / top up your account' quota hint.""" + classified = streaming._classify_provider_error( + "All 0 credential(s) exhausted for opencode_go", + Exception("All 0 credential(s) exhausted for opencode_go"), + ) + assert classified["type"] != "quota_exhausted" + assert "top up" not in classified["hint"].lower() + + +def test_real_account_quota_still_classifies_as_quota(): + """Guard against over-matching: genuine account-credit exhaustion phrases + must still classify as quota_exhausted, not credential_pool_empty.""" + for err in ("insufficient credits", "credits exhausted", "you have exceeded your current quota"): + classified = streaming._classify_provider_error(err, Exception(err)) + assert classified["type"] == "quota_exhausted", err