From 153207eaad9dc83ca5e1a20ffc2a67e8afeb8e79 Mon Sep 17 00:00:00 2001 From: Frank Song Date: Thu, 9 Jul 2026 10:24:49 +0800 Subject: [PATCH] fix(wakeup): allow distinct config key recovery --- api/providers.py | 45 ++++++ api/routes.py | 12 +- tests/test_issue3929_process_wakeup_pause.py | 138 ++++++++++++++++--- 3 files changed, 168 insertions(+), 27 deletions(-) diff --git a/api/providers.py b/api/providers.py index d1191076..bfb48a59 100644 --- a/api/providers.py +++ b/api/providers.py @@ -1351,6 +1351,51 @@ def provider_has_usable_pool_credential(provider_id: str, *, refresh: bool = Fal return False +def _credential_secret_fingerprint(secret: str) -> str: + value = str(secret or "").strip() + if not value: + return "" + return hashlib.sha256(value.encode("utf-8", "ignore")).hexdigest() + + +def _entry_secret_fingerprint(entry: dict) -> str: + value = str(entry.get("secret_fingerprint") or "").strip().lower() + if value.startswith("sha256:"): + value = value[len("sha256:"):] + return value + + +def _pool_entry_currently_unusable(entry: dict) -> bool: + status = str(entry.get("last_status") or "").strip().lower() + if status == "dead": + return True + if status == "exhausted": + ns = SimpleNamespace(**entry) + return _entry_is_pool_exhausted(ns) + return False + + +def provider_has_process_wakeup_recovery_credential(provider_id: str, *, refresh: bool = False) -> bool: + """Return True when a paused credential-pool wakeup lane can safely retry.""" + provider = str(provider_id or "").strip().lower() + if not provider: + return False + if provider_has_usable_pool_credential(provider, refresh=refresh): + return True + configured_key = _get_provider_api_key(provider) + if not configured_key: + return False + configured_fingerprint = _credential_secret_fingerprint(configured_key) + if not configured_fingerprint: + return False + for entry in _pool_entry_payloads(provider): + if not _pool_entry_currently_unusable(entry): + continue + if _entry_secret_fingerprint(entry) == configured_fingerprint: + return False + return True + + def _active_provider_id() -> str | None: cfg = get_config() model_cfg = cfg.get("model", {}) diff --git a/api/routes.py b/api/routes.py index b1fb2191..a28bab2f 100644 --- a/api/routes.py +++ b/api/routes.py @@ -8919,7 +8919,7 @@ from api.providers import ( get_providers, get_provider_quota, get_provider_cost_history, - provider_has_usable_pool_credential, + provider_has_process_wakeup_recovery_credential, set_provider_key, remove_provider_key, ) @@ -19891,14 +19891,14 @@ def _process_wakeup_revalidation_provider(model, provider) -> str: return str(candidate or "").strip() -def _process_wakeup_provider_has_usable_pool_credential( +def _process_wakeup_provider_has_recovery_credential( session, *, model, provider, provider_id: str | None = None, ) -> bool: - """Check paused credential-pool lane recovery in the owning session profile.""" + """Check paused credential-pool recovery in the owning session profile.""" provider_id = str( provider_id or _process_wakeup_revalidation_provider(model, provider) or "" ).strip() @@ -19911,8 +19911,8 @@ def _process_wakeup_provider_has_usable_pool_credential( "process_wakeup credential revalidation", logger_override=logger, ): - return provider_has_usable_pool_credential(provider_id, refresh=True) - return provider_has_usable_pool_credential(provider_id, refresh=True) + return provider_has_process_wakeup_recovery_credential(provider_id, refresh=True) + return provider_has_process_wakeup_recovery_credential(provider_id, refresh=True) def _refresh_process_wakeup_pause_credential_fingerprint(session) -> bool: @@ -20030,7 +20030,7 @@ def start_session_turn( model_provider, ) try: - _credential_recovered = _process_wakeup_provider_has_usable_pool_credential( + _credential_recovered = _process_wakeup_provider_has_recovery_credential( s, model=model, provider=model_provider, diff --git a/tests/test_issue3929_process_wakeup_pause.py b/tests/test_issue3929_process_wakeup_pause.py index e7ea1f1a..1f2bde4d 100644 --- a/tests/test_issue3929_process_wakeup_pause.py +++ b/tests/test_issue3929_process_wakeup_pause.py @@ -5,6 +5,7 @@ from __future__ import annotations import json from pathlib import Path import queue +import hashlib import sys import types from unittest import mock @@ -66,7 +67,11 @@ def _isolate_agent_locks(): @pytest.fixture(autouse=True) def _default_live_credential_revalidation(monkeypatch): - monkeypatch.setattr(routes, "provider_has_usable_pool_credential", lambda *_args, **_kwargs: False) + monkeypatch.setattr( + routes, + "provider_has_process_wakeup_recovery_credential", + lambda *_args, **_kwargs: False, + ) @pytest.fixture(autouse=True) @@ -340,7 +345,7 @@ def test_process_wakeup_pause_revalidates_when_credential_state_changes(tmp_path monkeypatch.setattr(routes, "_start_run", _fake_start_run) monkeypatch.setattr( routes, - "provider_has_usable_pool_credential", + "provider_has_process_wakeup_recovery_credential", lambda provider_id, *, refresh=False: provider_id == "test-provider", ) @@ -403,7 +408,11 @@ def test_process_wakeup_pause_keeps_changed_credential_state_until_provider_is_u lambda *_args, **_kwargs: ("test-model", "test-provider", False), ) monkeypatch.setattr(routes, "_start_run", _unexpected_start_run) - monkeypatch.setattr(routes, "provider_has_usable_pool_credential", lambda *_args, **_kwargs: False) + monkeypatch.setattr( + routes, + "provider_has_process_wakeup_recovery_credential", + lambda *_args, **_kwargs: False, + ) response = routes.start_session_turn( session.session_id, @@ -419,9 +428,9 @@ def test_process_wakeup_pause_keeps_changed_credential_state_until_provider_is_u assert saved.process_wakeup_pause["credential_state_fingerprint"] == changed_fingerprint -def test_process_wakeup_pause_requires_pool_recovery_not_generic_provider_key(tmp_path, monkeypatch): +def test_process_wakeup_pause_keeps_pause_when_config_key_matches_exhausted_pool_secret(tmp_path, monkeypatch): session = Session( - session_id="wakeup_pause_generic_key_not_pool_recovery", + session_id="wakeup_pause_config_key_matches_exhausted_pool", workspace=str(tmp_path), model="test-model", model_provider="test-provider", @@ -435,9 +444,24 @@ def test_process_wakeup_pause_requires_pool_recovery_not_generic_provider_key(tm assert pause is not None session.save() models.SESSIONS[session.session_id] = session + exhausted_fingerprint = hashlib.sha256(b"same-key").hexdigest() + monkeypatch.setattr(providers, "_get_provider_api_key", lambda _provider: "same-key") + monkeypatch.setattr( + providers, + "_pool_entry_payloads", + lambda _provider: [{ + "last_status": "dead", + "secret_fingerprint": exhausted_fingerprint, + }], + ) + monkeypatch.setattr( + routes, + "provider_has_process_wakeup_recovery_credential", + providers.provider_has_process_wakeup_recovery_credential, + ) def _unexpected_start_run(*_args, **_kwargs): - raise AssertionError("generic provider credentials must not prove pool-lane recovery") + raise AssertionError("same exhausted pool secret must not prove recovery") monkeypatch.setattr(routes, "_resolve_chat_workspace_with_recovery", lambda _s, _w: str(tmp_path)) monkeypatch.setattr(routes, "_read_profile_model_config", lambda _s, _p: (None, None, {})) @@ -447,17 +471,10 @@ def test_process_wakeup_pause_requires_pool_recovery_not_generic_provider_key(tm lambda *_args, **_kwargs: ("test-model", "test-provider", False), ) monkeypatch.setattr(routes, "_start_run", _unexpected_start_run) - monkeypatch.setattr( - routes, - "provider_has_usable_credential", - lambda *_args, **_kwargs: True, - raising=False, - ) - monkeypatch.setattr(routes, "provider_has_usable_pool_credential", lambda *_args, **_kwargs: False) response = routes.start_session_turn( session.session_id, - "[IMPORTANT: Background process completed with generic key only.]", + "[IMPORTANT: Background process completed with same exhausted config key.]", source="process_wakeup", ) @@ -468,6 +485,73 @@ def test_process_wakeup_pause_requires_pool_recovery_not_generic_provider_key(tm assert saved.process_wakeup_pause["suppressed_count"] == 1 +def test_process_wakeup_pause_clears_when_config_key_differs_from_exhausted_pool_secret(tmp_path, monkeypatch): + session = Session( + session_id="wakeup_pause_config_key_differs_from_exhausted_pool", + workspace=str(tmp_path), + model="test-model", + model_provider="test-provider", + ) + pause = models.record_process_wakeup_provider_unavailable_pause( + session, + classification="credential_pool_empty", + model="test-model", + provider="test-provider", + ) + assert pause is not None + session.save() + models.SESSIONS[session.session_id] = session + exhausted_fingerprint = hashlib.sha256(b"old-pool-key").hexdigest() + monkeypatch.setattr(providers, "_get_provider_api_key", lambda _provider: "new-config-key") + monkeypatch.setattr( + providers, + "_pool_entry_payloads", + lambda _provider: [{ + "last_status": "dead", + "secret_fingerprint": exhausted_fingerprint, + }], + ) + monkeypatch.setattr( + routes, + "provider_has_process_wakeup_recovery_credential", + providers.provider_has_process_wakeup_recovery_credential, + ) + + captured = {} + + def _fake_start_run(s, **kwargs): + captured["source"] = kwargs.get("source") + captured["model"] = kwargs.get("model") + captured["model_provider"] = kwargs.get("model_provider") + return {"stream_id": "stream-config-key-recovered", "session_id": s.session_id, "_status": 200} + + monkeypatch.setattr(routes, "_resolve_chat_workspace_with_recovery", lambda _s, _w: str(tmp_path)) + monkeypatch.setattr(routes, "_read_profile_model_config", lambda _s, _p: (None, None, {})) + monkeypatch.setattr( + routes, + "_resolve_compatible_session_model_state", + lambda *_args, **_kwargs: ("test-model", "test-provider", False), + ) + monkeypatch.setattr(routes, "_start_run", _fake_start_run) + + response = routes.start_session_turn( + session.session_id, + "[IMPORTANT: Background process completed with new config key.]", + source="process_wakeup", + ) + + assert response["_status"] == 200 + assert response["stream_id"] == "stream-config-key-recovered" + assert captured == { + "source": "process_wakeup", + "model": "test-model", + "model_provider": "test-provider", + } + saved = Session.load(session.session_id) + assert saved is not None + assert saved.process_wakeup_pause == {} + + def test_process_wakeup_pause_survives_rotation_style_auth_rewrite(tmp_path, monkeypatch): hermes_home = tmp_path / "hermes-home" hermes_home.mkdir() @@ -584,8 +668,8 @@ def test_process_wakeup_pause_revalidates_status_recovery_without_fingerprint_ch monkeypatch.setattr(profiles, "get_active_hermes_home", lambda: hermes_home) monkeypatch.setattr( routes, - "provider_has_usable_pool_credential", - providers.provider_has_usable_pool_credential, + "provider_has_process_wakeup_recovery_credential", + providers.provider_has_process_wakeup_recovery_credential, ) session = Session( session_id="wakeup_pause_status_recovery", @@ -666,7 +750,7 @@ def test_process_wakeup_pause_revalidates_at_provider_model_with_canonical_provi calls = [] - def _provider_has_usable_pool_credential(provider_id, *, refresh=False): + def _provider_has_process_wakeup_recovery_credential(provider_id, *, refresh=False): calls.append((provider_id, refresh)) return provider_id == "test-provider" @@ -684,7 +768,11 @@ def test_process_wakeup_pause_revalidates_at_provider_model_with_canonical_provi model="@test-provider:test-model", provider=None, ) - monkeypatch.setattr(routes, "provider_has_usable_pool_credential", _provider_has_usable_pool_credential) + monkeypatch.setattr( + routes, + "provider_has_process_wakeup_recovery_credential", + _provider_has_process_wakeup_recovery_credential, + ) monkeypatch.setattr(routes, "_start_run", _fake_start_run) response = routes.start_session_turn( @@ -740,7 +828,7 @@ def test_process_wakeup_pause_revalidation_uses_session_profile_not_default(tmp_ calls = [] - def _provider_has_usable_pool_credential(provider_id, *, refresh=False): + def _provider_has_process_wakeup_recovery_credential(provider_id, *, refresh=False): active_profile = profiles.get_active_profile_name() calls.append((provider_id, refresh, active_profile)) return active_profile == "default" @@ -754,7 +842,11 @@ def test_process_wakeup_pause_revalidation_uses_session_profile_not_default(tmp_ model="test-model", provider="test-provider", ) - monkeypatch.setattr(routes, "provider_has_usable_pool_credential", _provider_has_usable_pool_credential) + monkeypatch.setattr( + routes, + "provider_has_process_wakeup_recovery_credential", + _provider_has_process_wakeup_recovery_credential, + ) monkeypatch.setattr(routes, "_start_run", _unexpected_start_run) try: @@ -1192,7 +1284,11 @@ def test_process_wakeup_pause_empty_provider_lane_probes_after_fingerprint_chang model="claude-sonnet-test", provider=None, ) - monkeypatch.setattr(routes, "provider_has_usable_pool_credential", lambda *_args, **_kwargs: False) + monkeypatch.setattr( + routes, + "provider_has_process_wakeup_recovery_credential", + lambda *_args, **_kwargs: False, + ) monkeypatch.setattr(routes, "_start_run", _fake_start_run) response = routes.start_session_turn(