fix: isolate profile quota usage probes
This commit is contained in:
1 parent
773857d159
commit
6808e06083
2 files changed
+219
-15
No files matched your search
+141
-9
@@ -10,10 +10,13 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from api.config import (
|
||||
@@ -33,7 +36,53 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_OPENROUTER_KEY_URL = "https://openrouter.ai/api/v1/key"
|
||||
_PROVIDER_QUOTA_TIMEOUT_SECONDS = 3.0
|
||||
_ACCOUNT_USAGE_SUBPROCESS_TIMEOUT_SECONDS = 35.0
|
||||
_ACCOUNT_USAGE_PROVIDERS = frozenset({"openai-codex", "anthropic"})
|
||||
_ACCOUNT_USAGE_SUBPROCESS_CODE = r"""
|
||||
import json
|
||||
import sys
|
||||
|
||||
from agent.account_usage import fetch_account_usage
|
||||
|
||||
|
||||
def _iso(value):
|
||||
if value in (None, ""):
|
||||
return None
|
||||
if hasattr(value, "isoformat"):
|
||||
text = value.isoformat()
|
||||
return text.replace("+00:00", "Z")
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _snapshot_payload(snapshot):
|
||||
if snapshot is None:
|
||||
return None
|
||||
windows = []
|
||||
for window in getattr(snapshot, "windows", ()) or ():
|
||||
windows.append({
|
||||
"label": str(getattr(window, "label", "") or ""),
|
||||
"used_percent": getattr(window, "used_percent", None),
|
||||
"reset_at": _iso(getattr(window, "reset_at", None)),
|
||||
"detail": getattr(window, "detail", None),
|
||||
})
|
||||
return {
|
||||
"provider": str(getattr(snapshot, "provider", "") or ""),
|
||||
"source": str(getattr(snapshot, "source", "") or ""),
|
||||
"title": str(getattr(snapshot, "title", "") or ""),
|
||||
"plan": getattr(snapshot, "plan", None),
|
||||
"windows": windows,
|
||||
"details": list(getattr(snapshot, "details", ()) or ()),
|
||||
"available": bool(getattr(snapshot, "available", bool(windows))),
|
||||
"unavailable_reason": getattr(snapshot, "unavailable_reason", None),
|
||||
"fetched_at": _iso(getattr(snapshot, "fetched_at", None)),
|
||||
}
|
||||
|
||||
|
||||
provider = sys.argv[1]
|
||||
api_key = sys.argv[2] or None
|
||||
print(json.dumps(_snapshot_payload(fetch_account_usage(provider, api_key=api_key))))
|
||||
"""
|
||||
|
||||
# SECTION: Provider ↔ env var mapping
|
||||
|
||||
@@ -420,19 +469,102 @@ def _agent_fetch_account_usage(provider: str, *, base_url: str | None = None, ap
|
||||
return fetch_account_usage(provider, base_url=base_url, api_key=api_key)
|
||||
|
||||
|
||||
def _fetch_account_usage_with_profile_context(provider: str) -> Any:
|
||||
try:
|
||||
from api.profiles import cron_profile_context_for_home
|
||||
except ImportError:
|
||||
cron_profile_context_for_home = None
|
||||
def _account_usage_subprocess_env(home: Path, provider: str, api_key: str | None) -> dict[str, str]:
|
||||
env = dict(os.environ)
|
||||
env["HERMES_HOME"] = str(Path(home))
|
||||
|
||||
# Profile .env values should affect only the child quota probe, not the
|
||||
# WebUI process-global environment. This is especially important for
|
||||
# Anthropic account usage, where the agent resolver reads OAuth/API tokens
|
||||
# from environment variables.
|
||||
for key, value in _load_env_file(Path(home) / ".env").items():
|
||||
if value:
|
||||
env[key] = value
|
||||
|
||||
env_var = _PROVIDER_ENV_VAR.get((provider or "").strip().lower())
|
||||
if env_var and api_key:
|
||||
env[env_var] = api_key
|
||||
|
||||
try:
|
||||
from api.config import _AGENT_DIR
|
||||
except Exception:
|
||||
_AGENT_DIR = None
|
||||
pythonpath_parts: list[str] = []
|
||||
if _AGENT_DIR:
|
||||
pythonpath_parts.append(str(_AGENT_DIR))
|
||||
existing_pythonpath = env.get("PYTHONPATH", "")
|
||||
if existing_pythonpath:
|
||||
pythonpath_parts.append(existing_pythonpath)
|
||||
if pythonpath_parts:
|
||||
env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts)
|
||||
return env
|
||||
|
||||
|
||||
def _account_usage_payload_to_snapshot(payload: Any) -> Any:
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
windows = tuple(
|
||||
SimpleNamespace(
|
||||
label=window.get("label"),
|
||||
used_percent=window.get("used_percent"),
|
||||
reset_at=window.get("reset_at"),
|
||||
detail=window.get("detail"),
|
||||
)
|
||||
for window in (payload.get("windows") or ())
|
||||
if isinstance(window, dict)
|
||||
)
|
||||
return SimpleNamespace(
|
||||
provider=payload.get("provider"),
|
||||
source=payload.get("source"),
|
||||
title=payload.get("title"),
|
||||
plan=payload.get("plan"),
|
||||
windows=windows,
|
||||
details=tuple(payload.get("details") or ()),
|
||||
available=bool(payload.get("available")),
|
||||
unavailable_reason=payload.get("unavailable_reason"),
|
||||
fetched_at=payload.get("fetched_at"),
|
||||
)
|
||||
|
||||
|
||||
def _agent_fetch_account_usage_for_home(provider: str, home: Path, *, api_key: str | None = None) -> Any:
|
||||
try:
|
||||
from api.config import PYTHON_EXE
|
||||
except Exception:
|
||||
PYTHON_EXE = sys.executable or "python3"
|
||||
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[PYTHON_EXE, "-c", _ACCOUNT_USAGE_SUBPROCESS_CODE, provider, api_key or ""],
|
||||
env=_account_usage_subprocess_env(home, provider, api_key),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=_ACCOUNT_USAGE_SUBPROCESS_TIMEOUT_SECONDS,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.debug("Account usage probe for %s timed out", provider)
|
||||
return None
|
||||
except Exception:
|
||||
logger.debug("Account usage probe for %s failed to launch", provider, exc_info=True)
|
||||
return None
|
||||
|
||||
if proc.returncode != 0:
|
||||
logger.debug("Account usage probe for %s exited with status %s", provider, proc.returncode)
|
||||
return None
|
||||
try:
|
||||
payload = json.loads((proc.stdout or "").strip() or "null")
|
||||
except json.JSONDecodeError:
|
||||
logger.debug("Account usage probe for %s returned invalid JSON", provider)
|
||||
return None
|
||||
return _account_usage_payload_to_snapshot(payload)
|
||||
|
||||
|
||||
def _fetch_account_usage_with_profile_context(provider: str) -> Any:
|
||||
home = _get_hermes_home()
|
||||
api_key = _get_provider_api_key(provider)
|
||||
try:
|
||||
if cron_profile_context_for_home is None:
|
||||
return _agent_fetch_account_usage(provider, api_key=api_key)
|
||||
with cron_profile_context_for_home(home):
|
||||
return _agent_fetch_account_usage(provider, api_key=api_key)
|
||||
return _agent_fetch_account_usage_for_home(provider, home, api_key=api_key)
|
||||
except Exception:
|
||||
logger.debug("Failed to fetch account usage for %s", provider, exc_info=True)
|
||||
return None
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import inspect
|
||||
import os
|
||||
import threading
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
from io import BytesIO
|
||||
@@ -174,10 +176,10 @@ def test_codex_account_usage_is_fetched_under_active_profile_home(monkeypatch, t
|
||||
seen = {}
|
||||
previous_home = os.environ.get("HERMES_HOME")
|
||||
|
||||
def fake_fetch(provider, base_url=None, api_key=None):
|
||||
def fake_fetch(provider, home, api_key=None):
|
||||
seen["provider"] = provider
|
||||
seen["home"] = str(home)
|
||||
seen["api_key"] = api_key
|
||||
seen["hermes_home"] = os.environ.get("HERMES_HOME")
|
||||
return SimpleNamespace(
|
||||
provider="openai-codex",
|
||||
source="usage_api",
|
||||
@@ -203,7 +205,7 @@ def test_codex_account_usage_is_fetched_under_active_profile_home(monkeypatch, t
|
||||
unavailable_reason=None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(providers, "_agent_fetch_account_usage", fake_fetch)
|
||||
monkeypatch.setattr(providers, "_agent_fetch_account_usage_for_home", fake_fetch)
|
||||
try:
|
||||
result = providers.get_provider_quota()
|
||||
finally:
|
||||
@@ -211,8 +213,8 @@ def test_codex_account_usage_is_fetched_under_active_profile_home(monkeypatch, t
|
||||
|
||||
assert seen == {
|
||||
"provider": "openai-codex",
|
||||
"home": str(tmp_path),
|
||||
"api_key": None,
|
||||
"hermes_home": str(tmp_path),
|
||||
}
|
||||
assert os.environ.get("HERMES_HOME") == previous_home
|
||||
assert result["ok"] is True
|
||||
@@ -258,7 +260,7 @@ def test_codex_account_usage_unavailable_is_sanitized(monkeypatch, tmp_path):
|
||||
def fake_fetch(*_args, **_kwargs):
|
||||
raise RuntimeError("secret access token should not leak")
|
||||
|
||||
monkeypatch.setattr(providers, "_agent_fetch_account_usage", fake_fetch)
|
||||
monkeypatch.setattr(providers, "_agent_fetch_account_usage_for_home", fake_fetch)
|
||||
try:
|
||||
result = providers.get_provider_quota()
|
||||
finally:
|
||||
@@ -282,7 +284,7 @@ def test_anthropic_oauth_usage_unavailable_reason_is_reported(monkeypatch, tmp_p
|
||||
|
||||
monkeypatch.setattr(
|
||||
providers,
|
||||
"_agent_fetch_account_usage",
|
||||
"_agent_fetch_account_usage_for_home",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
provider="anthropic",
|
||||
source="oauth_usage_api",
|
||||
@@ -308,6 +310,76 @@ def test_anthropic_oauth_usage_unavailable_reason_is_reported(monkeypatch, tmp_p
|
||||
assert "OAuth-backed Claude accounts" in result["message"]
|
||||
|
||||
|
||||
def test_account_usage_profile_fetch_does_not_enter_cron_env_context():
|
||||
"""Quota probes must not reuse cron's process-global env/module swapper."""
|
||||
import api.providers as providers
|
||||
|
||||
body = inspect.getsource(providers._fetch_account_usage_with_profile_context)
|
||||
assert "cron_profile_context_for_home" not in body
|
||||
assert "_agent_fetch_account_usage_for_home" in body
|
||||
|
||||
|
||||
def test_account_usage_profile_env_is_child_scoped(monkeypatch, tmp_path):
|
||||
"""Profile .env values should be passed to the child probe only."""
|
||||
import api.providers as providers
|
||||
|
||||
home = tmp_path / "profile-a"
|
||||
home.mkdir()
|
||||
(home / ".env").write_text("ANTHROPIC_API_KEY=profile-key\n", encoding="utf-8")
|
||||
monkeypatch.setenv("ANTHROPIC_API_KEY", "process-key")
|
||||
|
||||
env = providers._account_usage_subprocess_env(home, "anthropic", None)
|
||||
|
||||
assert env["HERMES_HOME"] == str(home)
|
||||
assert env["ANTHROPIC_API_KEY"] == "profile-key"
|
||||
assert os.environ["ANTHROPIC_API_KEY"] == "process-key"
|
||||
|
||||
|
||||
def test_account_usage_profile_fetches_can_overlap_for_different_homes(monkeypatch, tmp_path):
|
||||
"""Different profile quota fetches should not serialize on cron's global lock."""
|
||||
import api.providers as providers
|
||||
|
||||
homes = {
|
||||
"quota-a": tmp_path / "a",
|
||||
"quota-b": tmp_path / "b",
|
||||
}
|
||||
for home in homes.values():
|
||||
home.mkdir()
|
||||
barrier = threading.Barrier(2, timeout=2)
|
||||
events = []
|
||||
errors = []
|
||||
|
||||
def fake_home():
|
||||
return homes[threading.current_thread().name]
|
||||
|
||||
def fake_fetch(provider, home, api_key=None):
|
||||
events.append(("enter", str(home)))
|
||||
barrier.wait()
|
||||
events.append(("exit", str(home)))
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(providers, "_get_hermes_home", fake_home)
|
||||
monkeypatch.setattr(providers, "_agent_fetch_account_usage_for_home", fake_fetch)
|
||||
|
||||
def worker():
|
||||
try:
|
||||
providers._fetch_account_usage_with_profile_context("openai-codex")
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=worker, name="quota-a"),
|
||||
threading.Thread(target=worker, name="quota-b"),
|
||||
]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
assert not errors
|
||||
assert [kind for kind, _home in events[:2]] == ["enter", "enter"]
|
||||
|
||||
|
||||
def test_provider_quota_route_is_registered():
|
||||
"""The backend must expose a route for the UI to poll quota status."""
|
||||
routes = (ROOT / "api" / "routes.py").read_text(encoding="utf-8")
|
||||
|
||||
Reference in new issue
Block a user