diff --git a/api/routes.py b/api/routes.py index 4642ac35..65e46b95 100644 --- a/api/routes.py +++ b/api/routes.py @@ -2068,6 +2068,52 @@ def _get_cached_session_list_payload( return cached stale = cached # now actually a stale payload when one exists, else None + if stale is not None: + event, is_owner = _session_list_cache_claim_rebuild(key) + if is_owner: + if diag is not None: + try: + diag.stage("session_list_cache_stale_background_rebuild") + except Exception: + pass + + def _rebuild_stale_session_list_cache(): + try: + rebuild_attempts = 0 + while True: + invalidation_stamp = _session_list_cache_invalidation_stamp(key) + try: + payload = builder() + except Exception: + logger.exception( + "session list stale-cache background rebuild failed" + ) + return + if _session_list_cache_invalidation_stamp(key) == invalidation_stamp: + _session_list_cache_set(key, payload) + return + rebuild_attempts += 1 + if rebuild_attempts >= 3: + return + finally: + _session_list_cache_done(key, event) + + try: + thread = threading.Thread( + target=_rebuild_stale_session_list_cache, + name="session-list-cache-rebuild", + daemon=True, + ) + thread.start() + except Exception: + _session_list_cache_done(key, event) + elif diag is not None: + try: + diag.stage("session_list_cache_stale_return") + except Exception: + pass + return stale + event, is_owner = _session_list_cache_claim_rebuild(key) if is_owner: if diag is not None: diff --git a/static/messages.js b/static/messages.js index d89badf8..79571240 100644 --- a/static/messages.js +++ b/static/messages.js @@ -1911,6 +1911,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ const _startedAt=(S.session&&S.session.pending_started_at)||Date.now()/1000; showLiveRunStatus(activeSid,{startedAt:_startedAt}); } + _suspendSessionStreamForLiveChat(activeSid); // On reconnect, restore accumulated text from INFLIGHT so we don't lose // progress made before the session switch. Without this the closure starts @@ -2018,6 +2019,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ delete INFLIGHT[activeSid]; clearInflightState(activeSid); _clearActivePaneInflightIfOwner(); + _resumeSessionStreamAfterLiveChat(activeSid); } function _isMarkerOnlyAssistantMessage(m){ if(!m||m.role!=='assistant') return false; @@ -6441,11 +6443,47 @@ function _stopHiddenActiveStreamPoll() { _sessionStreamHiddenPollSid = null; } +function _chatStreamActiveForSession(sid) { + if (!sid) return false; + if (typeof LIVE_STREAMS !== 'undefined' && LIVE_STREAMS[sid]) return true; + return !!( + S && + S.session && + S.session.session_id === sid && + S.activeStreamId + ); +} + +function _suspendSessionStreamForLiveChat(sid) { + if (!sid) return; + if (_sessionStreamSessionId !== sid && !_sessionEventSource) return; + _sessionStreamHiddenSid = sid; + stopSessionStream(); +} + +function _resumeSessionStreamAfterLiveChat(sid) { + if (!sid) return; + setTimeout(() => { + if (!S || !S.session || S.session.session_id !== sid) return; + if (_chatStreamActiveForSession(sid)) return; + _sessionStreamHiddenSid = null; + startSessionStream(sid); + }, 0); +} + function startSessionStream(sid) { if (!sid) return; // Already on this session? No-op (loadSession is a no-op when re-selecting // the same session; this defends against external re-callers). if (_sessionStreamSessionId === sid && _sessionEventSource) return; + // While the visible conversation has a live token stream, /api/chat/stream + // already carries bg_task_complete and terminal events for this turn. Keeping + // /api/session/stream open at the same time burns one of Chrome's six + // same-origin HTTP/1.1 sockets and can starve ordinary /api/session fetches. + if (_chatStreamActiveForSession(sid)) { + _sessionStreamHiddenSid = sid; + return; + } stopSessionStream(); _sessionStreamSessionId = sid; // Visibility hook (install once) — mirror ensureSessionEventsSSE() pattern. diff --git a/static/sessions.js b/static/sessions.js index 7d609024..cd7efd3c 100644 --- a/static/sessions.js +++ b/static/sessions.js @@ -2385,7 +2385,10 @@ async function _ensureMessagesLoaded(sid, opts) { const expandParam = reloadLimit ? '&expand_renderable=1' : ''; let data; try { - data = await api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0${reloadLimitParam}${expandParam}`); + data = await api( + `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0${reloadLimitParam}${expandParam}`, + {timeoutMs:120000} + ); } finally { _clearSameSessionForceReloadHint(sid); } @@ -2869,7 +2872,10 @@ async function _loadOlderMessages() { // Cumulative growth: each "load more" asks for currentLoaded + 30, and the // newly exposed head is what we expose to the user. const requestedLimit = Math.max(_INITIAL_MSG_LIMIT, (S.messages || []).length + _INITIAL_MSG_LIMIT); - const data = await api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_limit=${requestedLimit}`); + const data = await api( + `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_limit=${requestedLimit}`, + {timeoutMs:120000} + ); // Guard: api() may have redirected (401) and returned undefined. if (!data || !data.session) { _loadingOlder = false; return; } // - response shape sane @@ -2913,7 +2919,10 @@ async function _loadOlderMessages() { // Race fallback: keep the legacy index-page request as the // correctness-preserving alternative. Same guards reapplied because // we just awaited again. - const fallback = await api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_before=${_oldestIdx}&msg_limit=${_INITIAL_MSG_LIMIT}`); + const fallback = await api( + `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_before=${_oldestIdx}&msg_limit=${_INITIAL_MSG_LIMIT}`, + {timeoutMs:120000} + ); if (!fallback || !fallback.session) { _loadingOlder = false; return; } if (!S.session || S.session.session_id !== sid) return; if (_loadingSessionId !== null && _loadingSessionId !== sid) return; @@ -4362,9 +4371,7 @@ async function _loadSidebarSessionListPayload(sessionListQS, sessionRequestOpts) } })(); - const sessData = _sessionListHasLoadedOnce - ? await api('/api/sessions' + sessionListQS,{timeoutToast:false}) - : await api('/api/sessions' + sessionListQS,sessionRequestOpts); + const sessData = await api('/api/sessions' + sessionListQS,sessionRequestOpts); const projData = await projectPromise; return {sessData,projData}; @@ -4407,7 +4414,7 @@ let _gatewayPollTimer = null; let _gatewayProbeInFlight = false; let _gatewaySSEWarningShown = false; const _gatewayFallbackPollMs = 30000; -const _streamingPollMs = 5000; +const _streamingPollMs = 30000; const _sessionTimeRefreshMs = 60000; // #3107: the active-session "is it externally updated?" poll used to fire // every 5 s. On long sessions this caused visible scroll jitter and a diff --git a/static/ui.js b/static/ui.js index 5ddbc453..13815be1 100644 --- a/static/ui.js +++ b/static/ui.js @@ -24,12 +24,15 @@ const MAX_UPLOAD_MB=Math.round(MAX_UPLOAD_BYTES/1024/1024); let _queueDrainSid=null; const $=id=>document.getElementById(id); const OFFLINE_RECHECK_MS=2500; +const OFFLINE_HEALTH_TIMEOUT_MS=10000; +const OFFLINE_FETCH_FAILURES_BEFORE_BANNER=2; let _offlineVisible=false; let _offlineReason='browser'; let _offlineProbeTimer=null; let _offlineChecking=false; let _offlineProbePromise=null; let _offlineHealthProbePromise=null; +let _offlineFetchProbeFailures=0; let _offlineRawFetch=null; let _offlineFetchPatched=false; function _browserReportsOnline(){return !('onLine' in navigator)||navigator.onLine!==false;} @@ -82,7 +85,7 @@ async function _probeOfflineRecovery(){ // the initial banner display on the offline-event/startup paths. let ctrl=null,timer=null; try{ctrl=(typeof AbortController!=='undefined')?new AbortController():null;}catch(_){ctrl=null;} - if(ctrl)timer=setTimeout(()=>{try{ctrl.abort();}catch(_){}},3000); + if(ctrl)timer=setTimeout(()=>{try{ctrl.abort();}catch(_){}},OFFLINE_HEALTH_TIMEOUT_MS); try{ const opts={cache:'no-store',credentials:'include'}; if(ctrl)opts.signal=ctrl.signal; @@ -94,15 +97,22 @@ async function _probeOfflineRecovery(){ try{return await _offlineHealthProbePromise;} finally{_offlineHealthProbePromise=null;} } -async function _showOfflineBannerIfProbeFails(reason){ +async function _showOfflineBannerIfProbeFails(reason,opts){ + opts=opts||{}; const visibleAtStart=_offlineVisible; + const requireConsecutiveFailures=opts.requireConsecutiveFailures!==false; if(visibleAtStart)_setOfflineChecking(true); const ok=await _probeOfflineRecovery(); if(visibleAtStart)_setOfflineChecking(false); if(ok){ + _offlineFetchProbeFailures=0; if(_offlineVisible){_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();} return true; } + if(!visibleAtStart&&requireConsecutiveFailures){ + _offlineFetchProbeFailures+=1; + if(_offlineFetchProbeFailures{void _showOfflineBannerIfProbeFails('browser');}); + window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});}); window.addEventListener('online',()=>{if(_offlineVisible)checkOfflineRecoveryNow();}); - if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser'); + if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false}); } if(document.readyState==='loading')document.addEventListener('DOMContentLoaded',initOfflineMonitor,{once:true}); else initOfflineMonitor(); diff --git a/tests/test_4170_offline_probe.py b/tests/test_4170_offline_probe.py index 69217b8c..b3ee318b 100644 --- a/tests/test_4170_offline_probe.py +++ b/tests/test_4170_offline_probe.py @@ -37,17 +37,20 @@ def test_recovery_loop_probes_even_when_browser_reports_offline(): assert "_probeOfflineRecovery()" in body assert "if(!_browserReportsOnline()){showOfflineBanner('browser');return false;}" not in body assert body.find("_probeOfflineRecovery()") < body.find("showOfflineBanner(") - assert "if(ok){if(!_offlineVisible)return true;_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();return true;}" in body + assert "if(ok){_offlineFetchProbeFailures=0;if(!_offlineVisible)return true;_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();return true;}" in body assert "showOfflineBanner(_browserReportsOnline()?'network':'browser')" in body def test_probe_backed_browser_banner_helper_uses_health_as_source_of_truth(): body = _strip_comments(_fn_body(UI_JS, "async function _showOfflineBannerIfProbeFails(")) + assert "opts=opts||{};" in body assert "const visibleAtStart=_offlineVisible;" in body + assert "const requireConsecutiveFailures=opts.requireConsecutiveFailures!==false;" in body assert "if(visibleAtStart)_setOfflineChecking(true);" in body assert "const ok=await _probeOfflineRecovery();" in body assert "if(visibleAtStart)_setOfflineChecking(false);" in body assert "if(ok)" in body + assert "_offlineFetchProbeFailuresshowOfflineBanner('browser'))" not in body assert "if(!_browserReportsOnline())showOfflineBanner('browser');" not in body - assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser');});" in body - assert "if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser');" in body + assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});});" in body + assert "if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});" in body def test_fetch_catch_probes_before_browser_banner_and_rethrows(): diff --git a/tests/test_api_timeout.py b/tests/test_api_timeout.py index 09f05b8f..74d184f3 100644 --- a/tests/test_api_timeout.py +++ b/tests/test_api_timeout.py @@ -217,6 +217,29 @@ def test_update_flows_keep_explicit_longer_timeouts(): assert "api('/api/updates/force',{method:'POST',body:JSON.stringify({target}),timeoutMs:120000})" in src +def test_session_message_loads_keep_explicit_longer_timeouts(): + """Large state.db installs can take longer than the generic 30s API timeout.""" + src = _source(SESSIONS_JS) + assert ( + "api(\n" + " `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0${reloadLimitParam}${expandParam}`,\n" + " {timeoutMs:120000}\n" + " )" + ) in src + assert ( + "api(\n" + " `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_limit=${requestedLimit}`,\n" + " {timeoutMs:120000}\n" + " )" + ) in src + assert ( + "api(\n" + " `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_before=${_oldestIdx}&msg_limit=${_INITIAL_MSG_LIMIT}`,\n" + " {timeoutMs:120000}\n" + " )" + ) in src + + def test_passive_background_polls_suppress_timeout_toasts(): """Passive refreshes should be best-effort and not emit generic timeout toasts.""" workspace = _source(WORKSPACE_JS) @@ -226,7 +249,9 @@ def test_passive_background_polls_suppress_timeout_toasts(): panels = _source(PANELS_JS) assert "api('/api/client-events/log',{method:'POST',body:JSON.stringify(payload),timeoutMs:3000,timeoutToast:false})" in workspace - assert "api('/api/sessions' + sessionListQS,{timeoutToast:false})" in sessions + assert "const sessionRequestOpts={" in sessions + assert "timeoutToast:false" in sessions + assert "api('/api/sessions' + sessionListQS,sessionRequestOpts)" in sessions assert "api('/api/projects' + projectQS,{timeoutToast:false})" in sessions assert "api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=0&resolve_model=0`,{timeoutToast:false})" in sessions assert 'api("/api/approval/pending?session_id=" + encodeURIComponent(sid),{timeoutToast:false})' in messages diff --git a/tests/test_issue3996_sse_visibility.py b/tests/test_issue3996_sse_visibility.py index 78e989e8..91588790 100644 --- a/tests/test_issue3996_sse_visibility.py +++ b/tests/test_issue3996_sse_visibility.py @@ -41,7 +41,7 @@ def test_session_stream_has_visibility_hook(): assert "_hermesSessionStreamVisibilityHook" in MESSAGES_JS start_idx = MESSAGES_JS.find("function startSessionStream(sid)") assert start_idx != -1 - block = MESSAGES_JS[start_idx:start_idx + 2400] + block = MESSAGES_JS[start_idx:start_idx + 2600] assert "visibilitychange" in block assert "document.hidden" in block # Must skip opening a new EventSource while the tab is hidden. @@ -95,7 +95,7 @@ def test_session_stream_hidden_open_preserves_id_for_reopen(): _sessionStreamHiddenSid = sid before returning. """ start_idx = MESSAGES_JS.find("function startSessionStream(sid)") - block = MESSAGES_JS[start_idx:start_idx + 2400] + block = MESSAGES_JS[start_idx:start_idx + 2800] # Find the hidden-tab early-return skip path (distinct from the in-hook # `if (document.hidden)` branch) and assert it preserves the id. hidden_idx = block.find("!== 'undefined' && document.hidden) {") @@ -104,4 +104,3 @@ def test_session_stream_hidden_open_preserves_id_for_reopen(): assert "_sessionStreamHiddenSid = sid" in hidden_block, ( "hidden-tab skip must record the pending session id for reopen on re-show" ) - diff --git a/tests/test_issue856_background_completion_unread.py b/tests/test_issue856_background_completion_unread.py index 35c07d5b..e52d6453 100644 --- a/tests/test_issue856_background_completion_unread.py +++ b/tests/test_issue856_background_completion_unread.py @@ -160,6 +160,7 @@ def test_polling_transition_marks_completion_unread_without_sse_done(): ) assert "_markSessionCompletionUnread(sid, s.message_count);" in transition_block assert "_sessionStreamingById.set(sid, isStreaming);" in transition_block + assert "const _streamingPollMs = 30000;" in SESSIONS_JS assert "_applySessionListPayload(sessData,projData);" in refresh_block assert "_markPollingCompletionUnreadTransitions(_allSessions);" in apply_block assert "_allSessions.some(s => _isSessionEffectivelyStreaming(s))" in apply_block, ( diff --git a/tests/test_offline_banner.py b/tests/test_offline_banner.py index 3440d4f3..3842562c 100644 --- a/tests/test_offline_banner.py +++ b/tests/test_offline_banner.py @@ -35,8 +35,10 @@ def test_offline_banner_markup_styles_and_copy_exist(): def test_offline_monitor_patches_fetch_and_auto_reloads_after_health_probe(): assert "const OFFLINE_RECHECK_MS=2500" in UI_JS + assert "const OFFLINE_HEALTH_TIMEOUT_MS=10000" in UI_JS + assert "const OFFLINE_FETCH_FAILURES_BEFORE_BANNER=2" in UI_JS assert "window.fetch=async function(...args)" in UI_JS - assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser');});" in UI_JS + assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});});" in UI_JS assert "window.addEventListener('online',()=>{if(_offlineVisible)checkOfflineRecoveryNow();})" in UI_JS assert "setInterval(()=>{checkOfflineRecoveryNow();},OFFLINE_RECHECK_MS)" in UI_JS assert "new URL('health',document.baseURI||location.href)" in UI_JS @@ -60,6 +62,7 @@ def test_fetch_typeerror_is_gated_by_health_probe_not_blind_banner(): assert "function _isAbortError(e)" in UI_JS assert "!_isAbortError(e)&&(e instanceof TypeError||!_browserReportsOnline())" in fetch_patch assert "void _showOfflineBannerIfProbeFails(_browserReportsOnline()?'network':'browser');" in fetch_patch + assert "_offlineFetchProbeFailures str: + marker = f"function {name}(" + start = src.find(marker) + assert start != -1, f"{name}() not found" + brace = src.find("){", start) + assert brace != -1, f"{name}() body not found" + brace += 1 + depth = 1 + i = brace + 1 + in_str = None + escaped = False + while i < len(src) and depth: + ch = src[i] + if in_str: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == in_str: + in_str = None + elif ch in ('"', "'", "`"): + in_str = ch + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + i += 1 + assert depth == 0, f"{name}() body did not close" + return src[start:i] + + # --------------------------------------------------------------------------- # Module surface # --------------------------------------------------------------------------- @@ -392,6 +423,36 @@ def test_frontend_opens_session_stream(): assert "stopSessionStream" in js +def test_session_stream_pauses_while_chat_stream_is_active(): + """The active turn's chat SSE already carries live events for that session. + + Keeping /api/session/stream open for the same sid at the same time burns a + Chrome same-origin connection slot and can starve ordinary /api/session + fetches on HTTP/1.1. + """ + js = (REPO_ROOT / "static" / "messages.js").read_text() + assert "function _chatStreamActiveForSession(sid)" in js + assert "function _suspendSessionStreamForLiveChat(sid)" in js + assert "function _resumeSessionStreamAfterLiveChat(sid)" in js + + start_src = _js_function_decl(js, "startSessionStream") + assert "if (_chatStreamActiveForSession(sid))" in start_src + assert "_sessionStreamHiddenSid = sid;" in start_src + assert "return;" in start_src + assert start_src.index("if (_chatStreamActiveForSession(sid))") < start_src.index("new EventSource(") + + attach_ix = js.index("function attachLiveStream") + attach_src = js[attach_ix:js.index("function transcript()", attach_ix)] + assert "_suspendSessionStreamForLiveChat(activeSid);" in attach_src + assert attach_src.index("_suspendSessionStreamForLiveChat(activeSid);") < attach_src.index("new EventSource(") + + resume_src = _js_function_decl(js, "_resumeSessionStreamAfterLiveChat") + assert "S.session.session_id !== sid" in resume_src + assert "if (_chatStreamActiveForSession(sid)) return;" in resume_src + assert "_sessionStreamHiddenSid = null;" in resume_src + assert "startSessionStream(sid);" in resume_src + + def test_frontend_busy_race_gate_obsoleted_by_option_z_pivot(): """Per the Option Z PIVOT note baked into the handler body, the browser is no longer in the wakeup path at all — the server-side drain owns diff --git a/tests/test_session_sidebar_cache.py b/tests/test_session_sidebar_cache.py index cf40c3bd..c03c1e77 100644 --- a/tests/test_session_sidebar_cache.py +++ b/tests/test_session_sidebar_cache.py @@ -265,10 +265,126 @@ def test_session_list_cache_follower_reuses_stale_payload_during_slow_rebuild(mo owner_thread.join(2.0) follower_thread.join(2.0) + assert owner_result["payload"] == _session_cache_payload("stale") assert follower_result["payload"] == _session_cache_payload("stale") - assert owner_result["payload"] == _session_cache_payload("fresh") - assert "session_list_cache_wait_stale" in follower_diag.stages - assert "session_list_cache_wait_stale_fallback" in follower_diag.stages + assert "session_list_cache_stale_background_rebuild" in owner_diag.stages + assert ( + "session_list_cache_stale_return" in follower_diag.stages + or "session_list_cache_stale_background_rebuild" in follower_diag.stages + ) + + +def test_session_list_cache_owner_returns_stale_and_rebuilds_in_background(monkeypatch): + routes._session_list_cache_clear() + + key = routes._session_list_cache_key( + active_profile="default", + all_profiles=False, + show_cli_sessions=False, + show_previous_messaging_sessions=False, + show_cron_sessions=False, + ) + routes._session_list_cache_set(key, _session_cache_payload("stale")) + with routes._SESSIONS_CACHE_LOCK: + ts, stamp, payload = routes._SESSIONS_CACHE[key] + routes._SESSIONS_CACHE[key] = ( + ts - routes._SESSIONS_CACHE_TTL_SECONDS - 1.0, + stamp, + payload, + ) + monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda _key: ("changed",)) + + started = threading.Event() + release = threading.Event() + diag = _StageRecorder() + + def builder(): + started.set() + release.wait() + return _session_cache_payload("fresh") + + try: + result = routes._get_cached_session_list_payload(key=key, builder=builder, diag=diag) + assert result == _session_cache_payload("stale") + assert started.wait(1.0), "stale owner should kick off a background rebuild" + assert "session_list_cache_stale_background_rebuild" in diag.stages + # While background rebuild is still blocked, stale is still returned. + cached, fresh = routes._session_list_cache_get(key, allow_stale=True) + assert cached == _session_cache_payload("stale") + assert fresh is False + finally: + release.set() + + # The background owner should eventually populate fresh cache. + deadline = threading.Event() + for _ in range(20): + cached, fresh = routes._session_list_cache_get(key, allow_stale=True) + if cached == _session_cache_payload("fresh"): + break + deadline.wait(0.05) + assert cached == _session_cache_payload("fresh") + + +def test_session_list_cache_stale_background_rebuild_failure_releases_owner(monkeypatch): + routes._session_list_cache_clear() + + key = routes._session_list_cache_key( + active_profile="default", + all_profiles=False, + show_cli_sessions=False, + show_previous_messaging_sessions=False, + show_cron_sessions=False, + ) + routes._session_list_cache_set(key, _session_cache_payload("stale")) + with routes._SESSIONS_CACHE_LOCK: + ts, stamp, payload = routes._SESSIONS_CACHE[key] + routes._SESSIONS_CACHE[key] = ( + ts - routes._SESSIONS_CACHE_TTL_SECONDS - 1.0, + stamp, + payload, + ) + monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda _key: ("changed",)) + + started = threading.Event() + logged = [] + + def record_exception(message): + logged.append(message) + + monkeypatch.setattr(routes.logger, "exception", record_exception) + + def failing_builder(): + started.set() + raise RuntimeError("boom") + + result = routes._get_cached_session_list_payload(key=key, builder=failing_builder) + + assert result == _session_cache_payload("stale") + assert started.wait(1.0), "background rebuild should have attempted the builder" + for _ in range(20): + with routes._SESSIONS_CACHE_LOCK: + inflight_empty = key not in routes._SESSIONS_CACHE_INFLIGHT + if inflight_empty: + break + threading.Event().wait(0.05) + assert inflight_empty, "failed background rebuild must release the singleflight owner" + assert logged == ["session list stale-cache background rebuild failed"] + cached, fresh = routes._session_list_cache_get(key, allow_stale=True) + assert cached == _session_cache_payload("stale") + assert fresh is False + + def recovery_builder(): + return _session_cache_payload("fresh") + + # Once the failed owner releases, a later request can retry instead of being + # pinned behind the dead background rebuild. + routes._get_cached_session_list_payload(key=key, builder=recovery_builder) + for _ in range(20): + cached, _fresh = routes._session_list_cache_get(key, allow_stale=True) + if cached == _session_cache_payload("fresh"): + break + threading.Event().wait(0.05) + assert cached == _session_cache_payload("fresh") def test_session_list_cache_invalidated_on_session_list_publish(): diff --git a/tests/test_session_sidebar_resilience.py b/tests/test_session_sidebar_resilience.py index bba6627a..bef1a508 100644 --- a/tests/test_session_sidebar_resilience.py +++ b/tests/test_session_sidebar_resilience.py @@ -59,8 +59,15 @@ def test_sessions_and_projects_load_independently_so_projects_failure_cannot_bla def test_sessions_api_uses_longer_timeout_and_timeout_retry_for_boot_refresh(): sessions_src = _sessions_js() workspace_src = _workspace_js() + helper_start = sessions_src.find("async function _loadSidebarSessionListPayload") + assert helper_start > 0 + helper_end = sessions_src.find("async function _drainRenderSessionListQueue", helper_start) + assert helper_end > helper_start + helper = sessions_src[helper_start:helper_end] assert "timeoutMs:_sessionListHasLoadedOnce?30000:_SESSION_LIST_BOOT_TIMEOUT_MS" in sessions_src + assert "const sessData = await api('/api/sessions' + sessionListQS,sessionRequestOpts);" in helper + assert "api('/api/sessions' + sessionListQS,{timeoutToast:false})" not in helper assert "retryTimeouts:true" in sessions_src assert "retryStatuses:[502,503,504]" in sessions_src assert "retryTimeouts" in workspace_src