Reduce connection lost flapping under large session state

This commit is contained in:
Frank Song authored and nesquena-hermes committed 2026-06-29 19:52:02 +00:00
1 parent ce425c7127
commit cbb644da69
12 files changed
+339 -23

No files matched your search

+46
View File
@@ -2068,6 +2068,52 @@ def _get_cached_session_list_payload(
return cached return cached
stale = cached # now actually a stale payload when one exists, else None stale = cached # now actually a stale payload when one exists, else None
if stale is not None:
event, is_owner = _session_list_cache_claim_rebuild(key)
if is_owner:
if diag is not None:
try:
diag.stage("session_list_cache_stale_background_rebuild")
except Exception:
pass
def _rebuild_stale_session_list_cache():
try:
rebuild_attempts = 0
while True:
invalidation_stamp = _session_list_cache_invalidation_stamp(key)
try:
payload = builder()
except Exception:
logger.exception(
"session list stale-cache background rebuild failed"
)
return
if _session_list_cache_invalidation_stamp(key) == invalidation_stamp:
_session_list_cache_set(key, payload)
return
rebuild_attempts += 1
if rebuild_attempts >= 3:
return
finally:
_session_list_cache_done(key, event)
try:
thread = threading.Thread(
target=_rebuild_stale_session_list_cache,
name="session-list-cache-rebuild",
daemon=True,
)
thread.start()
except Exception:
_session_list_cache_done(key, event)
elif diag is not None:
try:
diag.stage("session_list_cache_stale_return")
except Exception:
pass
return stale
event, is_owner = _session_list_cache_claim_rebuild(key) event, is_owner = _session_list_cache_claim_rebuild(key)
if is_owner: if is_owner:
if diag is not None: if diag is not None:
+38
View File
@@ -1911,6 +1911,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
const _startedAt=(S.session&&S.session.pending_started_at)||Date.now()/1000; const _startedAt=(S.session&&S.session.pending_started_at)||Date.now()/1000;
showLiveRunStatus(activeSid,{startedAt:_startedAt}); showLiveRunStatus(activeSid,{startedAt:_startedAt});
} }
_suspendSessionStreamForLiveChat(activeSid);
// On reconnect, restore accumulated text from INFLIGHT so we don't lose // On reconnect, restore accumulated text from INFLIGHT so we don't lose
// progress made before the session switch. Without this the closure starts // progress made before the session switch. Without this the closure starts
@@ -2018,6 +2019,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
delete INFLIGHT[activeSid]; delete INFLIGHT[activeSid];
clearInflightState(activeSid); clearInflightState(activeSid);
_clearActivePaneInflightIfOwner(); _clearActivePaneInflightIfOwner();
_resumeSessionStreamAfterLiveChat(activeSid);
} }
function _isMarkerOnlyAssistantMessage(m){ function _isMarkerOnlyAssistantMessage(m){
if(!m||m.role!=='assistant') return false; if(!m||m.role!=='assistant') return false;
@@ -6441,11 +6443,47 @@ function _stopHiddenActiveStreamPoll() {
_sessionStreamHiddenPollSid = null; _sessionStreamHiddenPollSid = null;
} }
function _chatStreamActiveForSession(sid) {
if (!sid) return false;
if (typeof LIVE_STREAMS !== 'undefined' && LIVE_STREAMS[sid]) return true;
return !!(
S &&
S.session &&
S.session.session_id === sid &&
S.activeStreamId
);
}
function _suspendSessionStreamForLiveChat(sid) {
if (!sid) return;
if (_sessionStreamSessionId !== sid && !_sessionEventSource) return;
_sessionStreamHiddenSid = sid;
stopSessionStream();
}
function _resumeSessionStreamAfterLiveChat(sid) {
if (!sid) return;
setTimeout(() => {
if (!S || !S.session || S.session.session_id !== sid) return;
if (_chatStreamActiveForSession(sid)) return;
_sessionStreamHiddenSid = null;
startSessionStream(sid);
}, 0);
}
function startSessionStream(sid) { function startSessionStream(sid) {
if (!sid) return; if (!sid) return;
// Already on this session? No-op (loadSession is a no-op when re-selecting // Already on this session? No-op (loadSession is a no-op when re-selecting
// the same session; this defends against external re-callers). // the same session; this defends against external re-callers).
if (_sessionStreamSessionId === sid && _sessionEventSource) return; if (_sessionStreamSessionId === sid && _sessionEventSource) return;
// While the visible conversation has a live token stream, /api/chat/stream
// already carries bg_task_complete and terminal events for this turn. Keeping
// /api/session/stream open at the same time burns one of Chrome's six
// same-origin HTTP/1.1 sockets and can starve ordinary /api/session fetches.
if (_chatStreamActiveForSession(sid)) {
_sessionStreamHiddenSid = sid;
return;
}
stopSessionStream(); stopSessionStream();
_sessionStreamSessionId = sid; _sessionStreamSessionId = sid;
// Visibility hook (install once) — mirror ensureSessionEventsSSE() pattern. // Visibility hook (install once) — mirror ensureSessionEventsSSE() pattern.
+14 -7
View File
@@ -2385,7 +2385,10 @@ async function _ensureMessagesLoaded(sid, opts) {
const expandParam = reloadLimit ? '&expand_renderable=1' : ''; const expandParam = reloadLimit ? '&expand_renderable=1' : '';
let data; let data;
try { try {
data = await api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0${reloadLimitParam}${expandParam}`); data = await api(
`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0${reloadLimitParam}${expandParam}`,
{timeoutMs:120000}
);
} finally { } finally {
_clearSameSessionForceReloadHint(sid); _clearSameSessionForceReloadHint(sid);
} }
@@ -2869,7 +2872,10 @@ async function _loadOlderMessages() {
// Cumulative growth: each "load more" asks for currentLoaded + 30, and the // Cumulative growth: each "load more" asks for currentLoaded + 30, and the
// newly exposed head is what we expose to the user. // newly exposed head is what we expose to the user.
const requestedLimit = Math.max(_INITIAL_MSG_LIMIT, (S.messages || []).length + _INITIAL_MSG_LIMIT); const requestedLimit = Math.max(_INITIAL_MSG_LIMIT, (S.messages || []).length + _INITIAL_MSG_LIMIT);
const data = await api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_limit=${requestedLimit}`); const data = await api(
`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_limit=${requestedLimit}`,
{timeoutMs:120000}
);
// Guard: api() may have redirected (401) and returned undefined. // Guard: api() may have redirected (401) and returned undefined.
if (!data || !data.session) { _loadingOlder = false; return; } if (!data || !data.session) { _loadingOlder = false; return; }
// - response shape sane // - response shape sane
@@ -2913,7 +2919,10 @@ async function _loadOlderMessages() {
// Race fallback: keep the legacy index-page request as the // Race fallback: keep the legacy index-page request as the
// correctness-preserving alternative. Same guards reapplied because // correctness-preserving alternative. Same guards reapplied because
// we just awaited again. // we just awaited again.
const fallback = await api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_before=${_oldestIdx}&msg_limit=${_INITIAL_MSG_LIMIT}`); const fallback = await api(
`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_before=${_oldestIdx}&msg_limit=${_INITIAL_MSG_LIMIT}`,
{timeoutMs:120000}
);
if (!fallback || !fallback.session) { _loadingOlder = false; return; } if (!fallback || !fallback.session) { _loadingOlder = false; return; }
if (!S.session || S.session.session_id !== sid) return; if (!S.session || S.session.session_id !== sid) return;
if (_loadingSessionId !== null && _loadingSessionId !== sid) return; if (_loadingSessionId !== null && _loadingSessionId !== sid) return;
@@ -4362,9 +4371,7 @@ async function _loadSidebarSessionListPayload(sessionListQS, sessionRequestOpts)
} }
})(); })();
const sessData = _sessionListHasLoadedOnce const sessData = await api('/api/sessions' + sessionListQS,sessionRequestOpts);
? await api('/api/sessions' + sessionListQS,{timeoutToast:false})
: await api('/api/sessions' + sessionListQS,sessionRequestOpts);
const projData = await projectPromise; const projData = await projectPromise;
return {sessData,projData}; return {sessData,projData};
@@ -4407,7 +4414,7 @@ let _gatewayPollTimer = null;
let _gatewayProbeInFlight = false; let _gatewayProbeInFlight = false;
let _gatewaySSEWarningShown = false; let _gatewaySSEWarningShown = false;
const _gatewayFallbackPollMs = 30000; const _gatewayFallbackPollMs = 30000;
const _streamingPollMs = 5000; const _streamingPollMs = 30000;
const _sessionTimeRefreshMs = 60000; const _sessionTimeRefreshMs = 60000;
// #3107: the active-session "is it externally updated?" poll used to fire // #3107: the active-session "is it externally updated?" poll used to fire
// every 5 s. On long sessions this caused visible scroll jitter and a // every 5 s. On long sessions this caused visible scroll jitter and a
+15 -5
View File
@@ -24,12 +24,15 @@ const MAX_UPLOAD_MB=Math.round(MAX_UPLOAD_BYTES/1024/1024);
let _queueDrainSid=null; let _queueDrainSid=null;
const $=id=>document.getElementById(id); const $=id=>document.getElementById(id);
const OFFLINE_RECHECK_MS=2500; const OFFLINE_RECHECK_MS=2500;
const OFFLINE_HEALTH_TIMEOUT_MS=10000;
const OFFLINE_FETCH_FAILURES_BEFORE_BANNER=2;
let _offlineVisible=false; let _offlineVisible=false;
let _offlineReason='browser'; let _offlineReason='browser';
let _offlineProbeTimer=null; let _offlineProbeTimer=null;
let _offlineChecking=false; let _offlineChecking=false;
let _offlineProbePromise=null; let _offlineProbePromise=null;
let _offlineHealthProbePromise=null; let _offlineHealthProbePromise=null;
let _offlineFetchProbeFailures=0;
let _offlineRawFetch=null; let _offlineRawFetch=null;
let _offlineFetchPatched=false; let _offlineFetchPatched=false;
function _browserReportsOnline(){return !('onLine' in navigator)||navigator.onLine!==false;} function _browserReportsOnline(){return !('onLine' in navigator)||navigator.onLine!==false;}
@@ -82,7 +85,7 @@ async function _probeOfflineRecovery(){
// the initial banner display on the offline-event/startup paths. // the initial banner display on the offline-event/startup paths.
let ctrl=null,timer=null; let ctrl=null,timer=null;
try{ctrl=(typeof AbortController!=='undefined')?new AbortController():null;}catch(_){ctrl=null;} try{ctrl=(typeof AbortController!=='undefined')?new AbortController():null;}catch(_){ctrl=null;}
if(ctrl)timer=setTimeout(()=>{try{ctrl.abort();}catch(_){}},3000); if(ctrl)timer=setTimeout(()=>{try{ctrl.abort();}catch(_){}},OFFLINE_HEALTH_TIMEOUT_MS);
try{ try{
const opts={cache:'no-store',credentials:'include'}; const opts={cache:'no-store',credentials:'include'};
if(ctrl)opts.signal=ctrl.signal; if(ctrl)opts.signal=ctrl.signal;
@@ -94,15 +97,22 @@ async function _probeOfflineRecovery(){
try{return await _offlineHealthProbePromise;} try{return await _offlineHealthProbePromise;}
finally{_offlineHealthProbePromise=null;} finally{_offlineHealthProbePromise=null;}
} }
async function _showOfflineBannerIfProbeFails(reason){ async function _showOfflineBannerIfProbeFails(reason,opts){
opts=opts||{};
const visibleAtStart=_offlineVisible; const visibleAtStart=_offlineVisible;
const requireConsecutiveFailures=opts.requireConsecutiveFailures!==false;
if(visibleAtStart)_setOfflineChecking(true); if(visibleAtStart)_setOfflineChecking(true);
const ok=await _probeOfflineRecovery(); const ok=await _probeOfflineRecovery();
if(visibleAtStart)_setOfflineChecking(false); if(visibleAtStart)_setOfflineChecking(false);
if(ok){ if(ok){
_offlineFetchProbeFailures=0;
if(_offlineVisible){_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();} if(_offlineVisible){_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();}
return true; return true;
} }
if(!visibleAtStart&&requireConsecutiveFailures){
_offlineFetchProbeFailures+=1;
if(_offlineFetchProbeFailures<OFFLINE_FETCH_FAILURES_BEFORE_BANNER)return false;
}
showOfflineBanner(reason||(_browserReportsOnline()?'network':'browser')); showOfflineBanner(reason||(_browserReportsOnline()?'network':'browser'));
return false; return false;
} }
@@ -113,7 +123,7 @@ async function checkOfflineRecoveryNow(){
_setOfflineChecking(true); _setOfflineChecking(true);
const ok=await _probeOfflineRecovery(); const ok=await _probeOfflineRecovery();
_setOfflineChecking(false); _setOfflineChecking(false);
if(ok){if(!_offlineVisible)return true;_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();return true;} if(ok){_offlineFetchProbeFailures=0;if(!_offlineVisible)return true;_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();return true;}
showOfflineBanner(_browserReportsOnline()?'network':'browser'); showOfflineBanner(_browserReportsOnline()?'network':'browser');
return false; return false;
})(); })();
@@ -182,9 +192,9 @@ function _patchOfflineFetch(){
} }
function initOfflineMonitor(){ function initOfflineMonitor(){
_patchOfflineFetch(); _patchOfflineFetch();
window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser');}); window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});});
window.addEventListener('online',()=>{if(_offlineVisible)checkOfflineRecoveryNow();}); window.addEventListener('online',()=>{if(_offlineVisible)checkOfflineRecoveryNow();});
if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser'); if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});
} }
if(document.readyState==='loading')document.addEventListener('DOMContentLoaded',initOfflineMonitor,{once:true}); if(document.readyState==='loading')document.addEventListener('DOMContentLoaded',initOfflineMonitor,{once:true});
else initOfflineMonitor(); else initOfflineMonitor();
+6 -3
View File
@@ -37,17 +37,20 @@ def test_recovery_loop_probes_even_when_browser_reports_offline():
assert "_probeOfflineRecovery()" in body assert "_probeOfflineRecovery()" in body
assert "if(!_browserReportsOnline()){showOfflineBanner('browser');return false;}" not in body assert "if(!_browserReportsOnline()){showOfflineBanner('browser');return false;}" not in body
assert body.find("_probeOfflineRecovery()") < body.find("showOfflineBanner(") assert body.find("_probeOfflineRecovery()") < body.find("showOfflineBanner(")
assert "if(ok){if(!_offlineVisible)return true;_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();return true;}" in body assert "if(ok){_offlineFetchProbeFailures=0;if(!_offlineVisible)return true;_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();return true;}" in body
assert "showOfflineBanner(_browserReportsOnline()?'network':'browser')" in body assert "showOfflineBanner(_browserReportsOnline()?'network':'browser')" in body
def test_probe_backed_browser_banner_helper_uses_health_as_source_of_truth(): def test_probe_backed_browser_banner_helper_uses_health_as_source_of_truth():
body = _strip_comments(_fn_body(UI_JS, "async function _showOfflineBannerIfProbeFails(")) body = _strip_comments(_fn_body(UI_JS, "async function _showOfflineBannerIfProbeFails("))
assert "opts=opts||{};" in body
assert "const visibleAtStart=_offlineVisible;" in body assert "const visibleAtStart=_offlineVisible;" in body
assert "const requireConsecutiveFailures=opts.requireConsecutiveFailures!==false;" in body
assert "if(visibleAtStart)_setOfflineChecking(true);" in body assert "if(visibleAtStart)_setOfflineChecking(true);" in body
assert "const ok=await _probeOfflineRecovery();" in body assert "const ok=await _probeOfflineRecovery();" in body
assert "if(visibleAtStart)_setOfflineChecking(false);" in body assert "if(visibleAtStart)_setOfflineChecking(false);" in body
assert "if(ok)" in body assert "if(ok)" in body
assert "_offlineFetchProbeFailures<OFFLINE_FETCH_FAILURES_BEFORE_BANNER" in body
assert "showOfflineBanner(reason||(_browserReportsOnline()?'network':'browser'))" in body assert "showOfflineBanner(reason||(_browserReportsOnline()?'network':'browser'))" in body
assert body.find("_probeOfflineRecovery()") < body.find("showOfflineBanner(") assert body.find("_probeOfflineRecovery()") < body.find("showOfflineBanner(")
@@ -56,8 +59,8 @@ def test_offline_event_and_startup_false_online_probe_before_browser_banner():
body = _strip_comments(_fn_body(UI_JS, "function initOfflineMonitor(")) body = _strip_comments(_fn_body(UI_JS, "function initOfflineMonitor("))
assert "window.addEventListener('offline',()=>showOfflineBanner('browser'))" not in body assert "window.addEventListener('offline',()=>showOfflineBanner('browser'))" not in body
assert "if(!_browserReportsOnline())showOfflineBanner('browser');" not in body assert "if(!_browserReportsOnline())showOfflineBanner('browser');" not in body
assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser');});" in body assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});});" in body
assert "if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser');" in body assert "if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});" in body
def test_fetch_catch_probes_before_browser_banner_and_rethrows(): def test_fetch_catch_probes_before_browser_banner_and_rethrows():
+26 -1
View File
@@ -217,6 +217,29 @@ def test_update_flows_keep_explicit_longer_timeouts():
assert "api('/api/updates/force',{method:'POST',body:JSON.stringify({target}),timeoutMs:120000})" in src assert "api('/api/updates/force',{method:'POST',body:JSON.stringify({target}),timeoutMs:120000})" in src
def test_session_message_loads_keep_explicit_longer_timeouts():
"""Large state.db installs can take longer than the generic 30s API timeout."""
src = _source(SESSIONS_JS)
assert (
"api(\n"
" `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0${reloadLimitParam}${expandParam}`,\n"
" {timeoutMs:120000}\n"
" )"
) in src
assert (
"api(\n"
" `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_limit=${requestedLimit}`,\n"
" {timeoutMs:120000}\n"
" )"
) in src
assert (
"api(\n"
" `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_before=${_oldestIdx}&msg_limit=${_INITIAL_MSG_LIMIT}`,\n"
" {timeoutMs:120000}\n"
" )"
) in src
def test_passive_background_polls_suppress_timeout_toasts(): def test_passive_background_polls_suppress_timeout_toasts():
"""Passive refreshes should be best-effort and not emit generic timeout toasts.""" """Passive refreshes should be best-effort and not emit generic timeout toasts."""
workspace = _source(WORKSPACE_JS) workspace = _source(WORKSPACE_JS)
@@ -226,7 +249,9 @@ def test_passive_background_polls_suppress_timeout_toasts():
panels = _source(PANELS_JS) panels = _source(PANELS_JS)
assert "api('/api/client-events/log',{method:'POST',body:JSON.stringify(payload),timeoutMs:3000,timeoutToast:false})" in workspace assert "api('/api/client-events/log',{method:'POST',body:JSON.stringify(payload),timeoutMs:3000,timeoutToast:false})" in workspace
assert "api('/api/sessions' + sessionListQS,{timeoutToast:false})" in sessions assert "const sessionRequestOpts={" in sessions
assert "timeoutToast:false" in sessions
assert "api('/api/sessions' + sessionListQS,sessionRequestOpts)" in sessions
assert "api('/api/projects' + projectQS,{timeoutToast:false})" in sessions assert "api('/api/projects' + projectQS,{timeoutToast:false})" in sessions
assert "api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=0&resolve_model=0`,{timeoutToast:false})" in sessions assert "api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=0&resolve_model=0`,{timeoutToast:false})" in sessions
assert 'api("/api/approval/pending?session_id=" + encodeURIComponent(sid),{timeoutToast:false})' in messages assert 'api("/api/approval/pending?session_id=" + encodeURIComponent(sid),{timeoutToast:false})' in messages
+2 -3
View File
@@ -41,7 +41,7 @@ def test_session_stream_has_visibility_hook():
assert "_hermesSessionStreamVisibilityHook" in MESSAGES_JS assert "_hermesSessionStreamVisibilityHook" in MESSAGES_JS
start_idx = MESSAGES_JS.find("function startSessionStream(sid)") start_idx = MESSAGES_JS.find("function startSessionStream(sid)")
assert start_idx != -1 assert start_idx != -1
block = MESSAGES_JS[start_idx:start_idx + 2400] block = MESSAGES_JS[start_idx:start_idx + 2600]
assert "visibilitychange" in block assert "visibilitychange" in block
assert "document.hidden" in block assert "document.hidden" in block
# Must skip opening a new EventSource while the tab is hidden. # Must skip opening a new EventSource while the tab is hidden.
@@ -95,7 +95,7 @@ def test_session_stream_hidden_open_preserves_id_for_reopen():
_sessionStreamHiddenSid = sid before returning. _sessionStreamHiddenSid = sid before returning.
""" """
start_idx = MESSAGES_JS.find("function startSessionStream(sid)") start_idx = MESSAGES_JS.find("function startSessionStream(sid)")
block = MESSAGES_JS[start_idx:start_idx + 2400] block = MESSAGES_JS[start_idx:start_idx + 2800]
# Find the hidden-tab early-return skip path (distinct from the in-hook # Find the hidden-tab early-return skip path (distinct from the in-hook
# `if (document.hidden)` branch) and assert it preserves the id. # `if (document.hidden)` branch) and assert it preserves the id.
hidden_idx = block.find("!== 'undefined' && document.hidden) {") hidden_idx = block.find("!== 'undefined' && document.hidden) {")
@@ -104,4 +104,3 @@ def test_session_stream_hidden_open_preserves_id_for_reopen():
assert "_sessionStreamHiddenSid = sid" in hidden_block, ( assert "_sessionStreamHiddenSid = sid" in hidden_block, (
"hidden-tab skip must record the pending session id for reopen on re-show" "hidden-tab skip must record the pending session id for reopen on re-show"
) )
@@ -160,6 +160,7 @@ def test_polling_transition_marks_completion_unread_without_sse_done():
) )
assert "_markSessionCompletionUnread(sid, s.message_count);" in transition_block assert "_markSessionCompletionUnread(sid, s.message_count);" in transition_block
assert "_sessionStreamingById.set(sid, isStreaming);" in transition_block assert "_sessionStreamingById.set(sid, isStreaming);" in transition_block
assert "const _streamingPollMs = 30000;" in SESSIONS_JS
assert "_applySessionListPayload(sessData,projData);" in refresh_block assert "_applySessionListPayload(sessData,projData);" in refresh_block
assert "_markPollingCompletionUnreadTransitions(_allSessions);" in apply_block assert "_markPollingCompletionUnreadTransitions(_allSessions);" in apply_block
assert "_allSessions.some(s => _isSessionEffectivelyStreaming(s))" in apply_block, ( assert "_allSessions.some(s => _isSessionEffectivelyStreaming(s))" in apply_block, (
+4 -1
View File
@@ -35,8 +35,10 @@ def test_offline_banner_markup_styles_and_copy_exist():
def test_offline_monitor_patches_fetch_and_auto_reloads_after_health_probe(): def test_offline_monitor_patches_fetch_and_auto_reloads_after_health_probe():
assert "const OFFLINE_RECHECK_MS=2500" in UI_JS assert "const OFFLINE_RECHECK_MS=2500" in UI_JS
assert "const OFFLINE_HEALTH_TIMEOUT_MS=10000" in UI_JS
assert "const OFFLINE_FETCH_FAILURES_BEFORE_BANNER=2" in UI_JS
assert "window.fetch=async function(...args)" in UI_JS assert "window.fetch=async function(...args)" in UI_JS
assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser');});" in UI_JS assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});});" in UI_JS
assert "window.addEventListener('online',()=>{if(_offlineVisible)checkOfflineRecoveryNow();})" in UI_JS assert "window.addEventListener('online',()=>{if(_offlineVisible)checkOfflineRecoveryNow();})" in UI_JS
assert "setInterval(()=>{checkOfflineRecoveryNow();},OFFLINE_RECHECK_MS)" in UI_JS assert "setInterval(()=>{checkOfflineRecoveryNow();},OFFLINE_RECHECK_MS)" in UI_JS
assert "new URL('health',document.baseURI||location.href)" in UI_JS assert "new URL('health',document.baseURI||location.href)" in UI_JS
@@ -60,6 +62,7 @@ def test_fetch_typeerror_is_gated_by_health_probe_not_blind_banner():
assert "function _isAbortError(e)" in UI_JS assert "function _isAbortError(e)" in UI_JS
assert "!_isAbortError(e)&&(e instanceof TypeError||!_browserReportsOnline())" in fetch_patch assert "!_isAbortError(e)&&(e instanceof TypeError||!_browserReportsOnline())" in fetch_patch
assert "void _showOfflineBannerIfProbeFails(_browserReportsOnline()?'network':'browser');" in fetch_patch assert "void _showOfflineBannerIfProbeFails(_browserReportsOnline()?'network':'browser');" in fetch_patch
assert "_offlineFetchProbeFailures<OFFLINE_FETCH_FAILURES_BEFORE_BANNER" in UI_JS
assert "if(!_browserReportsOnline())showOfflineBanner('browser');" not in fetch_patch assert "if(!_browserReportsOnline())showOfflineBanner('browser');" not in fetch_patch
+61
View File
@@ -20,6 +20,37 @@ import pytest
REPO_ROOT = Path(__file__).resolve().parents[1] REPO_ROOT = Path(__file__).resolve().parents[1]
def _js_function_decl(src: str, name: str) -> str:
marker = f"function {name}("
start = src.find(marker)
assert start != -1, f"{name}() not found"
brace = src.find("){", start)
assert brace != -1, f"{name}() body not found"
brace += 1
depth = 1
i = brace + 1
in_str = None
escaped = False
while i < len(src) and depth:
ch = src[i]
if in_str:
if escaped:
escaped = False
elif ch == "\\":
escaped = True
elif ch == in_str:
in_str = None
elif ch in ('"', "'", "`"):
in_str = ch
elif ch == "{":
depth += 1
elif ch == "}":
depth -= 1
i += 1
assert depth == 0, f"{name}() body did not close"
return src[start:i]
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Module surface # Module surface
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -392,6 +423,36 @@ def test_frontend_opens_session_stream():
assert "stopSessionStream" in js assert "stopSessionStream" in js
def test_session_stream_pauses_while_chat_stream_is_active():
"""The active turn's chat SSE already carries live events for that session.
Keeping /api/session/stream open for the same sid at the same time burns a
Chrome same-origin connection slot and can starve ordinary /api/session
fetches on HTTP/1.1.
"""
js = (REPO_ROOT / "static" / "messages.js").read_text()
assert "function _chatStreamActiveForSession(sid)" in js
assert "function _suspendSessionStreamForLiveChat(sid)" in js
assert "function _resumeSessionStreamAfterLiveChat(sid)" in js
start_src = _js_function_decl(js, "startSessionStream")
assert "if (_chatStreamActiveForSession(sid))" in start_src
assert "_sessionStreamHiddenSid = sid;" in start_src
assert "return;" in start_src
assert start_src.index("if (_chatStreamActiveForSession(sid))") < start_src.index("new EventSource(")
attach_ix = js.index("function attachLiveStream")
attach_src = js[attach_ix:js.index("function transcript()", attach_ix)]
assert "_suspendSessionStreamForLiveChat(activeSid);" in attach_src
assert attach_src.index("_suspendSessionStreamForLiveChat(activeSid);") < attach_src.index("new EventSource(")
resume_src = _js_function_decl(js, "_resumeSessionStreamAfterLiveChat")
assert "S.session.session_id !== sid" in resume_src
assert "if (_chatStreamActiveForSession(sid)) return;" in resume_src
assert "_sessionStreamHiddenSid = null;" in resume_src
assert "startSessionStream(sid);" in resume_src
def test_frontend_busy_race_gate_obsoleted_by_option_z_pivot(): def test_frontend_busy_race_gate_obsoleted_by_option_z_pivot():
"""Per the Option Z PIVOT note baked into the handler body, the browser """Per the Option Z PIVOT note baked into the handler body, the browser
is no longer in the wakeup path at all — the server-side drain owns is no longer in the wakeup path at all — the server-side drain owns
+119 -3
View File
@@ -265,10 +265,126 @@ def test_session_list_cache_follower_reuses_stale_payload_during_slow_rebuild(mo
owner_thread.join(2.0) owner_thread.join(2.0)
follower_thread.join(2.0) follower_thread.join(2.0)
assert owner_result["payload"] == _session_cache_payload("stale")
assert follower_result["payload"] == _session_cache_payload("stale") assert follower_result["payload"] == _session_cache_payload("stale")
assert owner_result["payload"] == _session_cache_payload("fresh") assert "session_list_cache_stale_background_rebuild" in owner_diag.stages
assert "session_list_cache_wait_stale" in follower_diag.stages assert (
assert "session_list_cache_wait_stale_fallback" in follower_diag.stages "session_list_cache_stale_return" in follower_diag.stages
or "session_list_cache_stale_background_rebuild" in follower_diag.stages
)
def test_session_list_cache_owner_returns_stale_and_rebuilds_in_background(monkeypatch):
routes._session_list_cache_clear()
key = routes._session_list_cache_key(
active_profile="default",
all_profiles=False,
show_cli_sessions=False,
show_previous_messaging_sessions=False,
show_cron_sessions=False,
)
routes._session_list_cache_set(key, _session_cache_payload("stale"))
with routes._SESSIONS_CACHE_LOCK:
ts, stamp, payload = routes._SESSIONS_CACHE[key]
routes._SESSIONS_CACHE[key] = (
ts - routes._SESSIONS_CACHE_TTL_SECONDS - 1.0,
stamp,
payload,
)
monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda _key: ("changed",))
started = threading.Event()
release = threading.Event()
diag = _StageRecorder()
def builder():
started.set()
release.wait()
return _session_cache_payload("fresh")
try:
result = routes._get_cached_session_list_payload(key=key, builder=builder, diag=diag)
assert result == _session_cache_payload("stale")
assert started.wait(1.0), "stale owner should kick off a background rebuild"
assert "session_list_cache_stale_background_rebuild" in diag.stages
# While background rebuild is still blocked, stale is still returned.
cached, fresh = routes._session_list_cache_get(key, allow_stale=True)
assert cached == _session_cache_payload("stale")
assert fresh is False
finally:
release.set()
# The background owner should eventually populate fresh cache.
deadline = threading.Event()
for _ in range(20):
cached, fresh = routes._session_list_cache_get(key, allow_stale=True)
if cached == _session_cache_payload("fresh"):
break
deadline.wait(0.05)
assert cached == _session_cache_payload("fresh")
def test_session_list_cache_stale_background_rebuild_failure_releases_owner(monkeypatch):
routes._session_list_cache_clear()
key = routes._session_list_cache_key(
active_profile="default",
all_profiles=False,
show_cli_sessions=False,
show_previous_messaging_sessions=False,
show_cron_sessions=False,
)
routes._session_list_cache_set(key, _session_cache_payload("stale"))
with routes._SESSIONS_CACHE_LOCK:
ts, stamp, payload = routes._SESSIONS_CACHE[key]
routes._SESSIONS_CACHE[key] = (
ts - routes._SESSIONS_CACHE_TTL_SECONDS - 1.0,
stamp,
payload,
)
monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda _key: ("changed",))
started = threading.Event()
logged = []
def record_exception(message):
logged.append(message)
monkeypatch.setattr(routes.logger, "exception", record_exception)
def failing_builder():
started.set()
raise RuntimeError("boom")
result = routes._get_cached_session_list_payload(key=key, builder=failing_builder)
assert result == _session_cache_payload("stale")
assert started.wait(1.0), "background rebuild should have attempted the builder"
for _ in range(20):
with routes._SESSIONS_CACHE_LOCK:
inflight_empty = key not in routes._SESSIONS_CACHE_INFLIGHT
if inflight_empty:
break
threading.Event().wait(0.05)
assert inflight_empty, "failed background rebuild must release the singleflight owner"
assert logged == ["session list stale-cache background rebuild failed"]
cached, fresh = routes._session_list_cache_get(key, allow_stale=True)
assert cached == _session_cache_payload("stale")
assert fresh is False
def recovery_builder():
return _session_cache_payload("fresh")
# Once the failed owner releases, a later request can retry instead of being
# pinned behind the dead background rebuild.
routes._get_cached_session_list_payload(key=key, builder=recovery_builder)
for _ in range(20):
cached, _fresh = routes._session_list_cache_get(key, allow_stale=True)
if cached == _session_cache_payload("fresh"):
break
threading.Event().wait(0.05)
assert cached == _session_cache_payload("fresh")
def test_session_list_cache_invalidated_on_session_list_publish(): def test_session_list_cache_invalidated_on_session_list_publish():
+7
View File
@@ -59,8 +59,15 @@ def test_sessions_and_projects_load_independently_so_projects_failure_cannot_bla
def test_sessions_api_uses_longer_timeout_and_timeout_retry_for_boot_refresh(): def test_sessions_api_uses_longer_timeout_and_timeout_retry_for_boot_refresh():
sessions_src = _sessions_js() sessions_src = _sessions_js()
workspace_src = _workspace_js() workspace_src = _workspace_js()
helper_start = sessions_src.find("async function _loadSidebarSessionListPayload")
assert helper_start > 0
helper_end = sessions_src.find("async function _drainRenderSessionListQueue", helper_start)
assert helper_end > helper_start
helper = sessions_src[helper_start:helper_end]
assert "timeoutMs:_sessionListHasLoadedOnce?30000:_SESSION_LIST_BOOT_TIMEOUT_MS" in sessions_src assert "timeoutMs:_sessionListHasLoadedOnce?30000:_SESSION_LIST_BOOT_TIMEOUT_MS" in sessions_src
assert "const sessData = await api('/api/sessions' + sessionListQS,sessionRequestOpts);" in helper
assert "api('/api/sessions' + sessionListQS,{timeoutToast:false})" not in helper
assert "retryTimeouts:true" in sessions_src assert "retryTimeouts:true" in sessions_src
assert "retryStatuses:[502,503,504]" in sessions_src assert "retryStatuses:[502,503,504]" in sessions_src
assert "retryTimeouts" in workspace_src assert "retryTimeouts" in workspace_src