Reduce connection lost flapping under large session state
This commit is contained in:
1 parent
ce425c7127
commit
cbb644da69
12 files changed
+339
-23
No files matched your search
@@ -2068,6 +2068,52 @@ def _get_cached_session_list_payload(
|
|||||||
return cached
|
return cached
|
||||||
|
|
||||||
stale = cached # now actually a stale payload when one exists, else None
|
stale = cached # now actually a stale payload when one exists, else None
|
||||||
|
if stale is not None:
|
||||||
|
event, is_owner = _session_list_cache_claim_rebuild(key)
|
||||||
|
if is_owner:
|
||||||
|
if diag is not None:
|
||||||
|
try:
|
||||||
|
diag.stage("session_list_cache_stale_background_rebuild")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _rebuild_stale_session_list_cache():
|
||||||
|
try:
|
||||||
|
rebuild_attempts = 0
|
||||||
|
while True:
|
||||||
|
invalidation_stamp = _session_list_cache_invalidation_stamp(key)
|
||||||
|
try:
|
||||||
|
payload = builder()
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
"session list stale-cache background rebuild failed"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if _session_list_cache_invalidation_stamp(key) == invalidation_stamp:
|
||||||
|
_session_list_cache_set(key, payload)
|
||||||
|
return
|
||||||
|
rebuild_attempts += 1
|
||||||
|
if rebuild_attempts >= 3:
|
||||||
|
return
|
||||||
|
finally:
|
||||||
|
_session_list_cache_done(key, event)
|
||||||
|
|
||||||
|
try:
|
||||||
|
thread = threading.Thread(
|
||||||
|
target=_rebuild_stale_session_list_cache,
|
||||||
|
name="session-list-cache-rebuild",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
thread.start()
|
||||||
|
except Exception:
|
||||||
|
_session_list_cache_done(key, event)
|
||||||
|
elif diag is not None:
|
||||||
|
try:
|
||||||
|
diag.stage("session_list_cache_stale_return")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return stale
|
||||||
|
|
||||||
event, is_owner = _session_list_cache_claim_rebuild(key)
|
event, is_owner = _session_list_cache_claim_rebuild(key)
|
||||||
if is_owner:
|
if is_owner:
|
||||||
if diag is not None:
|
if diag is not None:
|
||||||
|
|||||||
@@ -1911,6 +1911,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
|
|||||||
const _startedAt=(S.session&&S.session.pending_started_at)||Date.now()/1000;
|
const _startedAt=(S.session&&S.session.pending_started_at)||Date.now()/1000;
|
||||||
showLiveRunStatus(activeSid,{startedAt:_startedAt});
|
showLiveRunStatus(activeSid,{startedAt:_startedAt});
|
||||||
}
|
}
|
||||||
|
_suspendSessionStreamForLiveChat(activeSid);
|
||||||
|
|
||||||
// On reconnect, restore accumulated text from INFLIGHT so we don't lose
|
// On reconnect, restore accumulated text from INFLIGHT so we don't lose
|
||||||
// progress made before the session switch. Without this the closure starts
|
// progress made before the session switch. Without this the closure starts
|
||||||
@@ -2018,6 +2019,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
|
|||||||
delete INFLIGHT[activeSid];
|
delete INFLIGHT[activeSid];
|
||||||
clearInflightState(activeSid);
|
clearInflightState(activeSid);
|
||||||
_clearActivePaneInflightIfOwner();
|
_clearActivePaneInflightIfOwner();
|
||||||
|
_resumeSessionStreamAfterLiveChat(activeSid);
|
||||||
}
|
}
|
||||||
function _isMarkerOnlyAssistantMessage(m){
|
function _isMarkerOnlyAssistantMessage(m){
|
||||||
if(!m||m.role!=='assistant') return false;
|
if(!m||m.role!=='assistant') return false;
|
||||||
@@ -6441,11 +6443,47 @@ function _stopHiddenActiveStreamPoll() {
|
|||||||
_sessionStreamHiddenPollSid = null;
|
_sessionStreamHiddenPollSid = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function _chatStreamActiveForSession(sid) {
|
||||||
|
if (!sid) return false;
|
||||||
|
if (typeof LIVE_STREAMS !== 'undefined' && LIVE_STREAMS[sid]) return true;
|
||||||
|
return !!(
|
||||||
|
S &&
|
||||||
|
S.session &&
|
||||||
|
S.session.session_id === sid &&
|
||||||
|
S.activeStreamId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function _suspendSessionStreamForLiveChat(sid) {
|
||||||
|
if (!sid) return;
|
||||||
|
if (_sessionStreamSessionId !== sid && !_sessionEventSource) return;
|
||||||
|
_sessionStreamHiddenSid = sid;
|
||||||
|
stopSessionStream();
|
||||||
|
}
|
||||||
|
|
||||||
|
function _resumeSessionStreamAfterLiveChat(sid) {
|
||||||
|
if (!sid) return;
|
||||||
|
setTimeout(() => {
|
||||||
|
if (!S || !S.session || S.session.session_id !== sid) return;
|
||||||
|
if (_chatStreamActiveForSession(sid)) return;
|
||||||
|
_sessionStreamHiddenSid = null;
|
||||||
|
startSessionStream(sid);
|
||||||
|
}, 0);
|
||||||
|
}
|
||||||
|
|
||||||
function startSessionStream(sid) {
|
function startSessionStream(sid) {
|
||||||
if (!sid) return;
|
if (!sid) return;
|
||||||
// Already on this session? No-op (loadSession is a no-op when re-selecting
|
// Already on this session? No-op (loadSession is a no-op when re-selecting
|
||||||
// the same session; this defends against external re-callers).
|
// the same session; this defends against external re-callers).
|
||||||
if (_sessionStreamSessionId === sid && _sessionEventSource) return;
|
if (_sessionStreamSessionId === sid && _sessionEventSource) return;
|
||||||
|
// While the visible conversation has a live token stream, /api/chat/stream
|
||||||
|
// already carries bg_task_complete and terminal events for this turn. Keeping
|
||||||
|
// /api/session/stream open at the same time burns one of Chrome's six
|
||||||
|
// same-origin HTTP/1.1 sockets and can starve ordinary /api/session fetches.
|
||||||
|
if (_chatStreamActiveForSession(sid)) {
|
||||||
|
_sessionStreamHiddenSid = sid;
|
||||||
|
return;
|
||||||
|
}
|
||||||
stopSessionStream();
|
stopSessionStream();
|
||||||
_sessionStreamSessionId = sid;
|
_sessionStreamSessionId = sid;
|
||||||
// Visibility hook (install once) — mirror ensureSessionEventsSSE() pattern.
|
// Visibility hook (install once) — mirror ensureSessionEventsSSE() pattern.
|
||||||
|
|||||||
+14
-7
@@ -2385,7 +2385,10 @@ async function _ensureMessagesLoaded(sid, opts) {
|
|||||||
const expandParam = reloadLimit ? '&expand_renderable=1' : '';
|
const expandParam = reloadLimit ? '&expand_renderable=1' : '';
|
||||||
let data;
|
let data;
|
||||||
try {
|
try {
|
||||||
data = await api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0${reloadLimitParam}${expandParam}`);
|
data = await api(
|
||||||
|
`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0${reloadLimitParam}${expandParam}`,
|
||||||
|
{timeoutMs:120000}
|
||||||
|
);
|
||||||
} finally {
|
} finally {
|
||||||
_clearSameSessionForceReloadHint(sid);
|
_clearSameSessionForceReloadHint(sid);
|
||||||
}
|
}
|
||||||
@@ -2869,7 +2872,10 @@ async function _loadOlderMessages() {
|
|||||||
// Cumulative growth: each "load more" asks for currentLoaded + 30, and the
|
// Cumulative growth: each "load more" asks for currentLoaded + 30, and the
|
||||||
// newly exposed head is what we expose to the user.
|
// newly exposed head is what we expose to the user.
|
||||||
const requestedLimit = Math.max(_INITIAL_MSG_LIMIT, (S.messages || []).length + _INITIAL_MSG_LIMIT);
|
const requestedLimit = Math.max(_INITIAL_MSG_LIMIT, (S.messages || []).length + _INITIAL_MSG_LIMIT);
|
||||||
const data = await api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_limit=${requestedLimit}`);
|
const data = await api(
|
||||||
|
`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_limit=${requestedLimit}`,
|
||||||
|
{timeoutMs:120000}
|
||||||
|
);
|
||||||
// Guard: api() may have redirected (401) and returned undefined.
|
// Guard: api() may have redirected (401) and returned undefined.
|
||||||
if (!data || !data.session) { _loadingOlder = false; return; }
|
if (!data || !data.session) { _loadingOlder = false; return; }
|
||||||
// - response shape sane
|
// - response shape sane
|
||||||
@@ -2913,7 +2919,10 @@ async function _loadOlderMessages() {
|
|||||||
// Race fallback: keep the legacy index-page request as the
|
// Race fallback: keep the legacy index-page request as the
|
||||||
// correctness-preserving alternative. Same guards reapplied because
|
// correctness-preserving alternative. Same guards reapplied because
|
||||||
// we just awaited again.
|
// we just awaited again.
|
||||||
const fallback = await api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_before=${_oldestIdx}&msg_limit=${_INITIAL_MSG_LIMIT}`);
|
const fallback = await api(
|
||||||
|
`/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_before=${_oldestIdx}&msg_limit=${_INITIAL_MSG_LIMIT}`,
|
||||||
|
{timeoutMs:120000}
|
||||||
|
);
|
||||||
if (!fallback || !fallback.session) { _loadingOlder = false; return; }
|
if (!fallback || !fallback.session) { _loadingOlder = false; return; }
|
||||||
if (!S.session || S.session.session_id !== sid) return;
|
if (!S.session || S.session.session_id !== sid) return;
|
||||||
if (_loadingSessionId !== null && _loadingSessionId !== sid) return;
|
if (_loadingSessionId !== null && _loadingSessionId !== sid) return;
|
||||||
@@ -4362,9 +4371,7 @@ async function _loadSidebarSessionListPayload(sessionListQS, sessionRequestOpts)
|
|||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
|
|
||||||
const sessData = _sessionListHasLoadedOnce
|
const sessData = await api('/api/sessions' + sessionListQS,sessionRequestOpts);
|
||||||
? await api('/api/sessions' + sessionListQS,{timeoutToast:false})
|
|
||||||
: await api('/api/sessions' + sessionListQS,sessionRequestOpts);
|
|
||||||
const projData = await projectPromise;
|
const projData = await projectPromise;
|
||||||
|
|
||||||
return {sessData,projData};
|
return {sessData,projData};
|
||||||
@@ -4407,7 +4414,7 @@ let _gatewayPollTimer = null;
|
|||||||
let _gatewayProbeInFlight = false;
|
let _gatewayProbeInFlight = false;
|
||||||
let _gatewaySSEWarningShown = false;
|
let _gatewaySSEWarningShown = false;
|
||||||
const _gatewayFallbackPollMs = 30000;
|
const _gatewayFallbackPollMs = 30000;
|
||||||
const _streamingPollMs = 5000;
|
const _streamingPollMs = 30000;
|
||||||
const _sessionTimeRefreshMs = 60000;
|
const _sessionTimeRefreshMs = 60000;
|
||||||
// #3107: the active-session "is it externally updated?" poll used to fire
|
// #3107: the active-session "is it externally updated?" poll used to fire
|
||||||
// every 5 s. On long sessions this caused visible scroll jitter and a
|
// every 5 s. On long sessions this caused visible scroll jitter and a
|
||||||
|
|||||||
+15
-5
@@ -24,12 +24,15 @@ const MAX_UPLOAD_MB=Math.round(MAX_UPLOAD_BYTES/1024/1024);
|
|||||||
let _queueDrainSid=null;
|
let _queueDrainSid=null;
|
||||||
const $=id=>document.getElementById(id);
|
const $=id=>document.getElementById(id);
|
||||||
const OFFLINE_RECHECK_MS=2500;
|
const OFFLINE_RECHECK_MS=2500;
|
||||||
|
const OFFLINE_HEALTH_TIMEOUT_MS=10000;
|
||||||
|
const OFFLINE_FETCH_FAILURES_BEFORE_BANNER=2;
|
||||||
let _offlineVisible=false;
|
let _offlineVisible=false;
|
||||||
let _offlineReason='browser';
|
let _offlineReason='browser';
|
||||||
let _offlineProbeTimer=null;
|
let _offlineProbeTimer=null;
|
||||||
let _offlineChecking=false;
|
let _offlineChecking=false;
|
||||||
let _offlineProbePromise=null;
|
let _offlineProbePromise=null;
|
||||||
let _offlineHealthProbePromise=null;
|
let _offlineHealthProbePromise=null;
|
||||||
|
let _offlineFetchProbeFailures=0;
|
||||||
let _offlineRawFetch=null;
|
let _offlineRawFetch=null;
|
||||||
let _offlineFetchPatched=false;
|
let _offlineFetchPatched=false;
|
||||||
function _browserReportsOnline(){return !('onLine' in navigator)||navigator.onLine!==false;}
|
function _browserReportsOnline(){return !('onLine' in navigator)||navigator.onLine!==false;}
|
||||||
@@ -82,7 +85,7 @@ async function _probeOfflineRecovery(){
|
|||||||
// the initial banner display on the offline-event/startup paths.
|
// the initial banner display on the offline-event/startup paths.
|
||||||
let ctrl=null,timer=null;
|
let ctrl=null,timer=null;
|
||||||
try{ctrl=(typeof AbortController!=='undefined')?new AbortController():null;}catch(_){ctrl=null;}
|
try{ctrl=(typeof AbortController!=='undefined')?new AbortController():null;}catch(_){ctrl=null;}
|
||||||
if(ctrl)timer=setTimeout(()=>{try{ctrl.abort();}catch(_){}},3000);
|
if(ctrl)timer=setTimeout(()=>{try{ctrl.abort();}catch(_){}},OFFLINE_HEALTH_TIMEOUT_MS);
|
||||||
try{
|
try{
|
||||||
const opts={cache:'no-store',credentials:'include'};
|
const opts={cache:'no-store',credentials:'include'};
|
||||||
if(ctrl)opts.signal=ctrl.signal;
|
if(ctrl)opts.signal=ctrl.signal;
|
||||||
@@ -94,15 +97,22 @@ async function _probeOfflineRecovery(){
|
|||||||
try{return await _offlineHealthProbePromise;}
|
try{return await _offlineHealthProbePromise;}
|
||||||
finally{_offlineHealthProbePromise=null;}
|
finally{_offlineHealthProbePromise=null;}
|
||||||
}
|
}
|
||||||
async function _showOfflineBannerIfProbeFails(reason){
|
async function _showOfflineBannerIfProbeFails(reason,opts){
|
||||||
|
opts=opts||{};
|
||||||
const visibleAtStart=_offlineVisible;
|
const visibleAtStart=_offlineVisible;
|
||||||
|
const requireConsecutiveFailures=opts.requireConsecutiveFailures!==false;
|
||||||
if(visibleAtStart)_setOfflineChecking(true);
|
if(visibleAtStart)_setOfflineChecking(true);
|
||||||
const ok=await _probeOfflineRecovery();
|
const ok=await _probeOfflineRecovery();
|
||||||
if(visibleAtStart)_setOfflineChecking(false);
|
if(visibleAtStart)_setOfflineChecking(false);
|
||||||
if(ok){
|
if(ok){
|
||||||
|
_offlineFetchProbeFailures=0;
|
||||||
if(_offlineVisible){_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();}
|
if(_offlineVisible){_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
if(!visibleAtStart&&requireConsecutiveFailures){
|
||||||
|
_offlineFetchProbeFailures+=1;
|
||||||
|
if(_offlineFetchProbeFailures<OFFLINE_FETCH_FAILURES_BEFORE_BANNER)return false;
|
||||||
|
}
|
||||||
showOfflineBanner(reason||(_browserReportsOnline()?'network':'browser'));
|
showOfflineBanner(reason||(_browserReportsOnline()?'network':'browser'));
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -113,7 +123,7 @@ async function checkOfflineRecoveryNow(){
|
|||||||
_setOfflineChecking(true);
|
_setOfflineChecking(true);
|
||||||
const ok=await _probeOfflineRecovery();
|
const ok=await _probeOfflineRecovery();
|
||||||
_setOfflineChecking(false);
|
_setOfflineChecking(false);
|
||||||
if(ok){if(!_offlineVisible)return true;_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();return true;}
|
if(ok){_offlineFetchProbeFailures=0;if(!_offlineVisible)return true;_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();return true;}
|
||||||
showOfflineBanner(_browserReportsOnline()?'network':'browser');
|
showOfflineBanner(_browserReportsOnline()?'network':'browser');
|
||||||
return false;
|
return false;
|
||||||
})();
|
})();
|
||||||
@@ -182,9 +192,9 @@ function _patchOfflineFetch(){
|
|||||||
}
|
}
|
||||||
function initOfflineMonitor(){
|
function initOfflineMonitor(){
|
||||||
_patchOfflineFetch();
|
_patchOfflineFetch();
|
||||||
window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser');});
|
window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});});
|
||||||
window.addEventListener('online',()=>{if(_offlineVisible)checkOfflineRecoveryNow();});
|
window.addEventListener('online',()=>{if(_offlineVisible)checkOfflineRecoveryNow();});
|
||||||
if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser');
|
if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});
|
||||||
}
|
}
|
||||||
if(document.readyState==='loading')document.addEventListener('DOMContentLoaded',initOfflineMonitor,{once:true});
|
if(document.readyState==='loading')document.addEventListener('DOMContentLoaded',initOfflineMonitor,{once:true});
|
||||||
else initOfflineMonitor();
|
else initOfflineMonitor();
|
||||||
|
|||||||
@@ -37,17 +37,20 @@ def test_recovery_loop_probes_even_when_browser_reports_offline():
|
|||||||
assert "_probeOfflineRecovery()" in body
|
assert "_probeOfflineRecovery()" in body
|
||||||
assert "if(!_browserReportsOnline()){showOfflineBanner('browser');return false;}" not in body
|
assert "if(!_browserReportsOnline()){showOfflineBanner('browser');return false;}" not in body
|
||||||
assert body.find("_probeOfflineRecovery()") < body.find("showOfflineBanner(")
|
assert body.find("_probeOfflineRecovery()") < body.find("showOfflineBanner(")
|
||||||
assert "if(ok){if(!_offlineVisible)return true;_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();return true;}" in body
|
assert "if(ok){_offlineFetchProbeFailures=0;if(!_offlineVisible)return true;_stopOfflineProbeTimer();await _recoverFromOfflineSoftly();return true;}" in body
|
||||||
assert "showOfflineBanner(_browserReportsOnline()?'network':'browser')" in body
|
assert "showOfflineBanner(_browserReportsOnline()?'network':'browser')" in body
|
||||||
|
|
||||||
|
|
||||||
def test_probe_backed_browser_banner_helper_uses_health_as_source_of_truth():
|
def test_probe_backed_browser_banner_helper_uses_health_as_source_of_truth():
|
||||||
body = _strip_comments(_fn_body(UI_JS, "async function _showOfflineBannerIfProbeFails("))
|
body = _strip_comments(_fn_body(UI_JS, "async function _showOfflineBannerIfProbeFails("))
|
||||||
|
assert "opts=opts||{};" in body
|
||||||
assert "const visibleAtStart=_offlineVisible;" in body
|
assert "const visibleAtStart=_offlineVisible;" in body
|
||||||
|
assert "const requireConsecutiveFailures=opts.requireConsecutiveFailures!==false;" in body
|
||||||
assert "if(visibleAtStart)_setOfflineChecking(true);" in body
|
assert "if(visibleAtStart)_setOfflineChecking(true);" in body
|
||||||
assert "const ok=await _probeOfflineRecovery();" in body
|
assert "const ok=await _probeOfflineRecovery();" in body
|
||||||
assert "if(visibleAtStart)_setOfflineChecking(false);" in body
|
assert "if(visibleAtStart)_setOfflineChecking(false);" in body
|
||||||
assert "if(ok)" in body
|
assert "if(ok)" in body
|
||||||
|
assert "_offlineFetchProbeFailures<OFFLINE_FETCH_FAILURES_BEFORE_BANNER" in body
|
||||||
assert "showOfflineBanner(reason||(_browserReportsOnline()?'network':'browser'))" in body
|
assert "showOfflineBanner(reason||(_browserReportsOnline()?'network':'browser'))" in body
|
||||||
assert body.find("_probeOfflineRecovery()") < body.find("showOfflineBanner(")
|
assert body.find("_probeOfflineRecovery()") < body.find("showOfflineBanner(")
|
||||||
|
|
||||||
@@ -56,8 +59,8 @@ def test_offline_event_and_startup_false_online_probe_before_browser_banner():
|
|||||||
body = _strip_comments(_fn_body(UI_JS, "function initOfflineMonitor("))
|
body = _strip_comments(_fn_body(UI_JS, "function initOfflineMonitor("))
|
||||||
assert "window.addEventListener('offline',()=>showOfflineBanner('browser'))" not in body
|
assert "window.addEventListener('offline',()=>showOfflineBanner('browser'))" not in body
|
||||||
assert "if(!_browserReportsOnline())showOfflineBanner('browser');" not in body
|
assert "if(!_browserReportsOnline())showOfflineBanner('browser');" not in body
|
||||||
assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser');});" in body
|
assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});});" in body
|
||||||
assert "if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser');" in body
|
assert "if(!_browserReportsOnline())void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});" in body
|
||||||
|
|
||||||
|
|
||||||
def test_fetch_catch_probes_before_browser_banner_and_rethrows():
|
def test_fetch_catch_probes_before_browser_banner_and_rethrows():
|
||||||
|
|||||||
@@ -217,6 +217,29 @@ def test_update_flows_keep_explicit_longer_timeouts():
|
|||||||
assert "api('/api/updates/force',{method:'POST',body:JSON.stringify({target}),timeoutMs:120000})" in src
|
assert "api('/api/updates/force',{method:'POST',body:JSON.stringify({target}),timeoutMs:120000})" in src
|
||||||
|
|
||||||
|
|
||||||
|
def test_session_message_loads_keep_explicit_longer_timeouts():
|
||||||
|
"""Large state.db installs can take longer than the generic 30s API timeout."""
|
||||||
|
src = _source(SESSIONS_JS)
|
||||||
|
assert (
|
||||||
|
"api(\n"
|
||||||
|
" `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0${reloadLimitParam}${expandParam}`,\n"
|
||||||
|
" {timeoutMs:120000}\n"
|
||||||
|
" )"
|
||||||
|
) in src
|
||||||
|
assert (
|
||||||
|
"api(\n"
|
||||||
|
" `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_limit=${requestedLimit}`,\n"
|
||||||
|
" {timeoutMs:120000}\n"
|
||||||
|
" )"
|
||||||
|
) in src
|
||||||
|
assert (
|
||||||
|
"api(\n"
|
||||||
|
" `/api/session?session_id=${encodeURIComponent(sid)}&messages=1&resolve_model=0&msg_before=${_oldestIdx}&msg_limit=${_INITIAL_MSG_LIMIT}`,\n"
|
||||||
|
" {timeoutMs:120000}\n"
|
||||||
|
" )"
|
||||||
|
) in src
|
||||||
|
|
||||||
|
|
||||||
def test_passive_background_polls_suppress_timeout_toasts():
|
def test_passive_background_polls_suppress_timeout_toasts():
|
||||||
"""Passive refreshes should be best-effort and not emit generic timeout toasts."""
|
"""Passive refreshes should be best-effort and not emit generic timeout toasts."""
|
||||||
workspace = _source(WORKSPACE_JS)
|
workspace = _source(WORKSPACE_JS)
|
||||||
@@ -226,7 +249,9 @@ def test_passive_background_polls_suppress_timeout_toasts():
|
|||||||
panels = _source(PANELS_JS)
|
panels = _source(PANELS_JS)
|
||||||
|
|
||||||
assert "api('/api/client-events/log',{method:'POST',body:JSON.stringify(payload),timeoutMs:3000,timeoutToast:false})" in workspace
|
assert "api('/api/client-events/log',{method:'POST',body:JSON.stringify(payload),timeoutMs:3000,timeoutToast:false})" in workspace
|
||||||
assert "api('/api/sessions' + sessionListQS,{timeoutToast:false})" in sessions
|
assert "const sessionRequestOpts={" in sessions
|
||||||
|
assert "timeoutToast:false" in sessions
|
||||||
|
assert "api('/api/sessions' + sessionListQS,sessionRequestOpts)" in sessions
|
||||||
assert "api('/api/projects' + projectQS,{timeoutToast:false})" in sessions
|
assert "api('/api/projects' + projectQS,{timeoutToast:false})" in sessions
|
||||||
assert "api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=0&resolve_model=0`,{timeoutToast:false})" in sessions
|
assert "api(`/api/session?session_id=${encodeURIComponent(sid)}&messages=0&resolve_model=0`,{timeoutToast:false})" in sessions
|
||||||
assert 'api("/api/approval/pending?session_id=" + encodeURIComponent(sid),{timeoutToast:false})' in messages
|
assert 'api("/api/approval/pending?session_id=" + encodeURIComponent(sid),{timeoutToast:false})' in messages
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ def test_session_stream_has_visibility_hook():
|
|||||||
assert "_hermesSessionStreamVisibilityHook" in MESSAGES_JS
|
assert "_hermesSessionStreamVisibilityHook" in MESSAGES_JS
|
||||||
start_idx = MESSAGES_JS.find("function startSessionStream(sid)")
|
start_idx = MESSAGES_JS.find("function startSessionStream(sid)")
|
||||||
assert start_idx != -1
|
assert start_idx != -1
|
||||||
block = MESSAGES_JS[start_idx:start_idx + 2400]
|
block = MESSAGES_JS[start_idx:start_idx + 2600]
|
||||||
assert "visibilitychange" in block
|
assert "visibilitychange" in block
|
||||||
assert "document.hidden" in block
|
assert "document.hidden" in block
|
||||||
# Must skip opening a new EventSource while the tab is hidden.
|
# Must skip opening a new EventSource while the tab is hidden.
|
||||||
@@ -95,7 +95,7 @@ def test_session_stream_hidden_open_preserves_id_for_reopen():
|
|||||||
_sessionStreamHiddenSid = sid before returning.
|
_sessionStreamHiddenSid = sid before returning.
|
||||||
"""
|
"""
|
||||||
start_idx = MESSAGES_JS.find("function startSessionStream(sid)")
|
start_idx = MESSAGES_JS.find("function startSessionStream(sid)")
|
||||||
block = MESSAGES_JS[start_idx:start_idx + 2400]
|
block = MESSAGES_JS[start_idx:start_idx + 2800]
|
||||||
# Find the hidden-tab early-return skip path (distinct from the in-hook
|
# Find the hidden-tab early-return skip path (distinct from the in-hook
|
||||||
# `if (document.hidden)` branch) and assert it preserves the id.
|
# `if (document.hidden)` branch) and assert it preserves the id.
|
||||||
hidden_idx = block.find("!== 'undefined' && document.hidden) {")
|
hidden_idx = block.find("!== 'undefined' && document.hidden) {")
|
||||||
@@ -104,4 +104,3 @@ def test_session_stream_hidden_open_preserves_id_for_reopen():
|
|||||||
assert "_sessionStreamHiddenSid = sid" in hidden_block, (
|
assert "_sessionStreamHiddenSid = sid" in hidden_block, (
|
||||||
"hidden-tab skip must record the pending session id for reopen on re-show"
|
"hidden-tab skip must record the pending session id for reopen on re-show"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -160,6 +160,7 @@ def test_polling_transition_marks_completion_unread_without_sse_done():
|
|||||||
)
|
)
|
||||||
assert "_markSessionCompletionUnread(sid, s.message_count);" in transition_block
|
assert "_markSessionCompletionUnread(sid, s.message_count);" in transition_block
|
||||||
assert "_sessionStreamingById.set(sid, isStreaming);" in transition_block
|
assert "_sessionStreamingById.set(sid, isStreaming);" in transition_block
|
||||||
|
assert "const _streamingPollMs = 30000;" in SESSIONS_JS
|
||||||
assert "_applySessionListPayload(sessData,projData);" in refresh_block
|
assert "_applySessionListPayload(sessData,projData);" in refresh_block
|
||||||
assert "_markPollingCompletionUnreadTransitions(_allSessions);" in apply_block
|
assert "_markPollingCompletionUnreadTransitions(_allSessions);" in apply_block
|
||||||
assert "_allSessions.some(s => _isSessionEffectivelyStreaming(s))" in apply_block, (
|
assert "_allSessions.some(s => _isSessionEffectivelyStreaming(s))" in apply_block, (
|
||||||
|
|||||||
@@ -35,8 +35,10 @@ def test_offline_banner_markup_styles_and_copy_exist():
|
|||||||
|
|
||||||
def test_offline_monitor_patches_fetch_and_auto_reloads_after_health_probe():
|
def test_offline_monitor_patches_fetch_and_auto_reloads_after_health_probe():
|
||||||
assert "const OFFLINE_RECHECK_MS=2500" in UI_JS
|
assert "const OFFLINE_RECHECK_MS=2500" in UI_JS
|
||||||
|
assert "const OFFLINE_HEALTH_TIMEOUT_MS=10000" in UI_JS
|
||||||
|
assert "const OFFLINE_FETCH_FAILURES_BEFORE_BANNER=2" in UI_JS
|
||||||
assert "window.fetch=async function(...args)" in UI_JS
|
assert "window.fetch=async function(...args)" in UI_JS
|
||||||
assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser');});" in UI_JS
|
assert "window.addEventListener('offline',()=>{void _showOfflineBannerIfProbeFails('browser',{requireConsecutiveFailures:false});});" in UI_JS
|
||||||
assert "window.addEventListener('online',()=>{if(_offlineVisible)checkOfflineRecoveryNow();})" in UI_JS
|
assert "window.addEventListener('online',()=>{if(_offlineVisible)checkOfflineRecoveryNow();})" in UI_JS
|
||||||
assert "setInterval(()=>{checkOfflineRecoveryNow();},OFFLINE_RECHECK_MS)" in UI_JS
|
assert "setInterval(()=>{checkOfflineRecoveryNow();},OFFLINE_RECHECK_MS)" in UI_JS
|
||||||
assert "new URL('health',document.baseURI||location.href)" in UI_JS
|
assert "new URL('health',document.baseURI||location.href)" in UI_JS
|
||||||
@@ -60,6 +62,7 @@ def test_fetch_typeerror_is_gated_by_health_probe_not_blind_banner():
|
|||||||
assert "function _isAbortError(e)" in UI_JS
|
assert "function _isAbortError(e)" in UI_JS
|
||||||
assert "!_isAbortError(e)&&(e instanceof TypeError||!_browserReportsOnline())" in fetch_patch
|
assert "!_isAbortError(e)&&(e instanceof TypeError||!_browserReportsOnline())" in fetch_patch
|
||||||
assert "void _showOfflineBannerIfProbeFails(_browserReportsOnline()?'network':'browser');" in fetch_patch
|
assert "void _showOfflineBannerIfProbeFails(_browserReportsOnline()?'network':'browser');" in fetch_patch
|
||||||
|
assert "_offlineFetchProbeFailures<OFFLINE_FETCH_FAILURES_BEFORE_BANNER" in UI_JS
|
||||||
assert "if(!_browserReportsOnline())showOfflineBanner('browser');" not in fetch_patch
|
assert "if(!_browserReportsOnline())showOfflineBanner('browser');" not in fetch_patch
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,37 @@ import pytest
|
|||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def _js_function_decl(src: str, name: str) -> str:
|
||||||
|
marker = f"function {name}("
|
||||||
|
start = src.find(marker)
|
||||||
|
assert start != -1, f"{name}() not found"
|
||||||
|
brace = src.find("){", start)
|
||||||
|
assert brace != -1, f"{name}() body not found"
|
||||||
|
brace += 1
|
||||||
|
depth = 1
|
||||||
|
i = brace + 1
|
||||||
|
in_str = None
|
||||||
|
escaped = False
|
||||||
|
while i < len(src) and depth:
|
||||||
|
ch = src[i]
|
||||||
|
if in_str:
|
||||||
|
if escaped:
|
||||||
|
escaped = False
|
||||||
|
elif ch == "\\":
|
||||||
|
escaped = True
|
||||||
|
elif ch == in_str:
|
||||||
|
in_str = None
|
||||||
|
elif ch in ('"', "'", "`"):
|
||||||
|
in_str = ch
|
||||||
|
elif ch == "{":
|
||||||
|
depth += 1
|
||||||
|
elif ch == "}":
|
||||||
|
depth -= 1
|
||||||
|
i += 1
|
||||||
|
assert depth == 0, f"{name}() body did not close"
|
||||||
|
return src[start:i]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Module surface
|
# Module surface
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -392,6 +423,36 @@ def test_frontend_opens_session_stream():
|
|||||||
assert "stopSessionStream" in js
|
assert "stopSessionStream" in js
|
||||||
|
|
||||||
|
|
||||||
|
def test_session_stream_pauses_while_chat_stream_is_active():
|
||||||
|
"""The active turn's chat SSE already carries live events for that session.
|
||||||
|
|
||||||
|
Keeping /api/session/stream open for the same sid at the same time burns a
|
||||||
|
Chrome same-origin connection slot and can starve ordinary /api/session
|
||||||
|
fetches on HTTP/1.1.
|
||||||
|
"""
|
||||||
|
js = (REPO_ROOT / "static" / "messages.js").read_text()
|
||||||
|
assert "function _chatStreamActiveForSession(sid)" in js
|
||||||
|
assert "function _suspendSessionStreamForLiveChat(sid)" in js
|
||||||
|
assert "function _resumeSessionStreamAfterLiveChat(sid)" in js
|
||||||
|
|
||||||
|
start_src = _js_function_decl(js, "startSessionStream")
|
||||||
|
assert "if (_chatStreamActiveForSession(sid))" in start_src
|
||||||
|
assert "_sessionStreamHiddenSid = sid;" in start_src
|
||||||
|
assert "return;" in start_src
|
||||||
|
assert start_src.index("if (_chatStreamActiveForSession(sid))") < start_src.index("new EventSource(")
|
||||||
|
|
||||||
|
attach_ix = js.index("function attachLiveStream")
|
||||||
|
attach_src = js[attach_ix:js.index("function transcript()", attach_ix)]
|
||||||
|
assert "_suspendSessionStreamForLiveChat(activeSid);" in attach_src
|
||||||
|
assert attach_src.index("_suspendSessionStreamForLiveChat(activeSid);") < attach_src.index("new EventSource(")
|
||||||
|
|
||||||
|
resume_src = _js_function_decl(js, "_resumeSessionStreamAfterLiveChat")
|
||||||
|
assert "S.session.session_id !== sid" in resume_src
|
||||||
|
assert "if (_chatStreamActiveForSession(sid)) return;" in resume_src
|
||||||
|
assert "_sessionStreamHiddenSid = null;" in resume_src
|
||||||
|
assert "startSessionStream(sid);" in resume_src
|
||||||
|
|
||||||
|
|
||||||
def test_frontend_busy_race_gate_obsoleted_by_option_z_pivot():
|
def test_frontend_busy_race_gate_obsoleted_by_option_z_pivot():
|
||||||
"""Per the Option Z PIVOT note baked into the handler body, the browser
|
"""Per the Option Z PIVOT note baked into the handler body, the browser
|
||||||
is no longer in the wakeup path at all — the server-side drain owns
|
is no longer in the wakeup path at all — the server-side drain owns
|
||||||
|
|||||||
@@ -265,10 +265,126 @@ def test_session_list_cache_follower_reuses_stale_payload_during_slow_rebuild(mo
|
|||||||
owner_thread.join(2.0)
|
owner_thread.join(2.0)
|
||||||
follower_thread.join(2.0)
|
follower_thread.join(2.0)
|
||||||
|
|
||||||
|
assert owner_result["payload"] == _session_cache_payload("stale")
|
||||||
assert follower_result["payload"] == _session_cache_payload("stale")
|
assert follower_result["payload"] == _session_cache_payload("stale")
|
||||||
assert owner_result["payload"] == _session_cache_payload("fresh")
|
assert "session_list_cache_stale_background_rebuild" in owner_diag.stages
|
||||||
assert "session_list_cache_wait_stale" in follower_diag.stages
|
assert (
|
||||||
assert "session_list_cache_wait_stale_fallback" in follower_diag.stages
|
"session_list_cache_stale_return" in follower_diag.stages
|
||||||
|
or "session_list_cache_stale_background_rebuild" in follower_diag.stages
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_session_list_cache_owner_returns_stale_and_rebuilds_in_background(monkeypatch):
|
||||||
|
routes._session_list_cache_clear()
|
||||||
|
|
||||||
|
key = routes._session_list_cache_key(
|
||||||
|
active_profile="default",
|
||||||
|
all_profiles=False,
|
||||||
|
show_cli_sessions=False,
|
||||||
|
show_previous_messaging_sessions=False,
|
||||||
|
show_cron_sessions=False,
|
||||||
|
)
|
||||||
|
routes._session_list_cache_set(key, _session_cache_payload("stale"))
|
||||||
|
with routes._SESSIONS_CACHE_LOCK:
|
||||||
|
ts, stamp, payload = routes._SESSIONS_CACHE[key]
|
||||||
|
routes._SESSIONS_CACHE[key] = (
|
||||||
|
ts - routes._SESSIONS_CACHE_TTL_SECONDS - 1.0,
|
||||||
|
stamp,
|
||||||
|
payload,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda _key: ("changed",))
|
||||||
|
|
||||||
|
started = threading.Event()
|
||||||
|
release = threading.Event()
|
||||||
|
diag = _StageRecorder()
|
||||||
|
|
||||||
|
def builder():
|
||||||
|
started.set()
|
||||||
|
release.wait()
|
||||||
|
return _session_cache_payload("fresh")
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = routes._get_cached_session_list_payload(key=key, builder=builder, diag=diag)
|
||||||
|
assert result == _session_cache_payload("stale")
|
||||||
|
assert started.wait(1.0), "stale owner should kick off a background rebuild"
|
||||||
|
assert "session_list_cache_stale_background_rebuild" in diag.stages
|
||||||
|
# While background rebuild is still blocked, stale is still returned.
|
||||||
|
cached, fresh = routes._session_list_cache_get(key, allow_stale=True)
|
||||||
|
assert cached == _session_cache_payload("stale")
|
||||||
|
assert fresh is False
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
|
||||||
|
# The background owner should eventually populate fresh cache.
|
||||||
|
deadline = threading.Event()
|
||||||
|
for _ in range(20):
|
||||||
|
cached, fresh = routes._session_list_cache_get(key, allow_stale=True)
|
||||||
|
if cached == _session_cache_payload("fresh"):
|
||||||
|
break
|
||||||
|
deadline.wait(0.05)
|
||||||
|
assert cached == _session_cache_payload("fresh")
|
||||||
|
|
||||||
|
|
||||||
|
def test_session_list_cache_stale_background_rebuild_failure_releases_owner(monkeypatch):
|
||||||
|
routes._session_list_cache_clear()
|
||||||
|
|
||||||
|
key = routes._session_list_cache_key(
|
||||||
|
active_profile="default",
|
||||||
|
all_profiles=False,
|
||||||
|
show_cli_sessions=False,
|
||||||
|
show_previous_messaging_sessions=False,
|
||||||
|
show_cron_sessions=False,
|
||||||
|
)
|
||||||
|
routes._session_list_cache_set(key, _session_cache_payload("stale"))
|
||||||
|
with routes._SESSIONS_CACHE_LOCK:
|
||||||
|
ts, stamp, payload = routes._SESSIONS_CACHE[key]
|
||||||
|
routes._SESSIONS_CACHE[key] = (
|
||||||
|
ts - routes._SESSIONS_CACHE_TTL_SECONDS - 1.0,
|
||||||
|
stamp,
|
||||||
|
payload,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda _key: ("changed",))
|
||||||
|
|
||||||
|
started = threading.Event()
|
||||||
|
logged = []
|
||||||
|
|
||||||
|
def record_exception(message):
|
||||||
|
logged.append(message)
|
||||||
|
|
||||||
|
monkeypatch.setattr(routes.logger, "exception", record_exception)
|
||||||
|
|
||||||
|
def failing_builder():
|
||||||
|
started.set()
|
||||||
|
raise RuntimeError("boom")
|
||||||
|
|
||||||
|
result = routes._get_cached_session_list_payload(key=key, builder=failing_builder)
|
||||||
|
|
||||||
|
assert result == _session_cache_payload("stale")
|
||||||
|
assert started.wait(1.0), "background rebuild should have attempted the builder"
|
||||||
|
for _ in range(20):
|
||||||
|
with routes._SESSIONS_CACHE_LOCK:
|
||||||
|
inflight_empty = key not in routes._SESSIONS_CACHE_INFLIGHT
|
||||||
|
if inflight_empty:
|
||||||
|
break
|
||||||
|
threading.Event().wait(0.05)
|
||||||
|
assert inflight_empty, "failed background rebuild must release the singleflight owner"
|
||||||
|
assert logged == ["session list stale-cache background rebuild failed"]
|
||||||
|
cached, fresh = routes._session_list_cache_get(key, allow_stale=True)
|
||||||
|
assert cached == _session_cache_payload("stale")
|
||||||
|
assert fresh is False
|
||||||
|
|
||||||
|
def recovery_builder():
|
||||||
|
return _session_cache_payload("fresh")
|
||||||
|
|
||||||
|
# Once the failed owner releases, a later request can retry instead of being
|
||||||
|
# pinned behind the dead background rebuild.
|
||||||
|
routes._get_cached_session_list_payload(key=key, builder=recovery_builder)
|
||||||
|
for _ in range(20):
|
||||||
|
cached, _fresh = routes._session_list_cache_get(key, allow_stale=True)
|
||||||
|
if cached == _session_cache_payload("fresh"):
|
||||||
|
break
|
||||||
|
threading.Event().wait(0.05)
|
||||||
|
assert cached == _session_cache_payload("fresh")
|
||||||
|
|
||||||
|
|
||||||
def test_session_list_cache_invalidated_on_session_list_publish():
|
def test_session_list_cache_invalidated_on_session_list_publish():
|
||||||
|
|||||||
@@ -59,8 +59,15 @@ def test_sessions_and_projects_load_independently_so_projects_failure_cannot_bla
|
|||||||
def test_sessions_api_uses_longer_timeout_and_timeout_retry_for_boot_refresh():
|
def test_sessions_api_uses_longer_timeout_and_timeout_retry_for_boot_refresh():
|
||||||
sessions_src = _sessions_js()
|
sessions_src = _sessions_js()
|
||||||
workspace_src = _workspace_js()
|
workspace_src = _workspace_js()
|
||||||
|
helper_start = sessions_src.find("async function _loadSidebarSessionListPayload")
|
||||||
|
assert helper_start > 0
|
||||||
|
helper_end = sessions_src.find("async function _drainRenderSessionListQueue", helper_start)
|
||||||
|
assert helper_end > helper_start
|
||||||
|
helper = sessions_src[helper_start:helper_end]
|
||||||
|
|
||||||
assert "timeoutMs:_sessionListHasLoadedOnce?30000:_SESSION_LIST_BOOT_TIMEOUT_MS" in sessions_src
|
assert "timeoutMs:_sessionListHasLoadedOnce?30000:_SESSION_LIST_BOOT_TIMEOUT_MS" in sessions_src
|
||||||
|
assert "const sessData = await api('/api/sessions' + sessionListQS,sessionRequestOpts);" in helper
|
||||||
|
assert "api('/api/sessions' + sessionListQS,{timeoutToast:false})" not in helper
|
||||||
assert "retryTimeouts:true" in sessions_src
|
assert "retryTimeouts:true" in sessions_src
|
||||||
assert "retryStatuses:[502,503,504]" in sessions_src
|
assert "retryStatuses:[502,503,504]" in sessions_src
|
||||||
assert "retryTimeouts" in workspace_src
|
assert "retryTimeouts" in workspace_src
|
||||||
|
|||||||
Reference in new issue
Block a user