Harden connection flapping aftercare
Keep sidebar retries boot-scoped, preserve immediate source-change cache rebuilds, and tighten session stream ownership before closing duplicate SSE sockets. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
cbb644da69
commit
47d8ac94d3
9 files changed
+52
-33
No files matched your search
@@ -175,6 +175,25 @@ def _session_list_cache_get(
|
|||||||
return None, False
|
return None, False
|
||||||
|
|
||||||
|
|
||||||
|
def _session_list_cache_stale_reason(key: tuple) -> str | None:
|
||||||
|
"""Return why an existing cache entry is stale, if it is stale."""
|
||||||
|
now = time.monotonic()
|
||||||
|
current_stamp = _session_list_cache_resolved_source_stamp(key)
|
||||||
|
with _SESSIONS_CACHE_LOCK:
|
||||||
|
entry = _SESSIONS_CACHE.get(key)
|
||||||
|
if not entry:
|
||||||
|
return None
|
||||||
|
ts, stamp, _payload = entry
|
||||||
|
if stamp != current_stamp:
|
||||||
|
return "source"
|
||||||
|
ttl = _SESSIONS_CACHE_TTL_SECONDS
|
||||||
|
if _session_list_cache_streaming_freeze_marker() is not None:
|
||||||
|
ttl = _SESSIONS_CACHE_STREAMING_TTL_SECONDS
|
||||||
|
if (now - ts) >= ttl:
|
||||||
|
return "age"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _session_list_cache_set(key: tuple, payload: dict) -> None:
|
def _session_list_cache_set(key: tuple, payload: dict) -> None:
|
||||||
if not isinstance(payload, dict):
|
if not isinstance(payload, dict):
|
||||||
return
|
return
|
||||||
|
|||||||
+3
-1
@@ -1668,6 +1668,7 @@ _session_list_runtime_sort_key = _route_session_list_cache._session_list_runtime
|
|||||||
_session_list_cache_set = _route_session_list_cache._session_list_cache_set
|
_session_list_cache_set = _route_session_list_cache._session_list_cache_set
|
||||||
_session_list_cache_source_stamp = _route_session_list_cache._session_list_cache_source_stamp
|
_session_list_cache_source_stamp = _route_session_list_cache._session_list_cache_source_stamp
|
||||||
_session_list_cache_state_db_fingerprint = _route_session_list_cache._session_list_cache_state_db_fingerprint
|
_session_list_cache_state_db_fingerprint = _route_session_list_cache._session_list_cache_state_db_fingerprint
|
||||||
|
_session_list_cache_stale_reason = _route_session_list_cache._session_list_cache_stale_reason
|
||||||
_session_list_cache_streaming_freeze_marker = _route_session_list_cache._session_list_cache_streaming_freeze_marker
|
_session_list_cache_streaming_freeze_marker = _route_session_list_cache._session_list_cache_streaming_freeze_marker
|
||||||
|
|
||||||
_ROUTE_SESSION_LIST_CACHE_DYNAMIC_EXPORTS = {
|
_ROUTE_SESSION_LIST_CACHE_DYNAMIC_EXPORTS = {
|
||||||
@@ -2068,7 +2069,8 @@ def _get_cached_session_list_payload(
|
|||||||
return cached
|
return cached
|
||||||
|
|
||||||
stale = cached # now actually a stale payload when one exists, else None
|
stale = cached # now actually a stale payload when one exists, else None
|
||||||
if stale is not None:
|
stale_reason = _session_list_cache_stale_reason(key) if stale is not None else None
|
||||||
|
if stale is not None and stale_reason != "source":
|
||||||
event, is_owner = _session_list_cache_claim_rebuild(key)
|
event, is_owner = _session_list_cache_claim_rebuild(key)
|
||||||
if is_owner:
|
if is_owner:
|
||||||
if diag is not None:
|
if diag is not None:
|
||||||
|
|||||||
+1
-1
@@ -6456,7 +6456,7 @@ function _chatStreamActiveForSession(sid) {
|
|||||||
|
|
||||||
function _suspendSessionStreamForLiveChat(sid) {
|
function _suspendSessionStreamForLiveChat(sid) {
|
||||||
if (!sid) return;
|
if (!sid) return;
|
||||||
if (_sessionStreamSessionId !== sid && !_sessionEventSource) return;
|
if (_sessionStreamSessionId !== sid) return;
|
||||||
_sessionStreamHiddenSid = sid;
|
_sessionStreamHiddenSid = sid;
|
||||||
stopSessionStream();
|
stopSessionStream();
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-7
@@ -4298,13 +4298,13 @@ async function _runRenderSessionListRefresh(opts, _gen){
|
|||||||
try{
|
try{
|
||||||
if(!($('sessionSearch').value||'').trim()) _contentSearchResults = [];
|
if(!($('sessionSearch').value||'').trim()) _contentSearchResults = [];
|
||||||
const sessionListQS = _sessionListQueryString();
|
const sessionListQS = _sessionListQueryString();
|
||||||
const sessionRequestOpts={
|
const sessionRequestOpts={timeoutToast:false};
|
||||||
timeoutToast:false,
|
if(!_sessionListHasLoadedOnce){
|
||||||
timeoutMs:_sessionListHasLoadedOnce?30000:_SESSION_LIST_BOOT_TIMEOUT_MS,
|
sessionRequestOpts.timeoutMs=_SESSION_LIST_BOOT_TIMEOUT_MS;
|
||||||
retries:1,
|
sessionRequestOpts.retries=1;
|
||||||
retryTimeouts:true,
|
sessionRequestOpts.retryTimeouts=true;
|
||||||
retryStatuses:[502,503,504],
|
sessionRequestOpts.retryStatuses=[502,503,504];
|
||||||
};
|
}
|
||||||
const {sessData, projData}=await _loadSidebarSessionListPayload(sessionListQS, sessionRequestOpts);
|
const {sessData, projData}=await _loadSidebarSessionListPayload(sessionListQS, sessionRequestOpts);
|
||||||
// Discard stale response — a newer renderSessionList() call superseded us.
|
// Discard stale response — a newer renderSessionList() call superseded us.
|
||||||
if (_gen !== _renderSessionListGen) return;
|
if (_gen !== _renderSessionListGen) return;
|
||||||
|
|||||||
@@ -290,10 +290,6 @@ global.api = (url, opts) => {{
|
|||||||
(async () => {{
|
(async () => {{
|
||||||
const run = _loadSidebarSessionListPayload('?v=1', {{
|
const run = _loadSidebarSessionListPayload('?v=1', {{
|
||||||
timeoutToast:false,
|
timeoutToast:false,
|
||||||
timeoutMs:90000,
|
|
||||||
retries:1,
|
|
||||||
retryTimeouts:true,
|
|
||||||
retryStatuses:[502,503,504],
|
|
||||||
}});
|
}});
|
||||||
|
|
||||||
await Promise.resolve();
|
await Promise.resolve();
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ def test_session_list_refreshes_are_coalesced_while_in_flight():
|
|||||||
assert "_renderSessionListQueuedRequest={" in src
|
assert "_renderSessionListQueuedRequest={" in src
|
||||||
assert "opts:_mergeRenderSessionListOptions" in src
|
assert "opts:_mergeRenderSessionListOptions" in src
|
||||||
assert "if (_gen !== _renderSessionListGen) return" in src
|
assert "if (_gen !== _renderSessionListGen) return" in src
|
||||||
assert "api('/api/sessions' + sessionListQS,{timeoutToast:false})" in src
|
assert "const sessionRequestOpts={timeoutToast:false};" in src
|
||||||
|
assert "api('/api/sessions' + sessionListQS,sessionRequestOpts)" in src
|
||||||
assert "api('/api/projects' + projectQS,{timeoutToast:false})" in src
|
assert "api('/api/projects' + projectQS,{timeoutToast:false})" in src
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -452,6 +452,10 @@ def test_session_stream_pauses_while_chat_stream_is_active():
|
|||||||
assert "_sessionStreamHiddenSid = null;" in resume_src
|
assert "_sessionStreamHiddenSid = null;" in resume_src
|
||||||
assert "startSessionStream(sid);" in resume_src
|
assert "startSessionStream(sid);" in resume_src
|
||||||
|
|
||||||
|
suspend_src = _js_function_decl(js, "_suspendSessionStreamForLiveChat")
|
||||||
|
assert "if (_sessionStreamSessionId !== sid) return;" in suspend_src
|
||||||
|
assert "stopSessionStream();" in suspend_src
|
||||||
|
|
||||||
|
|
||||||
def test_frontend_busy_race_gate_obsoleted_by_option_z_pivot():
|
def test_frontend_busy_race_gate_obsoleted_by_option_z_pivot():
|
||||||
"""Per the Option Z PIVOT note baked into the handler body, the browser
|
"""Per the Option Z PIVOT note baked into the handler body, the browser
|
||||||
|
|||||||
@@ -198,7 +198,7 @@ def test_session_list_cache_follower_wait_stage_when_rebuild_inflight(monkeypatc
|
|||||||
assert "session_list_cache_hit" in owner_diag.stages or "session_list_cache_stored" in owner_diag.stages
|
assert "session_list_cache_hit" in owner_diag.stages or "session_list_cache_stored" in owner_diag.stages
|
||||||
|
|
||||||
|
|
||||||
def test_session_list_cache_follower_reuses_stale_payload_during_slow_rebuild(monkeypatch):
|
def test_session_list_cache_source_changed_owner_rebuilds_while_follower_reuses_stale(monkeypatch):
|
||||||
routes._session_list_cache_clear()
|
routes._session_list_cache_clear()
|
||||||
|
|
||||||
key = routes._session_list_cache_key(
|
key = routes._session_list_cache_key(
|
||||||
@@ -216,10 +216,10 @@ def test_session_list_cache_follower_reuses_stale_payload_during_slow_rebuild(mo
|
|||||||
stamp,
|
stamp,
|
||||||
payload,
|
payload,
|
||||||
)
|
)
|
||||||
# Simulate the #4834 path: state.db/WAL/fingerprint changes after the
|
# Simulate state.db/WAL/fingerprint changing after the stale payload was
|
||||||
# stale payload was cached. Followers must still be able to use that stale
|
# cached. The owner must rebuild synchronously so committed external state is
|
||||||
# payload while the owner rebuild is blocked; otherwise sidebar polling can
|
# visible immediately, but followers can still use stale while that rebuild
|
||||||
# pile up behind a slow rebuild.
|
# is blocked; otherwise sidebar polling can pile up behind a slow rebuild.
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
routes,
|
routes,
|
||||||
"_session_list_cache_source_stamp",
|
"_session_list_cache_source_stamp",
|
||||||
@@ -265,17 +265,15 @@ def test_session_list_cache_follower_reuses_stale_payload_during_slow_rebuild(mo
|
|||||||
owner_thread.join(2.0)
|
owner_thread.join(2.0)
|
||||||
follower_thread.join(2.0)
|
follower_thread.join(2.0)
|
||||||
|
|
||||||
assert owner_result["payload"] == _session_cache_payload("stale")
|
assert owner_result["payload"] == _session_cache_payload("fresh")
|
||||||
assert follower_result["payload"] == _session_cache_payload("stale")
|
assert follower_result["payload"] == _session_cache_payload("stale")
|
||||||
assert "session_list_cache_stale_background_rebuild" in owner_diag.stages
|
assert "session_list_cache_rebuild_owner" in owner_diag.stages
|
||||||
assert (
|
assert "session_list_cache_wait_stale_fallback" in follower_diag.stages
|
||||||
"session_list_cache_stale_return" in follower_diag.stages
|
|
||||||
or "session_list_cache_stale_background_rebuild" in follower_diag.stages
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_session_list_cache_owner_returns_stale_and_rebuilds_in_background(monkeypatch):
|
def test_session_list_cache_owner_returns_stale_and_rebuilds_in_background(monkeypatch):
|
||||||
routes._session_list_cache_clear()
|
routes._session_list_cache_clear()
|
||||||
|
monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda _key: ("stable",))
|
||||||
|
|
||||||
key = routes._session_list_cache_key(
|
key = routes._session_list_cache_key(
|
||||||
active_profile="default",
|
active_profile="default",
|
||||||
@@ -292,8 +290,6 @@ def test_session_list_cache_owner_returns_stale_and_rebuilds_in_background(monke
|
|||||||
stamp,
|
stamp,
|
||||||
payload,
|
payload,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda _key: ("changed",))
|
|
||||||
|
|
||||||
started = threading.Event()
|
started = threading.Event()
|
||||||
release = threading.Event()
|
release = threading.Event()
|
||||||
diag = _StageRecorder()
|
diag = _StageRecorder()
|
||||||
@@ -327,6 +323,7 @@ def test_session_list_cache_owner_returns_stale_and_rebuilds_in_background(monke
|
|||||||
|
|
||||||
def test_session_list_cache_stale_background_rebuild_failure_releases_owner(monkeypatch):
|
def test_session_list_cache_stale_background_rebuild_failure_releases_owner(monkeypatch):
|
||||||
routes._session_list_cache_clear()
|
routes._session_list_cache_clear()
|
||||||
|
monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda _key: ("stable",))
|
||||||
|
|
||||||
key = routes._session_list_cache_key(
|
key = routes._session_list_cache_key(
|
||||||
active_profile="default",
|
active_profile="default",
|
||||||
@@ -343,8 +340,6 @@ def test_session_list_cache_stale_background_rebuild_failure_releases_owner(monk
|
|||||||
stamp,
|
stamp,
|
||||||
payload,
|
payload,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda _key: ("changed",))
|
|
||||||
|
|
||||||
started = threading.Event()
|
started = threading.Event()
|
||||||
logged = []
|
logged = []
|
||||||
|
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ def test_sessions_and_projects_load_independently_so_projects_failure_cannot_bla
|
|||||||
assert "_applySessionListPayload(sessData,projData)" in block
|
assert "_applySessionListPayload(sessData,projData)" in block
|
||||||
|
|
||||||
|
|
||||||
def test_sessions_api_uses_longer_timeout_and_timeout_retry_for_boot_refresh():
|
def test_sessions_api_uses_longer_timeout_and_timeout_retry_for_boot_refresh_only():
|
||||||
sessions_src = _sessions_js()
|
sessions_src = _sessions_js()
|
||||||
workspace_src = _workspace_js()
|
workspace_src = _workspace_js()
|
||||||
helper_start = sessions_src.find("async function _loadSidebarSessionListPayload")
|
helper_start = sessions_src.find("async function _loadSidebarSessionListPayload")
|
||||||
@@ -65,11 +65,13 @@ def test_sessions_api_uses_longer_timeout_and_timeout_retry_for_boot_refresh():
|
|||||||
assert helper_end > helper_start
|
assert helper_end > helper_start
|
||||||
helper = sessions_src[helper_start:helper_end]
|
helper = sessions_src[helper_start:helper_end]
|
||||||
|
|
||||||
assert "timeoutMs:_sessionListHasLoadedOnce?30000:_SESSION_LIST_BOOT_TIMEOUT_MS" in sessions_src
|
assert "const sessionRequestOpts={timeoutToast:false};" in sessions_src
|
||||||
|
assert "if(!_sessionListHasLoadedOnce){" in sessions_src
|
||||||
|
assert "sessionRequestOpts.timeoutMs=_SESSION_LIST_BOOT_TIMEOUT_MS;" in sessions_src
|
||||||
assert "const sessData = await api('/api/sessions' + sessionListQS,sessionRequestOpts);" in helper
|
assert "const sessData = await api('/api/sessions' + sessionListQS,sessionRequestOpts);" in helper
|
||||||
assert "api('/api/sessions' + sessionListQS,{timeoutToast:false})" not in helper
|
assert "api('/api/sessions' + sessionListQS,{timeoutToast:false})" not in helper
|
||||||
assert "retryTimeouts:true" in sessions_src
|
assert "sessionRequestOpts.retryTimeouts=true" in sessions_src
|
||||||
assert "retryStatuses:[502,503,504]" in sessions_src
|
assert "sessionRequestOpts.retryStatuses=[502,503,504]" in sessions_src
|
||||||
assert "retryTimeouts" in workspace_src
|
assert "retryTimeouts" in workspace_src
|
||||||
assert "retryStatuses" in workspace_src
|
assert "retryStatuses" in workspace_src
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user