From 8eed95a25e4e391d2f9739b5d7282d06e1545556 Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Wed, 24 Jun 2026 08:06:45 +0000 Subject: [PATCH] fix(#4808): widen /api/sessions cache TTL while streaming to stop per-poll rebuilds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The frontend polls /api/sessions every 5s while a turn streams (static/sessions.js _streamingPollMs=5000), but the cache TTL was 2.5s (_SESSIONS_CACHE_TTL_SECONDS) — so every streaming poll landed in a fresh window and forced a full all_sessions() rebuild on the hot path under the global store LOCK, pinning CPU to 100% and starving token rendering on large stores (recurrence of #4672). The #4680 freeze-marker only de-dupes rebuilds WITHIN one TTL window; with polls spaced wider than the TTL it never helped. Fix: while _session_list_cache_streaming_freeze_marker() indicates an active stream, use _SESSIONS_CACHE_STREAMING_TTL_SECONDS=10.0 (> the 5s poll) so the sidebar holds steady across streaming polls. Live runtime state (active stream, sort order, pending flags) is overlaid on every response regardless of cache, and structural/settings changes still evict immediately via the source stamp — only a streaming session's own persisted title/message_count can lag up to 10s, which the #4672 design already deems acceptable. Idle path byte-for-byte unchanged. Diagnosis credit: investigation reproduced 6 streaming polls -> 6 rebuilds before, 1 after. Co-authored-by: batkuip --- api/routes.py | 17 ++++++- tests/test_session_sidebar_cache.py | 72 +++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/api/routes.py b/api/routes.py index d711519a..22437be3 100644 --- a/api/routes.py +++ b/api/routes.py @@ -1572,6 +1572,16 @@ def _clear_live_models_cache() -> None: # ── /api/sessions response cache (hot-sidebar response) ────────────────────── _SESSIONS_CACHE_TTL_SECONDS = 2.5 +# #4808: while a turn is actively streaming the frontend polls /api/sessions on a +# fixed cadence (static/sessions.js `_streamingPollMs` = 5000ms). With the idle TTL +# of 2.5s, every streaming poll lands in a fresh window and forces a full +# all_sessions() rebuild on the hot path under the global store LOCK — pinning CPU +# and starving token rendering on large stores (recurrence of #4672). Hold the +# sidebar cache steady for longer than one poll interval while streaming; live +# runtime state (active stream, sort order, pending flags) is overlaid on every +# response regardless of cache (_session_list_cache_overlay_runtime_rows), and +# structural/settings changes still evict immediately via the source stamp. +_SESSIONS_CACHE_STREAMING_TTL_SECONDS = 10.0 _SESSIONS_CACHE_MAX_ENTRIES = 64 _SESSIONS_CACHE_WAIT_SECONDS = 0.25 _SESSIONS_CACHE_STALE_WAIT_SECONDS = 0.10 @@ -1630,7 +1640,12 @@ def _session_list_cache_get( if stamp != current_stamp: _SESSIONS_CACHE.pop(key, None) return None, False - fresh = (now - ts) < _SESSIONS_CACHE_TTL_SECONDS + # #4808: widen the freshness window while a turn is streaming so the fixed + # 5s streaming poll cadence doesn't force a full rebuild on every poll. + ttl = _SESSIONS_CACHE_TTL_SECONDS + if _session_list_cache_streaming_freeze_marker() is not None: + ttl = _SESSIONS_CACHE_STREAMING_TTL_SECONDS + fresh = (now - ts) < ttl if fresh: _SESSIONS_CACHE.move_to_end(key) return copy.deepcopy(payload), True diff --git a/tests/test_session_sidebar_cache.py b/tests/test_session_sidebar_cache.py index d191fba8..49f48c07 100644 --- a/tests/test_session_sidebar_cache.py +++ b/tests/test_session_sidebar_cache.py @@ -515,3 +515,75 @@ def test_source_stamp_still_tracks_wal_when_idle(tmp_path, monkeypatch): after = routes._session_list_cache_source_stamp(key) assert after != before + + +def _streaming_ttl_key(): + return routes._session_list_cache_key( + active_profile="default", + all_profiles=False, + show_cli_sessions=False, + show_previous_messaging_sessions=False, + show_cron_sessions=False, + ) + + +def _age_cache_entry(key, seconds): + """Backdate the cache entry's timestamp by `seconds` (keep stamp+payload).""" + with routes._SESSIONS_CACHE_LOCK: + ts, stamp, payload = routes._SESSIONS_CACHE[key] + routes._SESSIONS_CACHE[key] = (ts - seconds, stamp, payload) + + +def test_streaming_widens_cache_freshness_window(monkeypatch): + """#4808: while a turn streams, an entry older than the idle TTL but younger + than the streaming TTL must still read FRESH, so the fixed 5s streaming poll + does not force a rebuild every poll.""" + routes._session_list_cache_clear() + key = _streaming_ttl_key() + # Keep the source stamp stable across the get() calls (no structural change). + monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda k: ("stable",)) + + routes._session_list_cache_set(key, _session_cache_payload("live")) + # Age it past the idle 2.5s TTL but under the 10s streaming TTL. + _age_cache_entry(key, routes._SESSIONS_CACHE_TTL_SECONDS + 1.0) + + # Idle (no active stream) → stale → miss. + monkeypatch.setattr(routes, "_active_stream_ids", lambda: set()) + payload_idle, fresh_idle = routes._session_list_cache_get(key) + assert payload_idle is None and fresh_idle is False + + # Re-seed + re-age, now WITH an active stream → fresh (held). + routes._session_list_cache_set(key, _session_cache_payload("live")) + _age_cache_entry(key, routes._SESSIONS_CACHE_TTL_SECONDS + 1.0) + monkeypatch.setattr(routes, "_active_stream_ids", lambda: {"turn-1"}) + payload_stream, fresh_stream = routes._session_list_cache_get(key) + assert fresh_stream is True + assert payload_stream == _session_cache_payload("live") + + +def test_streaming_window_still_evicts_past_streaming_ttl(monkeypatch): + """Even while streaming, an entry older than the streaming TTL is evicted — + the hold-down is bounded, not indefinite.""" + routes._session_list_cache_clear() + key = _streaming_ttl_key() + monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda k: ("stable",)) + monkeypatch.setattr(routes, "_active_stream_ids", lambda: {"turn-1"}) + + routes._session_list_cache_set(key, _session_cache_payload("old")) + _age_cache_entry(key, routes._SESSIONS_CACHE_STREAMING_TTL_SECONDS + 1.0) + payload, fresh = routes._session_list_cache_get(key) + assert payload is None and fresh is False + + +def test_streaming_window_does_not_extend_idle_ttl(monkeypatch): + """Regression guard: with NO active stream the idle 2.5s TTL is unchanged — + an entry aged just past it must read stale (byte-for-byte idle behavior).""" + routes._session_list_cache_clear() + key = _streaming_ttl_key() + monkeypatch.setattr(routes, "_session_list_cache_source_stamp", lambda k: ("stable",)) + monkeypatch.setattr(routes, "_active_stream_ids", lambda: set()) + + routes._session_list_cache_set(key, _session_cache_payload("idle")) + _age_cache_entry(key, routes._SESSIONS_CACHE_TTL_SECONDS + 0.5) + payload, fresh = routes._session_list_cache_get(key) + assert payload is None and fresh is False