Release YL (v0.51.682): indexed compression-continuation lookup (#4991, fixes #4990)

Release YL (v0.51.682): indexed compression-continuation lookup (#4991, fixes #4990)

CHANGELOG.md

@@ -3,6 +3,12 @@
 
 ## [Unreleased]
 
+## [v0.51.682] — 2026-06-26 — Release YL (faster long-session open — indexed compression-continuation lookup)
+
+### Fixed
+
+- **Opening a session with a compression snapshot is faster, and still resolves to the correct live continuation.** The pre-compression snapshot continuation lookup scanned and JSON-parsed every session sidecar to find the live child; it now uses the session index as a fast path (one directory listing + one index read, no per-sidecar parse). Correctness is preserved by a membership-completeness guard: if any persisted continuation sidecar is missing from the index, the lookup falls back to the full sidecar scan — so a stale index can't silently drop a continuation and surface the archived snapshot instead of the live conversation. The index parse no longer mutates the shared ancestor set, and ties on `updated_at` resolve deterministically by `session_id` so the fast path and the scan path are byte-identical. Thanks @franksong2702. (#4991, fixes #4990)
+
 ## [v0.51.681] — 2026-06-26 — Release YK (Anthropic shows in the model picker when configured via OAuth token)
 
 ### Fixed

api/routes.py

@@ -6763,9 +6763,8 @@ def _pre_compression_continuation_session_id(session) -> str | None:
     # snapshot's profile and reject any child that isn't profile-matched.
     snapshot_profile = getattr(session, "profile", None)
 
-    def _child_rows() -> list:
+    def _child_rows_from_memory(seen_ids: set[str]) -> list:
         rows = []
-        seen_ids = set()
         try:
             with LOCK:
                 memory_sessions = list(SESSIONS.values())
@@ -6777,6 +6776,51 @@ def _pre_compression_continuation_session_id(session) -> str | None:
                 rows.append(child)
         except Exception:
             pass
+        return rows
+
+    def _child_rows_from_index(seen_ids: set[str]) -> list | None:
+        if not SESSION_INDEX_FILE.exists():
+            return None
+        try:
+            entries = json.loads(SESSION_INDEX_FILE.read_text(encoding="utf-8"))
+        except Exception:
+            return None
+        if not isinstance(entries, list):
+            return None
+        try:
+            persisted_sidecar_ids = {
+                path.stem
+                for path in SESSION_DIR.glob("*.json")
+                if not path.name.startswith("_") and is_safe_session_id(path.stem)
+            }
+        except Exception:
+            return None
+        indexed_ids: set[str] = set()
+        row_seen_ids = set(seen_ids)
+        rows = []
+        for entry in entries:
+            if not isinstance(entry, dict):
+                continue
+            child_sid = _safe_first(entry.get("session_id"))
+            if not child_sid or not is_safe_session_id(child_sid):
+                continue
+            indexed_ids.add(child_sid)
+            if child_sid in row_seen_ids or not _safe_first(entry.get("parent_session_id")):
+                continue
+            row_seen_ids.add(child_sid)
+            rows.append(entry)
+        # Guarantee here is index MEMBERSHIP-completeness, not per-entry content
+        # freshness: if any persisted continuation sidecar is absent from the index
+        # we bail to the full scan. A sidecar that IS in the index but whose entry is
+        # content-stale (mid-write) still yields a valid continuation of the same
+        # snapshot; proving freshness would require reading every sidecar, defeating
+        # the optimization, so membership-completeness is the intended bar.
+        if persisted_sidecar_ids - indexed_ids - seen_ids:
+            return None
+        return rows
+
+    def _child_rows_from_sidecars(seen_ids: set[str]) -> list:
+        rows = []
         try:
             for path in SESSION_DIR.glob("*.json"):
                 if path.name.startswith("_"):
@@ -6792,51 +6836,79 @@ def _pre_compression_continuation_session_id(session) -> str | None:
             pass
         return rows
 
-    children_by_parent: dict[str, list] = {}
-    for child in _child_rows():
-        parent_sid = _safe_first(getattr(child, "parent_session_id", None))
-        child_sid = _safe_first(getattr(child, "session_id", None))
-        if not parent_sid or not child_sid or child_sid == sid:
-            continue
-        # Cross-profile guard: only follow continuations within the snapshot's profile.
-        if not _profiles_match(getattr(child, "profile", None), snapshot_profile):
-            continue
-        children_by_parent.setdefault(parent_sid, []).append(child)
+    def _row_value(row, key, default=None):
+        return row.get(key, default) if isinstance(row, dict) else getattr(row, key, default)
 
-    candidates = []
-    frontier = [sid]
-    seen = {sid}
-    for _ in range(20):
-        if not frontier:
-            break
-        parent_sid = frontier.pop(0)
-        for child in children_by_parent.get(parent_sid, []):
-            child_sid = _safe_first(getattr(child, "session_id", None))
-            if not child_sid or child_sid in seen:
+    def _row_has_backing_state(row) -> bool:
+        child_sid = _safe_first(_row_value(row, "session_id"))
+        if not child_sid or not is_safe_session_id(child_sid):
+            return False
+        if not isinstance(row, dict):
+            return True
+        return (SESSION_DIR / f"{child_sid}.json").exists()
+
+    def _resolve_from_rows(rows: list) -> str | None:
+        children_by_parent: dict[str, list] = {}
+        for child in rows:
+            parent_sid = _safe_first(_row_value(child, "parent_session_id"))
+            child_sid = _safe_first(_row_value(child, "session_id"))
+            if not parent_sid or not child_sid or child_sid == sid:
                 continue
-            seen.add(child_sid)
-            if getattr(child, "pre_compression_snapshot", False):
-                frontier.append(child_sid)
-            else:
-                candidates.append(child)
+            # Cross-profile guard: only follow continuations within the snapshot's profile.
+            if not _profiles_match(_row_value(child, "profile"), snapshot_profile):
+                continue
+            children_by_parent.setdefault(parent_sid, []).append(child)
+
+        candidates = []
+        frontier = [sid]
+        seen = {sid}
+        for _ in range(20):
+            if not frontier:
+                break
+            parent_sid = frontier.pop(0)
+            for child in children_by_parent.get(parent_sid, []):
+                child_sid = _safe_first(_row_value(child, "session_id"))
+                if not child_sid or child_sid in seen or not _row_has_backing_state(child):
+                    continue
+                seen.add(child_sid)
+                if _row_value(child, "pre_compression_snapshot", False):
+                    frontier.append(child_sid)
+                else:
+                    candidates.append(child)
+
+        if not candidates:
+            return None
+        latest = max(
+            candidates,
+            key=lambda child: (
+                float(
+                    _safe_first(
+                        _row_value(child, "updated_at"),
+                        _row_value(child, "created_at"),
+                        0,
+                    ) or 0
+                ),
+                # Secondary tiebreaker so the index-fast-path and the sidecar-scan
+                # path resolve byte-identically on an updated_at/created_at tie
+                # (otherwise the chosen sid could differ by iteration order).
+                str(_safe_first(_row_value(child, "session_id"), "") or ""),
+            ),
+        )
+        latest_sid = _safe_first(_row_value(latest, "session_id", None)) or None
+        # Only hand the client a well-formed session id (it gets written to URL/localStorage).
+        if latest_sid and not is_safe_session_id(latest_sid):
+            return None
+        return latest_sid
+
+    memory_seen_ids: set[str] = set()
+    rows = _child_rows_from_memory(memory_seen_ids)
+    index_rows = _child_rows_from_index(memory_seen_ids)
+    if index_rows is not None:
+        return _resolve_from_rows(rows + index_rows)
+
+    rows.extend(_child_rows_from_sidecars(memory_seen_ids))
+    return _resolve_from_rows(rows)
 
-    if not candidates:
-        return None
-    latest = max(
-        candidates,
-        key=lambda child: float(
-            _safe_first(
-                getattr(child, "updated_at", None),
-                getattr(child, "created_at", None),
-                0,
-            ) or 0
-        ),
-    )
-    latest_sid = getattr(latest, "session_id", None) or None
-    # Only hand the client a well-formed session id (it gets written to URL/localStorage).
-    if latest_sid and not is_safe_session_id(latest_sid):
-        return None
-    return latest_sid
 from api.workspace import (
     load_workspaces,
     save_workspaces,

tests/test_mobile_reload_compression_recovery.py

@@ -1,5 +1,7 @@
 """Regression coverage for mobile reload recovery after compression session rotation."""
 
+import collections
+import json
 from pathlib import Path
 
 
@@ -7,6 +9,20 @@ ROOT = Path(__file__).resolve().parents[1]
 SESSIONS_JS = ROOT / "static" / "sessions.js"
 
 
+def _write_sidecar(session_dir: Path, sid: str, **overrides):
+    messages = overrides.pop("messages", [])
+    payload = {
+        "session_id": sid,
+        "title": sid,
+        "created_at": 100.0,
+        "updated_at": 100.0,
+        "profile": "work",
+    }
+    payload.update(overrides)
+    payload["messages"] = messages
+    (session_dir / f"{sid}.json").write_text(json.dumps(payload), encoding="utf-8")
+
+
 def _function_block(source: str, marker: str) -> str:
     start = source.index(marker)
     brace = source.index("{", start)
@@ -68,7 +84,6 @@ def test_continuation_lookup_is_profile_scoped(tmp_path, monkeypatch):
     # Empty session dir so only in-memory SESSIONS are considered.
     monkeypatch.setattr(config, "SESSION_DIR", tmp_path, raising=False)
     monkeypatch.setattr(routes, "SESSION_DIR", tmp_path, raising=False)
-    import collections
     fake = collections.OrderedDict()
     for s in (same_profile_child, foreign_child):
         fake[s.session_id] = s
@@ -82,3 +97,311 @@ def test_continuation_lookup_is_profile_scoped(tmp_path, monkeypatch):
     fake2[foreign_child.session_id] = foreign_child
     monkeypatch.setattr(routes, "SESSIONS", fake2, raising=False)
     assert routes._pre_compression_continuation_session_id(snapshot) is None
+
+
+def test_continuation_lookup_uses_index_without_scanning_sidecars(tmp_path, monkeypatch):
+    """Indexed continuation metadata should avoid an O(all sidecars) recovery scan."""
+    from api import routes, config, models
+
+    class _S:
+        def __init__(self, sid, profile, snap=False):
+            self.session_id = sid
+            self.profile = profile
+            self.parent_session_id = None
+            self.pre_compression_snapshot = snap
+            self.updated_at = 100.0
+            self.created_at = 100.0
+
+    snapshot = _S("snapindex001", "work", snap=True)
+    index_file = tmp_path / "_index.json"
+    (tmp_path / "childindex01.json").write_text("{}", encoding="utf-8")
+    for idx in range(50):
+        (tmp_path / f"noise{idx:08d}.json").write_text("{}", encoding="utf-8")
+    index_file.write_text(
+        json.dumps(
+            [
+                {
+                    "session_id": "childindex01",
+                    "profile": "work",
+                    "parent_session_id": "snapindex001",
+                    "pre_compression_snapshot": False,
+                    "updated_at": 300.0,
+                    "created_at": 200.0,
+                }
+            ]
+            + [{"session_id": f"noise{idx:08d}", "profile": "work"} for idx in range(50)]
+        ),
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(config, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(models, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_INDEX_FILE", index_file, raising=False)
+    monkeypatch.setattr(routes, "SESSIONS", collections.OrderedDict(), raising=False)
+    monkeypatch.setattr(
+        routes.Session,
+        "load_metadata_only",
+        staticmethod(lambda _sid: (_ for _ in ()).throw(AssertionError("must not scan sidecars"))),
+    )
+
+    assert routes._pre_compression_continuation_session_id(snapshot) == "childindex01"
+
+
+def test_empty_indexed_continuation_lookup_falls_back_to_sidecars(tmp_path, monkeypatch):
+    """A valid but stale/empty index must not suppress durable sidecar recovery."""
+    from api import routes, config
+
+    class _S:
+        def __init__(self, sid, profile, parent=None, snap=False, updated=100.0):
+            self.session_id = sid
+            self.profile = profile
+            self.parent_session_id = parent
+            self.pre_compression_snapshot = snap
+            self.updated_at = updated
+            self.created_at = updated
+
+    snapshot = _S("snapempty001", "work", snap=True)
+    index_file = tmp_path / "_index.json"
+    index_file.write_text("[]", encoding="utf-8")
+    (tmp_path / "childempty01.json").write_text("{}", encoding="utf-8")
+
+    monkeypatch.setattr(config, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_INDEX_FILE", index_file, raising=False)
+    monkeypatch.setattr(routes, "SESSIONS", collections.OrderedDict(), raising=False)
+    loaded = []
+    monkeypatch.setattr(
+        routes.Session,
+        "load_metadata_only",
+        staticmethod(
+            lambda sid: loaded.append(sid)
+            or _S("childempty01", "work", parent="snapempty001", updated=300.0)
+        ),
+    )
+
+    assert routes._pre_compression_continuation_session_id(snapshot) == "childempty01"
+    assert loaded == ["childempty01"]
+
+
+def test_stale_index_with_existing_candidate_falls_back_to_newer_sidecar(tmp_path, monkeypatch):
+    """A complete-looking result is not trusted when another sidecar is absent from the index."""
+    from api import routes, config, models
+
+    class _S:
+        def __init__(self, sid, profile, snap=False):
+            self.session_id = sid
+            self.profile = profile
+            self.parent_session_id = None
+            self.pre_compression_snapshot = snap
+            self.updated_at = 100.0
+            self.created_at = 100.0
+
+    snapshot = _S("snapstale001", "work", snap=True)
+    index_file = tmp_path / "_index.json"
+    _write_sidecar(
+        tmp_path,
+        "oldstale001",
+        parent_session_id="snapstale001",
+        updated_at=200.0,
+        created_at=150.0,
+    )
+    _write_sidecar(
+        tmp_path,
+        "newstale001",
+        parent_session_id="snapstale001",
+        updated_at=400.0,
+        created_at=350.0,
+    )
+    index_file.write_text(
+        json.dumps(
+            [
+                {
+                    "session_id": "oldstale001",
+                    "profile": "work",
+                    "parent_session_id": "snapstale001",
+                    "updated_at": 200.0,
+                    "created_at": 150.0,
+                }
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(config, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(models, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_INDEX_FILE", index_file, raising=False)
+    monkeypatch.setattr(routes, "SESSIONS", collections.OrderedDict(), raising=False)
+
+    assert routes._pre_compression_continuation_session_id(snapshot) == "newstale001"
+
+
+def test_stale_index_multihop_falls_back_to_missing_descendant_sidecar(tmp_path, monkeypatch):
+    """An indexed snapshot ancestor must not hide a newer descendant omitted from the index."""
+    from api import routes, config, models
+
+    class _S:
+        def __init__(self, sid, profile, snap=False):
+            self.session_id = sid
+            self.profile = profile
+            self.parent_session_id = None
+            self.pre_compression_snapshot = snap
+            self.updated_at = 100.0
+            self.created_at = 100.0
+
+    snapshot = _S("snapstale002", "work", snap=True)
+    index_file = tmp_path / "_index.json"
+    _write_sidecar(
+        tmp_path,
+        "oldstale002",
+        parent_session_id="snapstale002",
+        updated_at=200.0,
+        created_at=150.0,
+    )
+    _write_sidecar(
+        tmp_path,
+        "midstale002",
+        parent_session_id="snapstale002",
+        pre_compression_snapshot=True,
+        updated_at=300.0,
+        created_at=250.0,
+    )
+    _write_sidecar(
+        tmp_path,
+        "newstale002",
+        parent_session_id="midstale002",
+        updated_at=500.0,
+        created_at=450.0,
+    )
+    index_file.write_text(
+        json.dumps(
+            [
+                {
+                    "session_id": "oldstale002",
+                    "profile": "work",
+                    "parent_session_id": "snapstale002",
+                    "updated_at": 200.0,
+                    "created_at": 150.0,
+                },
+                {
+                    "session_id": "midstale002",
+                    "profile": "work",
+                    "parent_session_id": "snapstale002",
+                    "pre_compression_snapshot": True,
+                    "updated_at": 300.0,
+                    "created_at": 250.0,
+                },
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(config, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(models, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_INDEX_FILE", index_file, raising=False)
+    monkeypatch.setattr(routes, "SESSIONS", collections.OrderedDict(), raising=False)
+
+    assert routes._pre_compression_continuation_session_id(snapshot) == "newstale002"
+
+
+def test_indexed_continuation_lookup_follows_snapshot_hops_without_scanning(tmp_path, monkeypatch):
+    """Repeated compression can resolve through index-backed snapshot descendants."""
+    from api import routes, config
+
+    class _S:
+        def __init__(self, sid, profile, snap=False):
+            self.session_id = sid
+            self.profile = profile
+            self.parent_session_id = None
+            self.pre_compression_snapshot = snap
+            self.updated_at = 100.0
+            self.created_at = 100.0
+
+    snapshot = _S("snapindex003", "work", snap=True)
+    index_file = tmp_path / "_index.json"
+    _write_sidecar(tmp_path, "midindex001")
+    _write_sidecar(tmp_path, "finalindex1")
+    index_file.write_text(
+        json.dumps(
+            [
+                {
+                    "session_id": "midindex001",
+                    "profile": "work",
+                    "parent_session_id": "snapindex003",
+                    "pre_compression_snapshot": True,
+                    "updated_at": 200.0,
+                    "created_at": 150.0,
+                },
+                {
+                    "session_id": "finalindex1",
+                    "profile": "work",
+                    "parent_session_id": "midindex001",
+                    "pre_compression_snapshot": False,
+                    "updated_at": 300.0,
+                    "created_at": 250.0,
+                },
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(config, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_INDEX_FILE", index_file, raising=False)
+    monkeypatch.setattr(routes, "SESSIONS", collections.OrderedDict(), raising=False)
+    monkeypatch.setattr(
+        routes.Session,
+        "load_metadata_only",
+        staticmethod(lambda _sid: (_ for _ in ()).throw(AssertionError("must not scan sidecars"))),
+    )
+
+    assert routes._pre_compression_continuation_session_id(snapshot) == "finalindex1"
+
+
+def test_indexed_continuation_lookup_keeps_profile_scope(tmp_path, monkeypatch):
+    """The index fast path must preserve the cross-profile continuation guard."""
+    from api import routes, config, models
+
+    class _S:
+        def __init__(self, sid, profile, snap=False):
+            self.session_id = sid
+            self.profile = profile
+            self.parent_session_id = None
+            self.pre_compression_snapshot = snap
+            self.updated_at = 100.0
+            self.created_at = 100.0
+
+    snapshot = _S("snapindex002", "work", snap=True)
+    index_file = tmp_path / "_index.json"
+    _write_sidecar(
+        tmp_path,
+        "foreignidx01",
+        profile="personal",
+        parent_session_id="snapindex002",
+        updated_at=300.0,
+        created_at=200.0,
+    )
+    index_file.write_text(
+        json.dumps(
+            [
+                {
+                    "session_id": "foreignidx01",
+                    "profile": "personal",
+                    "parent_session_id": "snapindex002",
+                    "updated_at": 300.0,
+                    "created_at": 200.0,
+                }
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(config, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(models, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_DIR", tmp_path, raising=False)
+    monkeypatch.setattr(routes, "SESSION_INDEX_FILE", index_file, raising=False)
+    monkeypatch.setattr(routes, "SESSIONS", collections.OrderedDict(), raising=False)
+
+    assert routes._pre_compression_continuation_session_id(snapshot) is None