Merge pull request #2759 from nesquena/release/stage-404

Release CI: v0.51.111 (stage-404 — 1-PR — keep state.db replays out of sidecar tail)

CHANGELOG.md

@@ -3,6 +3,12 @@
 
 ## [Unreleased]
 
+## [v0.51.111] — 2026-05-22 — Release CI (stage-404 — 1-PR — keep state.db replays out of sidecar tail)
+
+### Fixed
+
+- **PR #2746** by @ai-ag2026 — Prevent replayed state.db rows from being appended after an already-correct sidecar transcript tail. `merge_session_messages_append_only()` previously tried to skip state.db rows replaying the sidecar, but two edge cases leaked through: (1) the final row of a replayed sidecar prefix was not skipped because the replay index had reached the sidecar sequence length, and (2) a replayed middle segment was not considered prefix replay, so old state.db rows could be appended after the saved assistant tail. That made `/api/session` appear to end on an old user prompt even when the saved sidecar already ended on the real assistant answer. The fix tracks per-(role, content) visible-occurrence counts in the sidecar and uses that as a replay budget when comparing state.db rows; legitimate repeated messages from state.db are still preserved. `_has_visible_duplicate()` is kept as a thin wrapper around the new `_matching_visible_duplicate()` for backwards compatibility. Regression test covers both full-replay and middle-segment replay shapes.
+
 ## [v0.51.110] — 2026-05-22 — Release CH (stage-403 — 2-PR batch — default personality from config + sort configured providers to top)
 
 ### Added

api/models.py

@@ -3048,24 +3048,28 @@ def _session_message_visible_key(msg: dict):
     )
 
 
-def _has_visible_duplicate(visible_key: tuple, visible_keys: set[tuple]) -> bool:
+def _matching_visible_duplicate(visible_key: tuple, visible_keys: set[tuple]):
     if visible_key in visible_keys:
-        return True
+        return visible_key
     role, content = visible_key
     if not content:
-        return False
+        return None
     for existing_role, existing_content in visible_keys:
         if role != existing_role or not existing_content:
             continue
         if content in existing_content or existing_content in content:
-            return True
+            return (existing_role, existing_content)
         loose_content = _loose_session_message_content(content)
         loose_existing = _loose_session_message_content(existing_content)
         if loose_content and loose_existing and (
             loose_content in loose_existing or loose_existing in loose_content
         ):
-            return True
-    return False
+            return (existing_role, existing_content)
+    return None
+
+
+def _has_visible_duplicate(visible_key: tuple, visible_keys: set[tuple]) -> bool:
+    return _matching_visible_duplicate(visible_key, visible_keys) is not None
 
 
 def merge_session_messages_append_only(sidecar_messages: list, state_messages: list) -> list:
@@ -3082,6 +3086,8 @@ def merge_session_messages_append_only(sidecar_messages: list, state_messages: l
     seen_content_keys = set()
     seen_visible_keys = set()
     sidecar_visible_sequence = []
+    sidecar_visible_keys = set()
+    sidecar_visible_counts = {}
     max_sidecar_timestamp = None
     for msg in sidecar_messages:
         timestamp = _message_timestamp_as_float(msg)
@@ -3092,9 +3098,12 @@ def merge_session_messages_append_only(sidecar_messages: list, state_messages: l
         seen_content_keys.add(_session_message_content_key(msg))
         visible_key = _session_message_visible_key(msg)
         seen_visible_keys.add(visible_key)
+        sidecar_visible_keys.add(visible_key)
+        sidecar_visible_counts[visible_key] = sidecar_visible_counts.get(visible_key, 0) + 1
         sidecar_visible_sequence.append(visible_key)
         merged_messages.append(msg)
     state_replay_idx = 0
+    skipped_state_visible_counts = {}
     for msg in state_messages:
         timestamp = _message_timestamp_as_float(msg)
         key = _session_message_merge_key(msg)
@@ -3107,7 +3116,12 @@ def merge_session_messages_append_only(sidecar_messages: list, state_messages: l
             ):
                 replays_sidecar_prefix = True
                 state_replay_idx += 1
-        if replays_sidecar_prefix and state_replay_idx < len(sidecar_visible_sequence):
+        if replays_sidecar_prefix:
+            matched_visible_key = _matching_visible_duplicate(visible_key, sidecar_visible_keys)
+            if matched_visible_key is not None:
+                skipped_state_visible_counts[matched_visible_key] = (
+                    skipped_state_visible_counts.get(matched_visible_key, 0) + 1
+                )
             continue
         if max_sidecar_timestamp is not None and timestamp is not None and timestamp <= max_sidecar_timestamp:
             if key in seen_message_keys:
@@ -3116,6 +3130,13 @@ def merge_session_messages_append_only(sidecar_messages: list, state_messages: l
                 continue
         if key in seen_message_keys:
             continue
+        matched_visible_key = _matching_visible_duplicate(visible_key, sidecar_visible_keys)
+        if matched_visible_key is not None:
+            skipped_count = skipped_state_visible_counts.get(matched_visible_key, 0)
+            sidecar_count = sidecar_visible_counts.get(matched_visible_key, 0)
+            if skipped_count < sidecar_count:
+                skipped_state_visible_counts[matched_visible_key] = skipped_count + 1
+                continue
         # State rows at or before the newest sidecar timestamp are normally
         # assumed to have already been observed by the sidecar. The <= gate
         # preserves sidecar-only ordering/metadata for equal timestamps and

tests/test_session_lost_response_regression.py

@@ -195,6 +195,62 @@ def test_state_db_prefix_with_float_timestamps_does_not_hide_sidecar_tail():
     assert "Sammel-PR" in merged[-1]["content"]
 
 
+def test_state_db_full_replay_does_not_append_after_sidecar_tail():
+    """A full state.db replay must not leak the final replayed user after the sidecar tail.
+
+    Regression for a display merge where the sidecar already ends on the real
+    assistant answer, but state.db replays the same visible turn sequence with
+    newer float timestamps.
+    """
+    sidecar_messages = [
+        {"role": "user", "content": "initial critique", "timestamp": 100},
+        {"role": "assistant", "content": "analysis", "timestamp": 100},
+        {"role": "user", "content": "Erstelle deine Version", "timestamp": 100},
+        {"role": "assistant", "content": "opened browser preview", "timestamp": 100},
+    ]
+    state_replay = [
+        {"role": "user", "content": "initial critique", "timestamp": 100.1},
+        {"role": "assistant", "content": "analysis", "timestamp": 100.2},
+        {
+            "role": "user",
+            "content": "[Workspace::v1: /tmp/project-workspace]\nErstelle deine Version",
+            "timestamp": 100.3,
+        },
+        {"role": "assistant", "content": "opened browser preview", "timestamp": 100.4},
+    ]
+
+    merged = merge_session_messages_append_only(sidecar_messages, state_replay)
+
+    assert [m["content"] for m in merged] == [m["content"] for m in sidecar_messages]
+    assert merged[-1]["role"] == "assistant"
+    assert merged[-1]["content"] == "opened browser preview"
+
+
+def test_state_db_middle_segment_replay_does_not_append_after_sidecar_tail():
+    """A replayed state.db segment from the middle must not be appended after the tail."""
+    sidecar_messages = [
+        {"role": "user", "content": "older setup", "timestamp": 100},
+        {"role": "assistant", "content": "older answer", "timestamp": 100},
+        {"role": "assistant", "content": "analysis before request", "timestamp": 100},
+        {"role": "user", "content": "Erstelle deine Version", "timestamp": 100},
+        {"role": "assistant", "content": "opened browser preview", "timestamp": 100},
+    ]
+    state_middle_replay = [
+        {"role": "assistant", "content": "analysis before request", "timestamp": 100.1},
+        {
+            "role": "user",
+            "content": "[Workspace::v1: /tmp/project-workspace]\nErstelle deine Version",
+            "timestamp": 100.2,
+        },
+    ]
+
+    merged = merge_session_messages_append_only(sidecar_messages, state_middle_replay)
+
+    assert [m["content"] for m in merged] == [m["content"] for m in sidecar_messages]
+    assert merged[-1]["role"] == "assistant"
+    assert merged[-1]["content"] == "opened browser preview"
+
+
 def test_lost_response_recovered_on_second_read(hermes_home):
     sid = "9f14583f0e4e4444aaaa111122223333"
     stream_id = "7c8b4108d52b4aba9af362d3a54f47ac"