Live-to-Final: prove historical hydration and fence terminal retry ownership (#6451)

* test: add historical hydration lifecycle proof

* fix: retry anchor persistence after terminal app errors

* fix: fence deferred anchor settlement ownership

* test: cover deferred anchor retry ownership

* fix: preserve terminal anchor retry across refresh

* fix: bind terminal anchor retry to turn owner

---------

Co-authored-by: Frank Song <franksong2702@gmail.com>

.github/workflows/conversation-lifecycle.yml

@@ -12,6 +12,7 @@ on:
       - 'api/**.py'
       - 'server.py'
       - 'tests/browser_conversation_lifecycle.py'
+      - 'tests/browser_historical_transcript_hydration.py'
       - 'requirements*.txt'
       - 'pyproject.toml'
       - '.github/workflows/conversation-lifecycle.yml'
@@ -22,6 +23,7 @@ on:
       - 'api/**.py'
       - 'server.py'
       - 'tests/browser_conversation_lifecycle.py'
+      - 'tests/browser_historical_transcript_hydration.py'
       - 'requirements*.txt'
       - 'pyproject.toml'
       - '.github/workflows/conversation-lifecycle.yml'
@@ -34,7 +36,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        scenario: [normal, terminal-error]
+        include:
+          - name: normal
+            script: tests/browser_conversation_lifecycle.py
+            scenario: normal
+          - name: terminal-error
+            script: tests/browser_conversation_lifecycle.py
+            scenario: terminal-error
+          - name: historical-transcript-hydration
+            script: tests/browser_historical_transcript_hydration.py
     steps:
       - uses: actions/checkout@v4
 
@@ -57,13 +67,14 @@ jobs:
       - name: Run conversation lifecycle gate
         env:
           LIFECYCLE_ARTIFACT_DIR: lifecycle-artifacts
+          HISTORICAL_HYDRATION_ARTIFACT_DIR: lifecycle-artifacts
           LIFECYCLE_SCENARIO: ${{ matrix.scenario }}
-        run: python tests/browser_conversation_lifecycle.py
+        run: python ${{ matrix.script }}
 
       - name: Upload failure artifacts
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: conversation-lifecycle-failure-${{ matrix.scenario }}
+          name: conversation-lifecycle-failure-${{ matrix.name }}
           path: lifecycle-artifacts
           if-no-files-found: ignore

TESTING.md

@@ -111,6 +111,9 @@ python tests/browser_conversation_lifecycle.py
 
 # Terminal-error lifecycle gate (new row in the proof matrix).
 LIFECYCLE_SCENARIO=terminal-error python tests/browser_conversation_lifecycle.py
+
+# Historical ID-linked transcript hydration row.
+python tests/browser_historical_transcript_hydration.py
 ```
 
 To certify that the gate catches its target failure, the test owns an opt-in
@@ -126,10 +129,16 @@ LIFECYCLE_TEST_BITE=drop-anchor-persistence \
 LIFECYCLE_SCENARIO=terminal-error \
 LIFECYCLE_TEST_BITE=drop-terminal-anchor-row \
   python tests/browser_conversation_lifecycle.py
+
+# Historical-hydration mutation: corrupt one persisted tool-result link so the
+# strict Anchor projection must fail instead of claiming the legacy transcript.
+HISTORICAL_HYDRATION_TEST_BITE=break-tool-link \
+  python tests/browser_historical_transcript_hydration.py
 ```
 
-The dedicated `Conversation lifecycle (informational)` workflow runs both current
-proof rows (`normal` and `terminal-error`) and stays non-blocking while the public
+The dedicated `Conversation lifecycle (informational)` workflow runs the current
+proof rows (`normal`, `terminal-error`, and `historical-transcript-hydration`) and
+stays non-blocking while the public
 matrix expands to additional behavior rows. The maintainer's private QA harness
 remains broader; later public slices will add session switching, reconnect/replay,
 cancellation, compression, and recovery.

static/messages.js

@@ -3714,16 +3714,20 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
       || (Array.isArray(scene&&scene.side_effects)&&scene.side_effects.length)
     );
   }
-  function _attachProjectedAnchorSceneToLastAssistant(messages){
+  function _attachProjectedAnchorSceneToLastAssistant(messages, targetMessage=null, targetIndex=null){
     if(!_anchorRegistry||!Array.isArray(messages)) return false;
-    let lastAsst=null;
-    let lastAsstIndex=-1;
-    for(let i=messages.length-1;i>=0;i--){
-      const candidate=messages[i];
-      if(candidate&&candidate.role==='assistant'){
-        lastAsst=candidate;
-        lastAsstIndex=i;
-        break;
+    let lastAsst=targetMessage;
+    let lastAsstIndex=Number.isInteger(targetIndex)?targetIndex:-1;
+    if(lastAsst){
+      if(lastAsstIndex<0||messages[lastAsstIndex]!==lastAsst) return false;
+    }else{
+      for(let i=messages.length-1;i>=0;i--){
+        const candidate=messages[i];
+        if(candidate&&candidate.role==='assistant'){
+          lastAsst=candidate;
+          lastAsstIndex=i;
+          break;
+        }
       }
     }
     if(!lastAsst) return false;
@@ -3734,13 +3738,93 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
       const hasWorklogRows=_anchorSceneHasWorklogWorthyRows(scene);
       const shouldPersistScene=hasWorklogRows||scene.mode==='hide_all_activity'||hasOwnedOutcomes;
       if(!shouldPersistScene) return false;
+      let sceneKey='';
+      try{ sceneKey=JSON.stringify(scene); }catch(_){ sceneKey=''; }
+      if(
+        sceneKey &&
+        lastAsst._anchor_stream_id===streamId &&
+        lastAsst._anchor_scene_persist_key===sceneKey
+      ) return hasWorklogRows;
       lastAsst._anchor_stream_id=streamId;
       lastAsst._anchor_activity_scene=scene;
+      lastAsst._anchor_scene_persist_key=sceneKey;
       _persistSettledAnchorScene(lastAsst, scene, lastAsstIndex);
       return hasWorklogRows;
     }
     return false;
   }
+  function _settledAnchorRetryOwnerKey(messages, targetIndex, retryStreamId){
+    if(!Array.isArray(messages)||!Number.isInteger(targetIndex)) return '';
+    const target=messages[targetIndex];
+    if(!target||target.role!=='assistant') return '';
+    let turnStart=0;
+    for(let idx=targetIndex-1;idx>=0;idx-=1){
+      if(messages[idx]&&messages[idx].role==='user'){
+        turnStart=idx;
+        break;
+      }
+    }
+    let hasStableOwnerSignal=false;
+    const ownerRows=[];
+    const addUnique=(items,value)=>{
+      const normalized=String(value||'').trim();
+      if(normalized&&!items.includes(normalized)) items.push(normalized);
+    };
+    for(let idx=turnStart;idx<=targetIndex;idx+=1){
+      const message=messages[idx];
+      if(!message||typeof message!=='object') return '';
+      const explicitIds=[];
+      for(const field of ['id','message_id','turn_id','_turn_id','run_id','_run_id']){
+        addUnique(explicitIds,message[field]);
+      }
+      const toolCallIds=[];
+      addUnique(toolCallIds,message.tool_call_id);
+      const addToolOwner=(tool)=>{
+        if(!tool||typeof tool!=='object') return;
+        addUnique(toolCallIds,tool.id||tool.tid||tool.tool_call_id||tool.tool_use_id||tool.call_id);
+      };
+      for(const tool of (Array.isArray(message.tool_calls)?message.tool_calls:[])) addToolOwner(tool);
+      for(const tool of (Array.isArray(message._partial_tool_calls)?message._partial_tool_calls:[])) addToolOwner(tool);
+      for(const part of (Array.isArray(message.content)?message.content:[])) addToolOwner(part);
+      explicitIds.sort();
+      toolCallIds.sort();
+      if(explicitIds.length||toolCallIds.length) hasStableOwnerSignal=true;
+      ownerRows.push({
+        message_ref:_anchorSceneMessageRef(message),
+        reasoning:String(message.reasoning||message._reasoning||message.reasoning_content||message.thinking||'').replace(/\s+/g,' ').trim(),
+        partial:!!message._partial,
+        explicit_ids:explicitIds,
+        tool_call_ids:toolCallIds,
+      });
+    }
+    if(!hasStableOwnerSignal) return '';
+    return JSON.stringify({
+      session_id:activeSid||'',
+      stream_id:String(retryStreamId||''),
+      target_index:targetIndex,
+      messages:ownerRows,
+    });
+  }
+  function _retrySettledAnchorScene(targetMessage, targetIndex, retryStreamId, retryRegistry, retryOwnerKey){
+    if(!targetMessage||!Number.isInteger(targetIndex)) return false;
+    if(!S.session||S.session.session_id!==activeSid) return false;
+    if(S.activeStreamId&&S.activeStreamId!==retryStreamId) return false;
+    if(!_anchorRegistryMap||_anchorRegistryMap.get(retryStreamId)!==retryRegistry) return false;
+    if(!Array.isArray(S.messages)) return false;
+    const currentTarget=S.messages[targetIndex];
+    if(currentTarget!==targetMessage){
+      const currentOwnerKey=_settledAnchorRetryOwnerKey(S.messages,targetIndex,retryStreamId);
+      if(!retryOwnerKey||!currentOwnerKey||currentOwnerKey!==retryOwnerKey) return false;
+      if(targetMessage._anchor_stream_id===retryStreamId){
+        if(currentTarget._anchor_stream_id==null) currentTarget._anchor_stream_id=targetMessage._anchor_stream_id;
+        if(currentTarget._anchor_scene_persist_key==null) currentTarget._anchor_scene_persist_key=targetMessage._anchor_scene_persist_key;
+        if(!currentTarget._anchor_activity_scene&&targetMessage._anchor_activity_scene){
+          currentTarget._anchor_activity_scene=targetMessage._anchor_activity_scene;
+        }
+      }
+    }
+    return _attachProjectedAnchorSceneToLastAssistant(S.messages,currentTarget,targetIndex);
+  }
   function _upsertAnchorProcessProse(displayText, options={}){
     const text=String(displayText||'').trim();
     if(!text||!_anchorRegistry) return null;
@@ -6230,6 +6314,8 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
         _scheduleAnchorRegistryCleanup();
         clearLiveToolCards();if(!assistantText)removeThinking();
         let isRecoveryControlMessage=false;
+        let _anchorRetryTarget=null;
+        let _anchorRetryIndex=-1;
         try{
           const isRateLimit=d.type==='rate_limit';
           const isQuotaExhausted=d.type==='quota_exhausted';
@@ -6264,10 +6350,26 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
             S.messages.push({role:'assistant',content:`**${label}:** ${d.message}${hint}`,provider_details:details,provider_details_label:detailsLabel,_compressionRecovery:recovery||undefined});
             _attachProjectedAnchorSceneToLastAssistant(S.messages);
           }
+          if(!isRecoveryControlMessage){
+            _anchorRetryTarget=[...S.messages].reverse().find(m=>m&&m.role==='assistant')||null;
+            _anchorRetryIndex=_anchorRetryTarget?S.messages.indexOf(_anchorRetryTarget):-1;
+          }
         }catch(_){
           S.messages.push({role:'assistant',content:'**Error:** An error occurred. Check server logs.'});
           _attachProjectedAnchorSceneToLastAssistant(S.messages);
         }
+        if(_anchorRetryTarget&&_anchorRetryIndex>=0){
+          const _retryTarget=_anchorRetryTarget;
+          const _retryIndex=_anchorRetryIndex;
+          const _retryStreamId=streamId;
+          const _retryRegistry=_anchorRegistry;
+          const _retryOwnerKey=_settledAnchorRetryOwnerKey(S.messages,_retryIndex,_retryStreamId);
+          // Retry only for the exact terminal assistant and registry generation.
+          // A refresh replacement must prove the same full turn and tool owner.
+          setTimeout(()=>{
+            _retrySettledAnchorScene(_retryTarget,_retryIndex,_retryStreamId,_retryRegistry,_retryOwnerKey);
+          },0);
+        }
         if(isRecoveryControlMessage){
           (async()=>{
             if(await _restoreSettledSession(source, {preserveVisibleOnShorterTerminalSnapshot:true})) return;

tests/browser_conversation_lifecycle.py

@@ -488,6 +488,11 @@ def _activity_snapshot(page) -> dict:
             Array.from(document.querySelectorAll('.assistant-turn')).pop() || null;
           const groups = turn ? Array.from(turn.querySelectorAll('[data-anchor-scene-owner="1"]')) : [];
           const rows = turn ? Array.from(turn.querySelectorAll('[data-anchor-scene-row="1"]')) : [];
+          const sceneRows = assistants.flatMap(message =>
+            message && message._anchor_activity_scene && Array.isArray(message._anchor_activity_scene.activity_rows)
+              ? message._anchor_activity_scene.activity_rows
+              : []
+          );
           const visibleFinal = turn ? Array.from(turn.querySelectorAll('.assistant-segment .msg-body'))
             .filter(el => {
               const segment = el.closest('.assistant-segment');
@@ -518,6 +523,11 @@ def _activity_snapshot(page) -> dict:
             })),
             rows: rows.map(row => ({
               role: row.getAttribute('data-anchor-row-role'),
+              rowId: row.getAttribute('data-anchor-row-id') || '',
+              toolCallId: (sceneRows.find(sceneRow =>
+                String(sceneRow && (sceneRow.row_id || sceneRow.local_id) || '') ===
+                String(row.getAttribute('data-anchor-row-id') || '')
+              ) || {}).tool_call_id || '',
               source: row.getAttribute('data-anchor-source-event-type'),
               status: row.getAttribute('data-anchor-row-status'),
               tool: row.getAttribute('data-tool-name'),

tests/browser_historical_transcript_hydration.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""Public browser proof for ID-linked historical tool transcript hydration.
+
+This boots the real WebUI with isolated state, imports a persisted transcript
+through the public API, and loads it through the normal browser session path.
+The supported OpenAI-compatible declaration/result/final chain must hydrate to
+one Anchor Worklog and remain equivalent after hard reload.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import sys
+import tempfile
+from pathlib import Path
+
+from browser_conversation_lifecycle import (
+    _activity_snapshot,
+    _capture_page_errors,
+    _expand_settled_worklog,
+    _start_webui_server,
+    _terminate_process,
+)
+
+
+FINAL_TEXT = "Historical hydration proof final answer."
+TOOL_IDS = ("historical-tool-first", "historical-tool-second")
+TEST_BITE = os.environ.get("HISTORICAL_HYDRATION_TEST_BITE", "").strip()
+
+
+def _historical_messages() -> list[dict]:
+    first_result_id = (
+        "missing-historical-tool-id"
+        if TEST_BITE == "break-tool-link"
+        else TOOL_IDS[0]
+    )
+    return [
+        {"role": "user", "content": "Inspect both historical files."},
+        {
+            "role": "assistant",
+            "content": None,
+            "tool_calls": [
+                {
+                    "id": TOOL_IDS[0],
+                    "type": "function",
+                    "function": {
+                        "name": "terminal",
+                        "arguments": '{"command":"pwd"}',
+                    },
+                },
+                {
+                    "id": TOOL_IDS[1],
+                    "type": "function",
+                    "function": {
+                        "name": "read_file",
+                        "arguments": '{"path":"README.md"}',
+                    },
+                },
+            ],
+        },
+        {
+            "role": "tool",
+            "tool_call_id": TOOL_IDS[1],
+            "content": "README historical contents",
+        },
+        {
+            "role": "tool",
+            "tool_call_id": first_result_id,
+            "content": "/isolated/workspace",
+        },
+        {"role": "assistant", "content": FINAL_TEXT},
+    ]
+
+
+def _assert_hydrated(snapshot: dict) -> None:
+    assert snapshot["live"] is False, snapshot
+    assert snapshot["groupCount"] == 1, snapshot
+    assert snapshot["clientState"]["busy"] is False, snapshot
+    rows = [row for row in snapshot["rows"] if row["role"] == "tool"]
+    assert [row["tool"] for row in rows] == ["terminal", "read_file"], snapshot
+    assert len(rows) == 2, snapshot
+    assert [row["toolCallId"] for row in rows] == list(TOOL_IDS), snapshot
+    assert all(row["rowId"].startswith(f"{row['toolCallId']}:") for row in rows), snapshot
+    assert "/isolated/workspace" in rows[0]["text"], snapshot
+    assert "README historical contents" not in rows[0]["text"], snapshot
+    assert "README historical contents" in rows[1]["text"], snapshot
+    assert "/isolated/workspace" not in rows[1]["text"], snapshot
+    assert sum(FINAL_TEXT in text for text in snapshot["visibleFinal"]) == 1, snapshot
+    assert snapshot["transcript"].count(FINAL_TEXT) == 1, snapshot
+
+
+def _semantic_snapshot(snapshot: dict) -> dict:
+    return {
+        "tools": [
+            {"tool": row["tool"], "text": row["text"]}
+            for row in snapshot["rows"]
+            if row["role"] == "tool"
+        ],
+        "final": snapshot["visibleFinal"],
+    }
+
+
+def _import_and_load(page) -> str:
+    return page.evaluate(
+        """async ({messages, finalText}) => {
+          const response = await fetch('/api/session/import', {
+            method: 'POST',
+            headers: {'Content-Type': 'application/json'},
+            body: JSON.stringify({
+              title: 'Historical hydration browser proof',
+              messages,
+            }),
+          });
+          const data = await response.json();
+          if (!response.ok || !data.session || !data.session.session_id) {
+            throw new Error(`session import failed: ${JSON.stringify(data)}`);
+          }
+          await loadSession(data.session.session_id);
+          await new Promise((resolve, reject) => {
+            const deadline = Date.now() + 10000;
+            const check = () => {
+              const text = (document.querySelector('#msgInner') || {}).innerText || '';
+              if (text.includes(finalText)) return resolve();
+              if (Date.now() >= deadline) return reject(new Error('loaded session never rendered final text'));
+              setTimeout(check, 50);
+            };
+            check();
+          });
+          return data.session.session_id;
+        }""",
+        {"messages": _historical_messages(), "finalText": FINAL_TEXT},
+    )
+
+
+def main() -> int:
+    if TEST_BITE not in {"", "break-tool-link"}:
+        raise ValueError(
+            f"Unsupported HISTORICAL_HYDRATION_TEST_BITE {TEST_BITE!r}; "
+            "expected one of '', 'break-tool-link'"
+        )
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        print("SETUP FAIL: playwright is not installed", file=sys.stderr)
+        return 2
+
+    repo_root = Path(__file__).resolve().parent.parent
+    state_tmp = tempfile.TemporaryDirectory(prefix="hermes-historical-hydration-")
+    state_dir = Path(state_tmp.name)
+    artifact_env = str(os.environ.get("HISTORICAL_HYDRATION_ARTIFACT_DIR") or "").strip()
+    artifact_dir_owned = not bool(artifact_env)
+    artifact_dir = Path(artifact_env) if artifact_env else Path(
+        tempfile.mkdtemp(prefix="hermes-historical-hydration-artifacts-")
+    )
+    artifact_dir.mkdir(parents=True, exist_ok=True)
+    agent_dir = state_dir / "no-agent"
+    workspace_dir = state_dir / "workspace"
+    agent_dir.mkdir()
+    workspace_dir.mkdir()
+    (agent_dir / "run_agent.py").write_text("\"\"\"Test-only agent stub.\"\"\"\n", encoding="utf-8")
+    env = os.environ.copy()
+    for key in list(env):
+        if key.endswith("_API_KEY"):
+            env.pop(key, None)
+    for key in (
+        "API_SERVER_KEY",
+        "HERMES_WEBUI_PASSWORD",
+        "HERMES_WEBUI_EXTENSION_DIR",
+        "HERMES_WEBUI_EXTENSION_MANIFEST",
+    ):
+        env.pop(key, None)
+    env.update({
+        "HERMES_WEBUI_HOST": "127.0.0.1",
+        "HERMES_WEBUI_STATE_DIR": str(state_dir / "webui-state"),
+        "HERMES_HOME": str(state_dir / "hermes-home"),
+        "HERMES_BASE_HOME": str(state_dir / "hermes-home"),
+        "HERMES_CONFIG_PATH": str(state_dir / "hermes-home" / "config.yaml"),
+        "HERMES_WEBUI_SKIP_ONBOARDING": "1",
+        "HERMES_WEBUI_AGENT_DIR": str(agent_dir),
+        "HERMES_WEBUI_DEFAULT_WORKSPACE": str(workspace_dir),
+        "NO_PROXY": "127.0.0.1,localhost",
+        "no_proxy": "127.0.0.1,localhost",
+    })
+
+    proc = log = log_path = playwright = browser = page = None
+    errors: list[tuple[str, str]] = []
+    exit_code = 1
+    try:
+        proc, log, log_path, base_url = _start_webui_server(repo_root, env, artifact_dir)
+        playwright = sync_playwright().start()
+        browser = playwright.chromium.launch(
+            headless=True,
+            args=["--no-sandbox", "--disable-dev-shm-usage"],
+        )
+        context = browser.new_context(base_url=base_url)
+        page = context.new_page()
+        errors = _capture_page_errors(page)
+        page.goto("/", wait_until="domcontentloaded")
+        page.wait_for_selector("#msg", state="visible", timeout=15000)
+        page.wait_for_function(
+            "() => typeof window._autoScrollFollow === 'boolean'",
+            timeout=15000,
+        )
+        session_id = _import_and_load(page)
+        assert session_id, "imported session id missing"
+        _expand_settled_worklog(page)
+        hydrated = _activity_snapshot(page)
+        _assert_hydrated(hydrated)
+        print("OK  historical load: ID-linked transcript hydrated to one Anchor Worklog")
+
+        page.reload(wait_until="domcontentloaded")
+        page.wait_for_function(
+            "text => (document.querySelector('#msgInner') || {}).innerText?.includes(text)",
+            arg=FINAL_TEXT,
+            timeout=15000,
+        )
+        _expand_settled_worklog(page)
+        reloaded = _activity_snapshot(page)
+        _assert_hydrated(reloaded)
+        assert _semantic_snapshot(reloaded) == _semantic_snapshot(hydrated), {
+            "hydrated": _semantic_snapshot(hydrated),
+            "reloaded": _semantic_snapshot(reloaded),
+        }
+        print("OK  hard reload: historical Anchor Worklog remains semantically identical")
+        if errors:
+            raise AssertionError(f"unexpected browser errors: {errors!r}")
+        context.close()
+        browser.close()
+        browser = None
+        print("\nHISTORICAL TRANSCRIPT HYDRATION GATE PASSED")
+        exit_code = 0
+        return 0
+    except Exception as error:
+        print(f"\nHISTORICAL TRANSCRIPT HYDRATION GATE FAILED: {error}", file=sys.stderr)
+        try:
+            if page is not None:
+                page.screenshot(path=str(artifact_dir / "failure.png"), full_page=True)
+                (artifact_dir / "snapshot.json").write_text(
+                    json.dumps(
+                        {
+                            "test_bite": TEST_BITE or None,
+                            "browser_errors": errors,
+                            "messages": page.evaluate(
+                                """() => (S.messages || []).map((message) => ({
+                                  role: message && message.role,
+                                  content: message && message.content,
+                                  tool_call_id: message && message.tool_call_id,
+                                  tool_calls: Array.isArray(message && message.tool_calls)
+                                    ? message.tool_calls.map((call) => call && call.id)
+                                    : null,
+                                  has_anchor_scene: Boolean(message && message._anchor_activity_scene),
+                                }))"""
+                            ),
+                            "dom": _activity_snapshot(page),
+                        },
+                        indent=2,
+                    ),
+                    encoding="utf-8",
+                )
+        except Exception as artifact_error:
+            print(f"Could not capture browser artifacts: {artifact_error}", file=sys.stderr)
+        print(f"Artifacts: {artifact_dir}", file=sys.stderr)
+        return 1
+    finally:
+        if browser is not None:
+            browser.close()
+        if playwright is not None:
+            playwright.stop()
+        _terminate_process(proc)
+        if log is not None:
+            log.close()
+        if proc is not None and proc.returncode not in (None, 0, -15):
+            print(f"WebUI server exit code: {proc.returncode}", file=sys.stderr)
+        if log_path is not None and log_path.exists() and proc is not None and proc.returncode not in (None, 0, -15):
+            print(log_path.read_text(encoding="utf-8", errors="replace")[-4000:], file=sys.stderr)
+        state_tmp.cleanup()
+        if artifact_dir_owned and exit_code == 0:
+            shutil.rmtree(artifact_dir, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_issue3820_chat_activity_display_mode.py

@@ -315,11 +315,11 @@ process.stdout.write(JSON.stringify({{appendResult: appendResult === undefined,
 
 
 def test_chat_activity_display_mode_settled_hide_all_scene_persists_without_worklog():
-    start = MESSAGES_JS.index("function _attachProjectedAnchorSceneToLastAssistant(messages){")
+    start = MESSAGES_JS.index("function _attachProjectedAnchorSceneToLastAssistant(")
     end = MESSAGES_JS.index("function _upsertAnchorProcessProse", start)
     block = MESSAGES_JS[start:end]
     persist_index = block.index("_persistSettledAnchorScene(lastAsst, scene, lastAsstIndex);")
-    return_index = block.index("return hasWorklogRows;")
+    return_index = block.rfind("return hasWorklogRows;")
 
     assert "const hasWorklogRows=_anchorSceneHasWorklogWorthyRows(scene);" in block
     assert "const hasOwnedOutcomes=_anchorSceneHasOwnedOutcomes(scene);" in block

tests/test_live_to_final_anchor_visible_order.py

@@ -854,6 +854,170 @@ def test_application_error_settlement_attaches_projected_anchor_scene_before_ren
     assert synthetic_push_idx < synthetic_attach_idx < render_idx
 
 
+def test_deferred_application_error_retry_is_owner_fenced_and_idempotent():
+    attach = _function_body(MESSAGES_JS, "_attachProjectedAnchorSceneToLastAssistant")
+    message_ref = _function_body(MESSAGES_JS, "_anchorSceneMessageRef")
+    owner_key = _function_body(MESSAGES_JS, "_settledAnchorRetryOwnerKey")
+    retry = _function_body(MESSAGES_JS, "_retrySettledAnchorScene")
+    script = f"""
+const activeSid='sid-A';
+const streamId='stream-A';
+const _anchorRegistry={{generation:'A'}};
+const _anchorRegistryMap=new Map([['stream-A',_anchorRegistry]]);
+const S={{session:{{session_id:'sid-A'}},activeStreamId:null,messages:[]}};
+let scene={{version:'activity_scene_v1',mode:'compact_worklog',activity_rows:[{{role:'tool'}}]}};
+let persisted=[];
+let deferredRetry=null;
+function _projectLiveAnchorActivityScene(){{ return scene; }}
+function _completeSettledAnchorSceneForTurn(messages,index,projected){{ return projected; }}
+function _anchorSceneHasOwnedOutcomes(){{ return false; }}
+function _anchorSceneHasWorklogWorthyRows(){{ return true; }}
+function _persistSettledAnchorScene(message,nextScene,index){{ persisted.push({{message,index,nextScene}}); }}
+function _attachProjectedAnchorSceneToLastAssistant(messages,targetMessage=null,targetIndex=null){{{attach}}}
+function _anchorSceneMessageRef(message){{{message_ref}}}
+function _settledAnchorRetryOwnerKey(messages,targetIndex,retryStreamId){{{owner_key}}}
+function _retrySettledAnchorScene(targetMessage,targetIndex,retryStreamId,retryRegistry,retryOwnerKey){{{retry}}}
+function scheduleRetry(targetMessage,targetIndex,retryStreamId,retryRegistry,retryOwnerKey){{
+  deferredRetry=()=>_retrySettledAnchorScene(targetMessage,targetIndex,retryStreamId,retryRegistry,retryOwnerKey);
+}}
+const sharedPrefix='x'.repeat(160);
+const originalMessages=[
+  {{role:'user',content:'inspect the current turn'}},
+  {{
+    role:'assistant',
+    content:sharedPrefix+' owner-A tail',
+    tool_calls:[{{id:'tool-A',function:{{name:'terminal',arguments:'{{}}'}}}}],
+  }},
+  {{role:'assistant',content:'**Error:** gateway failed'}},
+];
+const assistantA=originalMessages[2];
+S.messages=originalMessages;
+const refreshedMessages=JSON.parse(JSON.stringify(originalMessages));
+const initialResult=_attachProjectedAnchorSceneToLastAssistant(S.messages,assistantA,2);
+const persistedAfterInitial=persisted.length;
+const ownerKey=_settledAnchorRetryOwnerKey(S.messages,2,'stream-A');
+scheduleRetry(assistantA,2,'stream-A',_anchorRegistry,ownerKey);
+S.messages=refreshedMessages;
+const refreshedA=S.messages[2];
+const firstResult=deferredRetry();
+const persistedAfterRefresh=persisted.length;
+scene={{
+  version:'activity_scene_v1',
+  mode:'compact_worklog',
+  activity_rows:[{{role:'tool'}},{{role:'thinking'}}],
+}};
+scheduleRetry(assistantA,2,'stream-A',_anchorRegistry,ownerKey);
+const secondResult=deferredRetry();
+const persistedAfterLateRow=persisted.length;
+scheduleRetry(assistantA,2,'stream-A',_anchorRegistry,ownerKey);
+const thirdResult=deferredRetry();
+const positiveState={{
+  ownerKeyPresent:Boolean(ownerKey),
+  initialResult,
+  firstResult,
+  secondResult,
+  thirdResult,
+  persistedAfterInitial,
+  persistedAfterRefresh,
+  persistedAfterLateRow,
+  refreshedHasScene:Boolean(refreshedA._anchor_activity_scene),
+  persisted:persisted.length,
+}};
+
+const ambiguousB={{role:'assistant',content:'**Error:** gateway failed'}};
+S.messages=[
+  {{role:'user',content:'inspect the current turn'}},
+  {{
+    role:'assistant',
+    content:sharedPrefix+' owner-B tail',
+    tool_calls:[{{id:'tool-B',function:{{name:'terminal',arguments:'{{}}'}}}}],
+  }},
+  ambiguousB,
+];
+scheduleRetry(assistantA,2,'stream-A',_anchorRegistry,ownerKey);
+const ambiguousResult=deferredRetry();
+const ambiguousState={{
+  result:ambiguousResult,
+  bHasScene:Boolean(ambiguousB._anchor_activity_scene),
+  bStreamId:ambiguousB._anchor_stream_id||null,
+  persisted:persisted.length,
+}};
+
+const ownerlessA={{role:'assistant',content:'timestamp-less owner'}};
+S.messages=[ownerlessA];
+const ownerlessKey=_settledAnchorRetryOwnerKey(S.messages,0,'stream-A');
+scheduleRetry(ownerlessA,0,'stream-A',_anchorRegistry,ownerlessKey);
+const ownerlessExactResult=deferredRetry();
+const ownerlessB={{role:'assistant',content:'timestamp-less owner'}};
+S.messages=[ownerlessB];
+scheduleRetry(ownerlessA,0,'stream-A',_anchorRegistry,ownerlessKey);
+const ownerlessReplacementResult=deferredRetry();
+const ownerlessState={{
+  ownerKey:ownerlessKey,
+  exactResult:ownerlessExactResult,
+  replacementResult:ownerlessReplacementResult,
+  bHasScene:Boolean(ownerlessB._anchor_activity_scene),
+  persisted:persisted.length,
+}};
+
+S.messages=JSON.parse(JSON.stringify(originalMessages));
+S.activeStreamId='stream-B';
+scheduleRetry(assistantA,2,'stream-A',_anchorRegistry,ownerKey);
+const streamResult=deferredRetry();
+const streamState={{result:streamResult,persisted:persisted.length}};
+S.activeStreamId=null;
+_anchorRegistryMap.set('stream-A',{{generation:'B'}});
+scheduleRetry(assistantA,2,'stream-A',_anchorRegistry,ownerKey);
+const registryResult=deferredRetry();
+const registryState={{result:registryResult,persisted:persisted.length}};
+_anchorRegistryMap.set('stream-A',_anchorRegistry);
+S.session={{session_id:'sid-B'}};
+scheduleRetry(assistantA,2,'stream-A',_anchorRegistry,ownerKey);
+const sessionResult=deferredRetry();
+const sessionState={{result:sessionResult,persisted:persisted.length}};
+console.log(JSON.stringify({{
+  positiveState,
+  ambiguousState,
+  ownerlessState,
+  streamState,
+  registryState,
+  sessionState,
+}}));
+"""
+    result = subprocess.run([NODE, "-e", script], text=True, capture_output=True, check=False)
+
+    assert result.returncode == 0, result.stderr
+    data = json.loads(result.stdout)
+    assert data["positiveState"] == {
+        "ownerKeyPresent": True,
+        "initialResult": True,
+        "firstResult": True,
+        "secondResult": True,
+        "thirdResult": True,
+        "persistedAfterInitial": 1,
+        "persistedAfterRefresh": 1,
+        "persistedAfterLateRow": 2,
+        "refreshedHasScene": True,
+        "persisted": 2,
+    }
+    assert data["ambiguousState"] == {
+        "result": False,
+        "bHasScene": False,
+        "bStreamId": None,
+        "persisted": 2,
+    }
+    assert data["ownerlessState"] == {
+        "ownerKey": "",
+        "exactResult": True,
+        "replacementResult": False,
+        "bHasScene": False,
+        "persisted": 3,
+    }
+    assert data["streamState"] == {"result": False, "persisted": 3}
+    assert data["registryState"] == {"result": False, "persisted": 3}
+    assert data["sessionState"] == {"result": False, "persisted": 3}
+
+
 def test_connection_error_terminal_message_attaches_projected_anchor_scene_before_render():
     error = _function_body(MESSAGES_JS, "_handleStreamError")
 

tests/test_stable_assistant_turn_anchor_registry.py

@@ -866,7 +866,7 @@ function _projectLiveAnchorActivityScene(){{
 }}
 function _completeSettledAnchorSceneForTurn(messages,index,scene){{ return scene; }}
 function _persistSettledAnchorScene(){{ persisted+=1; }}
-function _attachProjectedAnchorSceneToLastAssistant(messages){{{attach_scene}}}
+function _attachProjectedAnchorSceneToLastAssistant(messages,targetMessage=null,targetIndex=null){{{attach_scene}}}
 const messages=[{{role:'assistant',content:'final answer'}}];
 const renderedWorklog=_attachProjectedAnchorSceneToLastAssistant(messages);
 console.log(JSON.stringify({{