Merge pull request #6786 from nesquena/stage/6670-0805

Release exp-v0.52.164: active-turn user-row dedup on mid-run reload (#6670)

CHANGELOG.md

@@ -21,6 +21,8 @@
 
 ### Fixed
 
+- **A mid-run reload no longer briefly shows the active turn's prompt twice.** While a turn is live the server can reconcile the current user row into the returned transcript (via the state.db merge) while `pending_user_message` is still set, and that row is then followed by the same turn's assistant/tool output — so the strict tail scan reported "no current user row" and the client materialized the pending prompt a second time, showing a duplicate user bubble until the settle render. The client now also matches the active turn's row by exact identity — the server-stamped `_active_turn_token` (`stream_id:started_at`) or a timestamp equal to `pending_started_at` within a precision-only epsilon — and adopts it instead of re-rendering. Matching is identity-only, never proximity: a rapid same-text repeat (e.g. sending "continue" twice ~1s apart) is treated as ambiguous and still materializes as its own turn, so a legitimate repeated prompt is never swallowed. Thanks @happy5318. (#6670)
+
 - **"Force recovery" now repairs a dirty Stable checkout that has no promoted release ref.** On the Stable channel a WebUI checkout with local modifications but no promoted ref to move to previously reported "already up to date" and did nothing, leaving the working tree stuck. Force recovery now detects a *confirmed* dirty tree (strict `git diff-index` exit-1 only — a timeout or probe failure is treated as unknown and never triggers a destructive reset) and discards the local changes by resetting to the current `HEAD`, so the commit itself never moves and the channel is unchanged. Clean or unknown checkouts stay a no-op, the ancestor-rewind guard is preserved, and ignored build/cache artifacts survive (no `git clean -x`); a `git clean -fd` failure (e.g. a Windows reserved-name file, #4914) stays non-fatal. Thanks @rodboev. (#6688)
 
 - **OIDC-only deployments get a clear startup warning for ambiguous whitespace `allow_values`.** When `webui_oidc.allow_values` (or `HERMES_WEBUI_OIDC_ALLOW_VALUES`) was given as a scalar with internal whitespace like `"alice@example.com bob@example.com"`, whitespace is not a separator so the whole string is treated as one allow-list entry — silently locking out the intended users. Startup (and OIDC resolution) now emit a warning naming the ambiguity and pointing to the comma-delimited scalar or YAML-array forms, while a genuinely intentional multi-word group is called out as already correct. Parsing and enforcement are byte-for-byte unchanged — this is a diagnostic only, it never broadens authorization. Thanks @rodboev. (#6663, #6603)

static/ui.js

@@ -10373,22 +10373,126 @@ function _pendingCurrentTailUserMessage(messages){
   return null;
 }
 
+// Precision-only epsilon when matching a transcript row against
+// session.pending_started_at. The server stamps the active turn's user row with
+// the exact pending_started_at float; state.db round-trips can lose
+// sub-microsecond precision, so 1e-6 absorbs float drift without ever treating a
+// whole-second (or sub-second) difference as identity. Anything wider is
+// ambiguous — a fast "继续"/"continue" double-send lands ~1s after the previous
+// turn — and must NOT match: fail toward materializing the pending turn (a
+// harmless transient duplicate the settle render clears) rather than hiding a
+// turn and moving its attachments onto an earlier row.
+const _PENDING_ACTIVE_TURN_TS_EPSILON=1e-6;
+
+function _messageTimestampSeconds(msg){
+  if(!msg) return null;
+  const raw=msg._ts!=null?msg._ts:msg.timestamp;
+  const value=Number(raw);
+  return Number.isFinite(value)&&value>0?value:null;
+}
+
+/**
+ * Exact-identity match for the active turn's user row via its
+ * `_active_turn_token`, mirroring the server's `build_active_turn_token`
+ * ("{stream_id}:{started_at}") stamped by the eager-checkpoint path. The token
+ * embeds the stream_id, which no other turn can share, so a row carrying the
+ * current session's token IS the active turn — no timestamp tolerance needed.
+ */
+function _activeTurnTokenMatches(msg, session){
+  if(!msg||typeof msg._active_turn_token!=='string') return false;
+  const streamId=session&&session.active_stream_id;
+  const startedAt=Number(session&&session.pending_started_at);
+  if(!streamId||!Number.isFinite(startedAt)||startedAt<=0) return false;
+  const sep=msg._active_turn_token.lastIndexOf(':');
+  if(sep<=0) return false;
+  if(msg._active_turn_token.slice(0,sep).trim()!==String(streamId).trim()) return false;
+  const tokenStarted=Number(msg._active_turn_token.slice(sep+1));
+  return Number.isFinite(tokenStarted)&&tokenStarted>0
+    && Math.abs(tokenStarted-startedAt)<=_PENDING_ACTIVE_TURN_TS_EPSILON;
+}
+
+/**
+ * Find the active turn's user row even when the current turn's output already
+ * follows it in the transcript.
+ *
+ * While a turn is live the server can reconcile the current user row into the
+ * returned transcript (the sidecar/state.db merge picks it up from state.db,
+ * where the agent writes it immediately) while `pending_user_message` is still
+ * set. The row is then followed by that same turn's assistant/tool rows, so the
+ * strict tail scan in `_pendingCurrentTailUserMessage` stops at the completed
+ * assistant and reports "no current user row" — and the caller materializes the
+ * pending prompt a SECOND time, rendering a duplicate user bubble until the
+ * settle render replaces the list.
+ *
+ * Scanning past assistant/tool rows alone would be wrong: a user who submits the
+ * same text twice in a row (a plain "继续" follow-up) legitimately gets two
+ * identical user turns, and matching on text would swallow the new one. The
+ * discriminator is therefore exact identity, never proximity: the active turn's
+ * row either carries the server-stamped `_active_turn_token` (stream_id +
+ * started_at — unique to this turn), or its timestamp equals `pending_started_at`
+ * within a precision-only epsilon that absorbs float/state.db drift but never a
+ * full second. A whole-second (or sub-second) mismatch is ambiguous and returns
+ * null so the caller materializes the pending turn — the transient duplicate is
+ * harmless, hiding a turn + moving its attachments is not. Text equality is
+ * still required downstream, so a false match needs identical text AND an exact
+ * identity signal.
+ */
+function _pendingActiveTurnUserMessage(messages, session){
+  const startedAt=Number(session?.pending_started_at);
+  if(!Number.isFinite(startedAt)||startedAt<=0) return null;
+  const list=Array.isArray(messages)?messages:[];
+  for(let i=list.length-1;i>=0;i--){
+    const msg=list[i];
+    if(!msg||String(msg.role||'')!=='user') continue;
+    if(typeof _isContextCompactionMessage==='function'&&_isContextCompactionMessage(msg)) continue;
+    // Unambiguous: the row carries the active turn's exact token
+    // (stream_id + started_at) stamped by the server's eager-checkpoint path.
+    if(typeof _activeTurnTokenMatches==='function'&&_activeTurnTokenMatches(msg,session)) return msg;
+    // Unambiguous: the row's timestamp IS pending_started_at within
+    // precision-only float drift (never a whole second).
+    const ts=_messageTimestampSeconds(msg);
+    if(ts===null) continue;
+    if(Math.abs(ts-startedAt)<=_PENDING_ACTIVE_TURN_TS_EPSILON) return msg;
+  }
+  // Any wider drift (whole-second truncation, a rapid repeat ~1s later) is
+  // ambiguous: return null so getPendingSessionMessage() materializes the
+  // pending turn rather than guessing.
+  return null;
+}
+
 function getPendingSessionMessage(session, messagesOverride=null){
   const text=String(session?.pending_user_message||'').trim();
   if(!text) return null;
   const attachments=Array.isArray(session?.pending_attachments)?session.pending_attachments.filter(Boolean):[];
   const sourceMessages=Array.isArray(messagesOverride)?messagesOverride:session?.messages;
   const messages=Array.isArray(sourceMessages)?sourceMessages:[];
+  const pendingCandidate={role:'user',content:text};
+  const _matchesPending=(row)=>{
+    if(!row) return false;
+    return typeof _sameTranscriptMessage==='function'
+      ? _sameTranscriptMessage(row,pendingCandidate)
+      : String(msgContent(row)||'').trim()===text;
+  };
+  const _adoptExistingRow=(row)=>{
+    if(attachments.length&&!row.attachments?.length) row.attachments=attachments;
+    return null;
+  };
   const currentTailUser=_pendingCurrentTailUserMessage(messages);
   if(currentTailUser){
-    const pendingCandidate={role:'user',content:text};
-    const sameCurrentTurn=typeof _sameTranscriptMessage==='function'
-      ? _sameTranscriptMessage(currentTailUser,pendingCandidate)
-      : String(msgContent(currentTailUser)||'').trim()===text;
-    if(sameCurrentTurn){
-      if(attachments.length&&!currentTailUser.attachments?.length) currentTailUser.attachments=attachments;
-      return null;
-    }
+    const sameCurrentTurn=_matchesPending(currentTailUser);
+    if(sameCurrentTurn) return _adoptExistingRow(currentTailUser);
+  }
+  // Fallback: the current turn's user row is already in the transcript but the
+  // strict tail scan above could not see it because this turn's assistant/tool
+  // output follows it. Matched by pending_started_at, so previous turns that
+  // repeat the same text are unaffected. Guarded with typeof so a partial load
+  // (or a static probe that extracts only some helpers) degrades to the
+  // original strict-tail behaviour instead of throwing.
+  const activeTurnUser=typeof _pendingActiveTurnUserMessage==='function'
+    ? _pendingActiveTurnUserMessage(messages,session)
+    : null;
+  if(activeTurnUser&&activeTurnUser!==currentTailUser&&_matchesPending(activeTurnUser)){
+    return _adoptExistingRow(activeTurnUser);
   }
   return {
     role:'user',

tests/test_run_journal_frontend_static.py

@@ -24,6 +24,20 @@ def _function_body(src: str, signature: str) -> str:
     raise AssertionError(f"could not extract function body for {signature!r}")
 
 
+def _optional_function_body(src: str, signature: str) -> str:
+    """Extract a helper if present, else return "".
+
+    Used for helpers whose ABSENCE is itself the regression under test: the
+    probe must still run so the behavioral assertion reports the duplicate,
+    rather than dying with a substring-not-found extraction error that says
+    nothing about observable behavior.
+    """
+    try:
+        return _function_body(src, signature)
+    except (ValueError, AssertionError):
+        return ""
+
+
 def _run_session_identity_probe() -> dict:
     prompt = "same submitted prompt\nwith a second line"
     workspace_prompt = f"[Workspace::v1: /tmp/hermes-webui]\n{prompt}"
@@ -197,12 +211,16 @@ def _run_pending_session_message_probe() -> dict:
             _function_body(SESSIONS_SRC, "function _normalizeUserTranscriptText"),
             _function_body(SESSIONS_SRC, "function _sameTranscriptMessage"),
             _function_body(UI_SRC, "function _pendingCurrentTailUserMessage"),
+            _optional_function_body(UI_SRC, "function _messageTimestampSeconds"),
+            _optional_function_body(UI_SRC, "function _activeTurnTokenMatches"),
+            _optional_function_body(UI_SRC, "function _pendingActiveTurnUserMessage"),
             _function_body(UI_SRC, "function _isContextCompactionText"),
             _function_body(UI_SRC, "function _isContextCompactionMessage"),
             _function_body(UI_SRC, "function getPendingSessionMessage"),
         ]
     )
     script = f"""
+const _PENDING_ACTIVE_TURN_TS_EPSILON=1e-6;
 {helpers}
 const prompt = {json.dumps(prompt)};
 const historical = {{role:'user', content:prompt, _ts:1}};
@@ -270,6 +288,76 @@ const repeatedCompletedPromptCount = repeatedCompletedMessages.filter(
 ).length;
 const compactionCurrentTail = _pendingCurrentTailUserMessage([historical, historicalAnswer, currentTailForCompaction, compactionMarker]);
 
+// Mid-run reload: the server already reconciled the active turn's user row into
+// the transcript AND this turn's assistant/tool output follows it, while
+// pending_user_message is still set. The strict tail scan cannot see the user
+// row (it stops at the completed assistant), so without the pending_started_at
+// fallback the prompt is materialized a second time -> duplicate user bubble.
+const midRunStartedAt = 500;
+const midRunTranscript = [
+  {{role:'user', content:'earlier question', timestamp:100}},
+  {{role:'assistant', content:'earlier answer', timestamp:200}},
+  {{role:'user', content:prompt, timestamp:midRunStartedAt}},
+  {{role:'assistant', content:'', timestamp:midRunStartedAt+15}},
+  {{role:'tool', content:'{{"ok":true}}', timestamp:midRunStartedAt+16}},
+  {{role:'assistant', content:'partial answer', timestamp:midRunStartedAt+60}},
+];
+const midRunResult = getPendingSessionMessage(
+  {{pending_user_message:prompt, pending_started_at:midRunStartedAt}},
+  midRunTranscript
+);
+const midRunWorkspaceTranscript = [
+  {{role:'user', content:'earlier question', timestamp:100}},
+  {{role:'assistant', content:'earlier answer', timestamp:200}},
+  {{role:'user', content:{json.dumps(current_workspace_prompt)}, timestamp:midRunStartedAt}},
+  {{role:'assistant', content:'partial answer', timestamp:midRunStartedAt+30}},
+];
+const midRunWorkspaceResult = getPendingSessionMessage(
+  {{pending_user_message:prompt, pending_started_at:midRunStartedAt}},
+  midRunWorkspaceTranscript
+);
+// A same-text row from an OLDER turn must never be adopted: only the row whose
+// timestamp matches pending_started_at identifies the active turn.
+const staleSameTextTranscript = [
+  {{role:'user', content:prompt, timestamp:midRunStartedAt-600}},
+  {{role:'assistant', content:'old answer', timestamp:midRunStartedAt-500}},
+];
+const staleSameTextResult = getPendingSessionMessage(
+  {{pending_user_message:prompt, pending_started_at:midRunStartedAt}},
+  staleSameTextTranscript
+);
+// Legacy/absent pending_started_at disables the fallback rather than guessing.
+const noStartedAtResult = getPendingSessionMessage(
+  {{pending_user_message:prompt}},
+  midRunTranscript
+);
+
+// Regression (review round 2): two completed identical-text turns whose
+// started_at differ by ~1s. The old 1.5s tolerance adopted the EARLIER row as
+// the "active turn", hiding the second (still-pending) turn and copying its
+// attachments onto the old row. With the precision-only epsilon the second turn
+// is materialized and the first row stays untouched.
+const rapidRepeatTurnOne = {{role:'user', content:prompt, timestamp:100}};
+const rapidRepeatAnswerOne = {{role:'assistant', content:'done', timestamp:101}};
+const rapidRepeatSecondAttachments = [{{name:'second.txt', path:'second.txt', mime:'text/plain'}}];
+const rapidRepeatResult = getPendingSessionMessage(
+  {{pending_user_message:prompt, pending_started_at:101, pending_attachments:rapidRepeatSecondAttachments}},
+  [rapidRepeatTurnOne, rapidRepeatAnswerOne]
+);
+// Exact token identity: the eager-checkpoint row carries _active_turn_token
+// built from the same stream_id + started_at as the session — adopted even
+// though its timestamp (499.9) is NOT within the epsilon of pending_started_at.
+const tokenIdentityTranscript = [
+  {{role:'user', content:'earlier question', timestamp:100}},
+  {{role:'assistant', content:'earlier answer', timestamp:200}},
+  {{role:'user', content:prompt, timestamp:499.9, _active_turn_token:'stream-abc:500'}},
+  {{role:'assistant', content:'partial', timestamp:560}},
+];
+const tokenIdentityResult = getPendingSessionMessage(
+  {{pending_user_message:prompt, pending_started_at:midRunStartedAt, active_stream_id:'stream-abc'}},
+  tokenIdentityTranscript
+);
+
 process.stdout.write(JSON.stringify({{
   historicalSameTextSurvives: !!fromHistoricalSameText && fromHistoricalSameText.content===prompt && fromHistoricalSameText._pending===true,
   historicalWorkspaceSurvives: !!fromHistoricalWorkspace && fromHistoricalWorkspace.content===prompt && fromHistoricalWorkspace._pending===true,
@@ -282,6 +370,13 @@ process.stdout.write(JSON.stringify({{
   compactionBoundaryCurrentTail: compactionCurrentTail&&compactionCurrentTail.role==='user'&&compactionCurrentTail.content===prompt,
   compactionCurrentTailAttachmentsCopied: Array.isArray(currentTailForCompaction.attachments) && currentTailForCompaction.attachments[0].name==='note.txt',
   repeatedCompletedPromptsRemainValid: repeatedCompletedPromptCount===3,
+  midRunReloadDedupe: midRunResult===null,
+  midRunWorkspaceReloadDedupe: midRunWorkspaceResult===null,
+  staleSameTextStillMaterializes: !!staleSameTextResult && staleSameTextResult._pending===true,
+  legacyNoStartedAtStillMaterializes: !!noStartedAtResult && noStartedAtResult._pending===true,
+  rapidRepeatSecondTurnReturned: !!rapidRepeatResult && rapidRepeatResult._pending===true && rapidRepeatResult.content===prompt && Array.isArray(rapidRepeatResult.attachments) && rapidRepeatResult.attachments[0].name==='second.txt',
+  rapidRepeatFirstRowKeptClean: !Array.isArray(rapidRepeatTurnOne.attachments),
+  tokenIdentityDedupe: tokenIdentityResult===null,
   isContextCompactionText: _isContextCompactionText(compactionMarker.content),
   isContextCompactionMessage: _isContextCompactionMessage(compactionMarker),
 }}));
@@ -496,6 +591,16 @@ def test_get_pending_session_message_keeps_deferred_repeat_prompt_by_behavior():
     assert result["compactionBoundaryCurrentTail"] is True
     assert result["compactionCurrentTailAttachmentsCopied"] is True
     assert result["repeatedCompletedPromptsRemainValid"] is True
+    # #6649 follow-up: mid-run reload must not duplicate the active turn's user row
+    assert result["midRunReloadDedupe"] is True
+    assert result["midRunWorkspaceReloadDedupe"] is True
+    assert result["staleSameTextStillMaterializes"] is True
+    assert result["legacyNoStartedAtStillMaterializes"] is True
+    # #6670 review round 2: ~1s-apart identical turns must not be collapsed by
+    # the active-turn fallback (no over-dedup, no attachment mis-attachment).
+    assert result["rapidRepeatSecondTurnReturned"] is True
+    assert result["rapidRepeatFirstRowKeptClean"] is True
+    assert result["tokenIdentityDedupe"] is True
     assert result["isContextCompactionText"] is True
     assert result["isContextCompactionMessage"] is True