From 3063c4930fc301c75a6d36fc361bd6f453bed8bf Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Mon, 8 Jun 2026 18:16:52 -0700 Subject: [PATCH] =?UTF-8?q?Release=20v0.51.336=20=E2=80=94=20Release=20KZ?= =?UTF-8?q?=20(fix=20per-token=20inline-thinking=20perf=20regression,=20#3?= =?UTF-8?q?633=20follow-up)=20(#3854)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(streaming): make per-token inline-thinking extraction linear (#3633 follow-up) Codex post-merge perf catch on #3633: _parseStreamState() and syncInflightAssistantMessage() call _extractInlineThinkingFromContent on the FULL accumulated assistantText on EVERY streamed token. The #3633 rewrite made that a full char-by-char walk, so cost was O(n^2) over a stream — a Node harness measured ~88s (no-tag) / ~103s (leading block) for 2000x100-char tokens, which would freeze the main thread on long reasoning-model responses. Two fixes (Python api/streaming.py + JS static/messages.js twin, line-for-line parity): 1. Fast path: if the text contains no complete thinking opener AND (when streaming) its tail is not a prefix of an opener, return unchanged without the char walk — two cheap substring scans. Handles the common no-tag case. 2. Bulk-skip plain trailing content: track the next complete opener via str.find/indexOf (_next_inline_thinking_opener / _nextThinkingOpener); once no opener remains ahead, append the remainder and stop instead of walking it (streaming still suppresses a trailing partial-opener prefix). Handles the leading-block-then-long-answer case. Result: ~88s/103s → ~0.5s/0.9s (Python), ~0.18s/0.21s (JS). All behavioral cases (persist + streaming, code-awareness, position-aware unclosed, leading whitespace) verified unchanged in both twins. Added a per-token streaming perf regression test and wired _nextThinkingOpener into the node driver harness. Co-authored-by: rodboev * fix(streaming): perf bulk-skip must respect code context for partial-opener tails Codex catch on the perf fix: the no-complete-opener bulk-skip suppressed a trailing partial opener (e.g. ' * docs(changelog): v0.51.336 KZ — inline-thinking streaming perf fix --------- Co-authored-by: rodboev Co-authored-by: Hermes Agent --- CHANGELOG.md | 5 ++ api/streaming.py | 67 +++++++++++++++++++ static/messages.js | 63 +++++++++++++++++ .../test_issue3455_think_block_extraction.py | 6 +- ...st_issue3599_inline_thinking_extraction.py | 43 +++++++++++- 5 files changed, 182 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99b1bceb..8c0b290a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ## [Unreleased] +## [v0.51.336] — 2026-06-08 — Release KZ (fix inline-thinking streaming perf regression) + +### Fixed +- **Long streaming responses no longer slow down as they grow.** A follow-up to v0.51.335: the inline-thinking extractor runs on the full in-progress message on every streamed token, and the v0.51.335 rewrite made that scan re-walk the whole buffer each time (quadratic over a long response — most noticeable on reasoning-model replies with a `` block). It now fast-paths content with no thinking tags and skips already-settled trailing text, keeping per-token work flat. No behavior change — verified identical to the prior release across streaming, reload, persistence, and code-block cases. (#3633 follow-up) + ## [v0.51.335] — 2026-06-08 — Release KY (normalize inline thinking extraction) ### Fixed diff --git a/api/streaming.py b/api/streaming.py index 024e3c2f..76a39960 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -1517,6 +1517,30 @@ def _inline_thinking_fence_marker_at(text, index): return '' +def _next_inline_thinking_opener(text, start): + """Index of the earliest complete thinking opener at/after `start`, or -1. + Cheap str.find per opener — lets the scanner bulk-skip plain trailing content + instead of walking it char-by-char (#3633 Codex per-token perf catch).""" + best = -1 + for open_tag, _close in _INLINE_THINKING_TAG_PAIRS: + i = text.find(open_tag, start) + if i != -1 and (best == -1 or i < best): + best = i + return best + + +def _text_tail_is_partial_opener(text): + """True when the END of `text` is a non-empty proper prefix of some thinking + opener (e.g. ````). Used to decide whether a streaming + tail might be a forming block worth code-aware handling.""" + for open_tag, _close in _INLINE_THINKING_TAG_PAIRS: + m = min(len(open_tag) - 1, len(text)) + for n in range(m, 0, -1): + if open_tag.startswith(text[-n:]): + return True + return False + + def _line_is_indented_code(text, line_start): """True when the line beginning at `line_start` is a markdown indented code block line (>=4 leading spaces or a leading tab, and not blank). `line_start` @@ -1570,6 +1594,26 @@ def _extract_inline_thinking_from_content(raw_content, existing_reasoning='', *, text = '' if raw_content is None else str(raw_content) if not text: return text, str(existing_reasoning or '').strip() + # Fast path (#3633 Codex perf catch — _parseStreamState / syncInflight call + # this on the FULL accumulator on every streamed token, so the common no-tag + # case must not do the O(length) char walk per call). If the text contains no + # complete thinking opener AND — when streaming — its tail is not a prefix of + # any opener (a partial opener mid-stream), there is nothing to extract: + # return the text unchanged. Two cheap substring scans instead of a full walk. + if not any(open_tag in text for open_tag, _close in _INLINE_THINKING_TAG_PAIRS): + tail_is_partial_opener = False + if streaming: + for open_tag, _close in _INLINE_THINKING_TAG_PAIRS: + # Does the END of text look like the START of an opener? + max_prefix = min(len(open_tag) - 1, len(text)) + for n in range(max_prefix, 0, -1): + if open_tag.startswith(text[-n:]): + tail_is_partial_opener = True + break + if tail_is_partial_opener: + break + if not tail_is_partial_opener: + return text, str(existing_reasoning or '').strip() visible = [] extracted = [] cursor = 0 @@ -1589,7 +1633,30 @@ def _extract_inline_thinking_from_content(raw_content, existing_reasoning='', *, # indented code / whitespace and has NO leading thinking wrapper keeps its # leading whitespace — #3633 Codex catch). leading_removed = False + # Index of the next opener at/after `index` (recomputed only when we pass it). + # When no opener remains ahead, the rest of the text is plain and can be + # appended in one slice — this keeps a stream that DID contain a leading + # thinking block from re-walking the whole growing answer tail every token + # (#3633 Codex perf catch: the per-token full walk was O(n^2) over a stream). + next_opener = _next_inline_thinking_opener(text, 0) while index < length: + if next_opener == -1 or index > next_opener: + next_opener = _next_inline_thinking_opener(text, index) + if next_opener == -1: + # No further COMPLETE opener ahead. The remaining tail is plain + # visible content and can be appended in one slice — EXCEPT during + # streaming when the tail is a prefix of an opener (e.g. "... 0 and text[index - 1] == '\n': line_is_indented_code = _line_is_indented_code(text, index) diff --git a/static/messages.js b/static/messages.js index e8fb707b..79ac7af8 100644 --- a/static/messages.js +++ b/static/messages.js @@ -126,6 +126,29 @@ function _thinkingFenceMarkerAt(text, index){ return ''; } +function _nextThinkingOpener(text, start){ + // Index of the earliest complete thinking opener at/after `start`, or -1. + // Cheap indexOf per opener — lets the scanner bulk-skip plain trailing content + // instead of walking it char-by-char (#3633 Codex per-token perf catch). + let best=-1; + for(const p of _thinkPairs){ + const i=text.indexOf(p.open,start); + if(i!==-1&&(best===-1||i"). Decides whether a streaming tail might be a + // forming block worth code-aware handling. + for(const p of _thinkPairs){ + const m=Math.min(p.open.length-1,text.length); + for(let n=m;n>0;n--){ if(p.open.startsWith(text.slice(text.length-n))) return true; } + } + return false; +} + function _lineIsIndentedCode(text, lineStart){ // True when the line beginning at lineStart is a markdown indented code block // line (>=4 leading spaces or a leading tab, and not blank). lineStart must be @@ -166,6 +189,27 @@ function _extractInlineThinkingFromContent(rawContent, existingReasoning, option const reasoning=String(existingReasoning||'').trim(); return {reasoning,content:text,thinkingText:reasoning,displayText:text,inThinking:false}; } + // Fast path (#3633 Codex perf catch — _parseStreamState / syncInflightAssistantMessage + // call this on the FULL accumulator on every streamed token, so the common no-tag + // case must not do the O(length) char walk per call). If no complete opener is + // present AND — when streaming — the tail is not a prefix of an opener, there is + // nothing to extract: return the text unchanged (two cheap substring scans). + if(!_thinkPairs.some(p=>text.indexOf(p.open)!==-1)){ + let tailIsPartialOpener=false; + if(streaming){ + for(const p of _thinkPairs){ + const maxPrefix=Math.min(p.open.length-1,text.length); + for(let n=maxPrefix;n>0;n--){ + if(p.open.startsWith(text.slice(text.length-n))){tailIsPartialOpener=true;break;} + } + if(tailIsPartialOpener) break; + } + } + if(!tailIsPartialOpener){ + const reasoning=String(existingReasoning||'').trim(); + return {reasoning,content:text,thinkingText:reasoning,displayText:text,inThinking:false}; + } + } const visible=[]; const extracted=[]; let cursor=0; @@ -183,7 +227,26 @@ function _extractInlineThinkingFromContent(rawContent, existingReasoning, option // and has no leading thinking wrapper keeps its leading whitespace (#3633 // Codex catch). let leadingRemoved=false; + // Index of the next complete opener at/after `index` — lets the scanner bulk-skip + // plain trailing content instead of walking it char-by-char every streamed token + // (#3633 Codex per-token perf catch). + let nextOpener=_nextThinkingOpener(text,0); while(indexnextOpener) nextOpener=_nextThinkingOpener(text,index); + if(nextOpener===-1){ + // No further COMPLETE opener ahead — remaining tail is plain and is + // appended in one slice, EXCEPT during streaming when the tail is a prefix + // of an opener ("...0&&text[index-1]==='\n') lineIsIndentedCode=_lineIsIndentedCode(text,index); const marker=_thinkingFenceMarkerAt(text,index); diff --git a/tests/test_issue3455_think_block_extraction.py b/tests/test_issue3455_think_block_extraction.py index 827031fe..efac21dd 100644 --- a/tests/test_issue3455_think_block_extraction.py +++ b/tests/test_issue3455_think_block_extraction.py @@ -53,6 +53,8 @@ _DRIVER = """ %s %s %s +%s +%s const args = JSON.parse(process.argv[2]); process.stdout.write(JSON.stringify(_splitThinkFromContent(args.raw, args.existing || ''))); """ @@ -64,12 +66,14 @@ def driver(tmp_path_factory): pytest.skip("node not available") pairs = _extract_block(MESSAGES_JS, "const _thinkPairs=") fence = _extract_block(MESSAGES_JS, "function _thinkingFenceMarkerAt(") + nextopener = _extract_block(MESSAGES_JS, "function _nextThinkingOpener(") + tailpartial = _extract_block(MESSAGES_JS, "function _textTailIsPartialOpener(") indented = _extract_block(MESSAGES_JS, "function _lineIsIndentedCode(") merge = _extract_block(MESSAGES_JS, "function _mergeInlineThinkingReasoning(") extract = _extract_block(MESSAGES_JS, "function _extractInlineThinkingFromContent(") fn = _extract_block(MESSAGES_JS, "function _splitThinkFromContent(") p = tmp_path_factory.mktemp("think3455") / "driver.js" - p.write_text(_DRIVER % (pairs, fence, indented, merge, extract, fn), encoding="utf-8") + p.write_text(_DRIVER % (pairs, fence, nextopener, tailpartial, indented, merge, extract, fn), encoding="utf-8") return str(p) diff --git a/tests/test_issue3599_inline_thinking_extraction.py b/tests/test_issue3599_inline_thinking_extraction.py index 59f61b65..cc6a8c78 100644 --- a/tests/test_issue3599_inline_thinking_extraction.py +++ b/tests/test_issue3599_inline_thinking_extraction.py @@ -1,6 +1,9 @@ from pathlib import Path -from api.streaming import _split_thinking_from_content +from api.streaming import ( + _extract_inline_thinking_from_content, + _split_thinking_from_content, +) REPO = Path(__file__).resolve().parents[1] @@ -147,6 +150,44 @@ def test_extraction_is_linear_on_long_no_newline_content(): assert elapsed < 1.0, f"extraction took {elapsed:.2f}s — likely quadratic" +def test_per_token_streaming_scan_is_not_quadratic(): + """#3633 Codex CORE perf catch: _parseStreamState / syncInflightAssistantMessage + call the extractor on the FULL accumulator on every streamed token. Simulate a + long stream (both no-tag and leading-thinking-block cases) and assert the + cumulative cost stays bounded — a per-token full walk over the growing buffer + was O(n^2) (~88s no-tag / ~103s with-tag for 2000x100-char tokens).""" + import time + + def sim(n_tokens, tok_len, lead_tag): + acc = "short reasoning" if lead_tag else "" + start = time.time() + for _ in range(n_tokens): + acc += "x" * tok_len + # mimic the two per-token extractor calls (_streamDisplay + _parseStreamState) + _extract_inline_thinking_from_content(acc, "", streaming=True) + _extract_inline_thinking_from_content(acc, "", streaming=True) + return time.time() - start + + no_tag = sim(2000, 100, False) + with_tag = sim(2000, 100, True) + assert no_tag < 3.0, f"no-tag per-token stream took {no_tag:.1f}s — quadratic" + assert with_tag < 3.0, f"with-tag per-token stream took {with_tag:.1f}s — quadratic" + + +def test_streaming_partial_opener_tail_respects_code_context(): + """#3633 perf-fix follow-up (Codex): the bulk-skip fast path must not suppress + a trailing partial opener that sits inside code — only a partial opener in + PLAIN text is a forming block. Mirrors master parity for inline-backtick, + fenced, and indented code; a plain partial tail is still suppressed.""" + ext = _extract_inline_thinking_from_content + # Inside code → the partial opener tail stays visible. + assert ext("answer `