From a17370ad4af7af73770fb760e12e3d625eb6efad Mon Sep 17 00:00:00 2001 From: silent-reader-cn Date: Mon, 13 Jul 2026 10:36:18 +0800 Subject: [PATCH] fix(media): harden streamed MEDIA rendering --- static/messages.js | 65 ++++++- static/ui.js | 7 +- tests/test_smd_media_in_stream.py | 214 ++++++++++++++++++++++++ tests/test_svg_audio_video_rendering.py | 17 ++ 4 files changed, 292 insertions(+), 11 deletions(-) diff --git a/static/messages.js b/static/messages.js index 3dbc17d9..97785fbe 100644 --- a/static/messages.js +++ b/static/messages.js @@ -3733,7 +3733,9 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ node.appendChild(body); const baseRenderer=fade?_streamFadeRenderer(body):_safeSmdRenderer(body); const renderer=_smdRendererWithoutUnderscoreEmphasis(baseRenderer); - st={node,parser:window.smd.parser(renderer),writtenText:'',fade}; + const parser=window.smd.parser(renderer); + _smdBindParserIdentity(renderer,parser,body); + st={node,parser,writtenText:'',fade}; _anchorProseSmdCache.set(key,st); // Bound memory across turns: keys embed the stream id, so stale entries // from finished streams age out here. @@ -4055,6 +4057,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ const baseRenderer=fade ? _streamFadeRenderer(el) : _safeSmdRenderer(el); const renderer=_smdRendererWithoutUnderscoreEmphasis(baseRenderer); _smdParser=window.smd.parser(renderer); + _smdBindParserIdentity(renderer,_smdParser,el); } function _smdRendererWithoutUnderscoreEmphasis(renderer){ if(!renderer||!window.smd) return renderer; @@ -4098,6 +4101,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ // Clear the per-parser MEDIA tail buffer — any incomplete MEDIA // prefix the parser was holding is no longer relevant. if(typeof _smdMediaTailClear==='function') _smdMediaTailClear(_smdParser); + if(typeof _smdClearParserIdentity==='function') _smdClearParserIdentity(assistantBody,_smdParser); _smdParser=null; _smdWrittenLen=0; _smdWrittenText=''; @@ -4256,7 +4260,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ const baseAddText=renderer.add_text; const baseSetAttr=renderer.set_attr; const parserFor = (data)=>{ - return (data && data.parser) || (el && el.__smdParser) || (data && data.nodes && data.nodes[data.index]) || __SMD_PARSER_FALLBACK; + return _smdParserKey(data, el); }; renderer.add_text=(data,text)=>{ const parent=data&&data.nodes&&data.nodes[data.index]; @@ -4343,6 +4347,16 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ // "foo.png") is buffered in a per-parser tail buffer and completed // on the next add_text call. const _MEDIA_TAIL_MAX = 4096; // bytes; defensive cap on per-parser buffer + const _SMD_MEDIA_PREFIX = 'MEDIA:'; + function _smdMediaPrefixTail(value){ + const text=String(value||''); + const max=Math.min(_SMD_MEDIA_PREFIX.length,text.length); + for(let len=max;len>0;len-=1){ + const suffix=text.slice(text.length-len); + if(_SMD_MEDIA_PREFIX.startsWith(suffix)) return suffix; + } + return ''; + } function _smdMediaTailSet(tailMap, parser, chunk, parent, baseAddText, data){ if(chunk) tailMap.set(parser, {chunk, parent, baseAddText, data}); else tailMap.delete(parser); @@ -4381,6 +4395,13 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ const combined = lead ? lead + value : value; // Fast path: no MEDIA tokens in the (possibly combined) string. if(!/MEDIA:/.test(combined)){ + const prefixTail=_smdMediaPrefixTail(combined); + if(prefixTail && tails && parser && prefixTail.length < _MEDIA_TAIL_MAX){ + const stable=combined.slice(0, combined.length-prefixTail.length); + if(stable && baseAddText) baseAddText(data, stable); + _smdMediaTailSet(tails, parser, prefixTail, parent, baseAddText, data); + return; + } if(baseAddText) baseAddText(data, combined); return; } @@ -4407,7 +4428,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ last = combined.length; break; } - _smdAppendMediaNode(parent, m[1]); + if(!_smdAppendMediaNode(parent, m[1]) && baseAddText) baseAddText(data, m[0]); last = matchEnd; } // Tail buffer — hold trailing bytes that look like an unterminated @@ -4415,10 +4436,12 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ const rest = combined.slice(last); if(rest){ const tailMatch = /MEDIA:[^\s\)\]]*$/.exec(rest); - if(tailMatch && rest.length < _MEDIA_TAIL_MAX){ - const tailStart = tailMatch.index; + const prefixTail = tailMatch ? '' : _smdMediaPrefixTail(rest); + const tailValue = tailMatch ? tailMatch[0] : prefixTail; + if(tailValue && rest.length < _MEDIA_TAIL_MAX){ + const tailStart = tailMatch ? tailMatch.index : rest.length-prefixTail.length; if(tailStart>0 && baseAddText) baseAddText(data, rest.slice(0, tailStart)); - unmatchedTail = rest.slice(tailStart); + unmatchedTail = tailValue; } else if(baseAddText){ baseAddText(data, rest); } @@ -4445,8 +4468,25 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ const frag=document.createDocumentFragment(); while(host.firstChild) frag.appendChild(host.firstChild); parent.appendChild(frag); + _smdScheduleMediaPostProcess(parent); return true; } + function _smdScheduleMediaPostProcess(root){ + if(!root) return; + if(typeof _postProcessWithAnchorSuppression!=='function' + && typeof postProcessRenderedMessages!=='function' + && typeof _applyMediaPlaybackPreferences!=='function') return; + const run=()=>{ + try{ + if(typeof _postProcessWithAnchorSuppression==='function') _postProcessWithAnchorSuppression(root); + else if(typeof postProcessRenderedMessages==='function') postProcessRenderedMessages(root); + if(typeof _applyMediaPlaybackPreferences==='function') _applyMediaPlaybackPreferences(root); + }catch(_){} + }; + if(typeof requestAnimationFrame==='function') requestAnimationFrame(run); + else if(typeof setTimeout==='function') setTimeout(run,0); + else run(); + } // Per-parser tail buffer keyed by parser instance so concurrent // smd parsers (live prose + anchor-scene rows + tool-card streams) // keep their own pending bytes. Cleared inside _smdEndParser / @@ -4456,6 +4496,17 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ // a string, so WeakMap.set doesn't throw TypeError when all three // parser-identity sources are unavailable (Greptile #3). const __SMD_PARSER_FALLBACK = {}; + function _smdParserKey(data, el){ + return (data && data.parser) || (el && el.__smdParser) || __SMD_PARSER_FALLBACK; + } + function _smdBindParserIdentity(renderer, parser, el){ + if(renderer&&renderer.data) renderer.data.parser=parser; + if(el) el.__smdParser=parser; + } + function _smdClearParserIdentity(el, parser){ + if(!el || (parser && el.__smdParser!==parser)) return; + try{delete el.__smdParser;}catch(_){el.__smdParser=null;} + } function _smdMediaTailClear(parser){ if(_SMD_MEDIA_TAIL && parser) _SMD_MEDIA_TAIL.delete(parser); // Also clear the fallback key if it was ever set @@ -4466,7 +4517,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ const baseSetAttr=renderer.set_attr; const baseAddText=renderer.add_text; const parserFor = (data)=>{ - return (data && data.parser) || (el && el.__smdParser) || (data && data.nodes && data.nodes[data.index]) || __SMD_PARSER_FALLBACK; + return _smdParserKey(data, el); }; renderer.add_text=(data,text)=>{ const parent=data&&data.nodes&&data.nodes[data.index]; diff --git a/static/ui.js b/static/ui.js index 12a3d882..1a19a6f7 100644 --- a/static/ui.js +++ b/static/ui.js @@ -2468,8 +2468,8 @@ function _applyMediaPlaybackRate(media, rate=_getStoredMediaPlaybackRate()){ } function _mediaKindForName(name=''){ const clean=String(name||'').split('?')[0].toLowerCase(); - if(_AUDIO_EXTS.test(clean)) return 'audio'; if(_VIDEO_EXTS.test(clean)) return 'video'; + if(_AUDIO_EXTS.test(clean)) return 'audio'; if(_IMAGE_EXTS.test(clean)) return 'image'; return ''; } @@ -2545,9 +2545,8 @@ function _inlineMediaHtmlForRef(ref, sessionId){ return `${safeName}${dlSvg}`; } if(_SVG_EXTS.test(ref)) return `${esc(typeof t==='function'?t('media_svg_label'):'svg')}`; - if(_AUDIO_EXTS.test(ref)||_VIDEO_EXTS.test(ref)){ - const kind=_AUDIO_EXTS.test(ref)?'audio':'video'; - return _mediaPlayerHtml(kind,apiUrl+'&inline=1',ref.split('/').pop()||ref); + if(localKind==='audio'||localKind==='video'){ + return _mediaPlayerHtml(localKind,apiUrl+'&inline=1',ref.split('/').pop()||ref); } if(_PDF_EXTS.test(ref)){ const fname=esc(ref.split('/').pop()||ref); diff --git a/tests/test_smd_media_in_stream.py b/tests/test_smd_media_in_stream.py index 7a35d37b..10c18591 100644 --- a/tests/test_smd_media_in_stream.py +++ b/tests/test_smd_media_in_stream.py @@ -30,12 +30,141 @@ notes head-on: """ from __future__ import annotations +import json import pathlib +import shutil +import subprocess import unittest REPO_ROOT = pathlib.Path(__file__).parent.parent UI_JS = (REPO_ROOT / "static" / "ui.js").read_text(encoding="utf-8") MESSAGES_JS = (REPO_ROOT / "static" / "messages.js").read_text(encoding="utf-8") +NODE = shutil.which("node") + + +def _extract_js_function(src: str, name: str) -> str: + start = src.index(f"function {name}") + brace = src.index("{", start) + depth = 1 + pos = brace + 1 + while pos < len(src) and depth: + if src[pos] == "{": + depth += 1 + elif src[pos] == "}": + depth -= 1 + pos += 1 + return src[start:pos] + + +def _run_real_smd_media_cases() -> dict: + helpers = "\n".join( + [ + _extract_js_function(MESSAGES_JS, "_smdMediaPrefixTail"), + _extract_js_function(MESSAGES_JS, "_smdMediaTailSet"), + _extract_js_function(MESSAGES_JS, "_smdMediaTailEntryChunk"), + _extract_js_function(MESSAGES_JS, "_smdMediaRefHasReliableBoundary"), + _extract_js_function(MESSAGES_JS, "_smdMediaTailFlush"), + _extract_js_function(MESSAGES_JS, "_smdMediaAwareAddText"), + _extract_js_function(MESSAGES_JS, "_smdAppendMediaNode"), + _extract_js_function(MESSAGES_JS, "_smdScheduleMediaPostProcess"), + _extract_js_function(MESSAGES_JS, "_smdParserKey"), + _extract_js_function(MESSAGES_JS, "_smdBindParserIdentity"), + _extract_js_function(MESSAGES_JS, "_smdMediaTailClear"), + ] + ) + script = ( + "import * as smd from './static/vendor/smd.min.js';\n" + "globalThis.window = { smd };\n" + "globalThis.requestAnimationFrame = cb => cb();\n" + "const _MEDIA_TAIL_MAX = 4096;\n" + "const _SMD_MEDIA_PREFIX = 'MEDIA:';\n" + "const _SMD_MEDIA_TAIL = new WeakMap();\n" + "const __SMD_PARSER_FALLBACK = {};\n" + "let postProcessCalls = 0;\n" + "let playbackCalls = 0;\n" + "function _postProcessWithAnchorSuppression(root){ postProcessCalls += root.querySelectorAll('.pdf-preview-load').length; }\n" + "function _applyMediaPlaybackPreferences(){ playbackCalls += 1; }\n" + "function esc(value){ return String(value ?? '').replace(/[&<>\"']/g, c => ({'&':'&','<':'<','>':'>','\"':'"',\"'\":'''}[c])); }\n" + "function _inlineMediaHtmlForRef(ref){\n" + " const raw = String(ref || '');\n" + " if(/\\.pdf$/i.test(raw)) return `
PDF
`;\n" + " return ``;\n" + "}\n" + "class FakeNode{\n" + " constructor(type, tag='', text=''){ this.nodeType=type; this.tagName=tag; this.children=[]; this.parentNode=null; this.attributes={}; this.data=text; }\n" + " get childNodes(){ return this.children; }\n" + " get firstChild(){ return this.children[0] || null; }\n" + " appendChild(child){\n" + " if(child.nodeType===11){ while(child.firstChild) this.appendChild(child.firstChild); return child; }\n" + " if(child.parentNode){ const old=child.parentNode.children.indexOf(child); if(old>=0) child.parentNode.children.splice(old,1); }\n" + " child.parentNode=this; this.children.push(child); return child;\n" + " }\n" + " setAttribute(name,value){ this.attributes[name]=String(value); }\n" + " getAttribute(name){ return this.attributes[name] ?? null; }\n" + " querySelectorAll(selector){\n" + " const cls=selector.startsWith('.') ? selector.slice(1) : '';\n" + " const out=[];\n" + " const visit=node=>{ for(const child of node.children){ const classes=(child.attributes.class||'').split(/\\s+/); if(cls&&classes.includes(cls)) out.push(child); visit(child); } };\n" + " visit(this); return out;\n" + " }\n" + " get textContent(){ return this.nodeType===3 ? this.data : this.children.map(c=>c.textContent).join(''); }\n" + " get outerHTML(){\n" + " if(this.nodeType===3) return esc(this.data);\n" + " if(this.nodeType===11) return this.children.map(c=>c.outerHTML).join('');\n" + " const attrs=Object.entries(this.attributes).map(([k,v])=>` ${k}=\"${esc(v)}\"`).join('');\n" + " return `<${this.tagName}${attrs}>${this.children.map(c=>c.outerHTML).join('')}`;\n" + " }\n" + "}\n" + "globalThis.document = {\n" + " createElement: tag => new FakeNode(1, tag),\n" + " createTextNode: text => new FakeNode(3, '#text', String(text)),\n" + " createDocumentFragment: () => new FakeNode(11, '#fragment'),\n" + "};\n" + "globalThis.DOMParser = class { parseFromString(html){\n" + " const host=document.createElement('div');\n" + " const cls=html.includes('pdf-preview-load') ? 'pdf-preview-load' : 'media-node';\n" + " const node=document.createElement(html.includes('pdf-preview-load') ? 'div' : 'span');\n" + " node.setAttribute('class', cls);\n" + " const ref=(html.match(/data-ref=\"([^\"]*)\"/)||html.match(/data-path=\"([^\"]*)\"/)||[])[1]||'';\n" + " if(ref) node.setAttribute(cls==='pdf-preview-load' ? 'data-path' : 'data-ref', ref);\n" + " host.appendChild(node);\n" + " return { body: { firstChild: host } };\n" + "} };\n" + f"{helpers}\n" + "function renderChunks(chunks){\n" + " postProcessCalls = 0; playbackCalls = 0;\n" + " const root=document.createElement('div');\n" + " const renderer=smd.default_renderer(root);\n" + " const baseAddText=renderer.add_text;\n" + " renderer.add_text=(data,text)=>{\n" + " const parent=data&&data.nodes&&data.nodes[data.index];\n" + " _smdMediaAwareAddText(baseAddText, parent, data, text, _SMD_MEDIA_TAIL, _smdParserKey(data, root));\n" + " };\n" + " const parser=smd.parser(renderer);\n" + " _smdBindParserIdentity(renderer, parser, root);\n" + " for(const chunk of chunks) smd.parser_write(parser, chunk);\n" + " smd.parser_end(parser);\n" + " _smdMediaTailFlush(parser);\n" + " _smdMediaTailClear(parser);\n" + " return { html: root.outerHTML, text: root.textContent, postProcessCalls, playbackCalls };\n" + "}\n" + "const marker='MEDIA:';\n" + "const prefixSplits={};\n" + "for(let i=1;i= 1, "URL video handler should test extension on src" +def test_webm_prefers_video_when_audio_and_video_regexes_overlap(): + """Verify .webm is not shadowed by the audio regex.""" + with open('static/ui.js', encoding="utf-8") as f: + src = f.read() + kind_start = src.find("function _mediaKindForName") + kind_body = src[kind_start:kind_start + 400] + assert "_VIDEO_EXTS.test(clean)" in kind_body + assert "_AUDIO_EXTS.test(clean)" in kind_body + assert kind_body.index("_VIDEO_EXTS.test(clean)") < kind_body.index("_AUDIO_EXTS.test(clean)"), \ + "_mediaKindForName must check video before audio so .webm renders as video" + inline_start = src.find("function _inlineMediaHtmlForRef") + inline_body = src[inline_start:inline_start + 4500] + assert "const kind=_AUDIO_EXTS.test(ref)?'audio':'video';" not in inline_body, \ + "Local MEDIA rendering must reuse _mediaKindForName instead of reintroducing audio-first .webm ordering" + assert "if(localKind==='audio'||localKind==='video')" in inline_body + + def test_attachment_svg_audio_video(): """Verify file attachments for SVG/audio/video get inline previews.""" with open('static/ui.js', encoding="utf-8") as f: