jinghui/hermes-webui
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: honor markdown fence lengths

Browse Source
This commit is contained in:
Michael Lam committed 2026-05-05 08:36:17 -07:00
1 parent 1cde702d47
commit f76921d322
8 files changed
+178 -52
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

static/ui.js
+31 -14
View File
@@ -50,6 +50,15 @@ function _setCompressionSessionLock(sid){
window._compressionLockSid=sid||null;
}
const esc=s=>String(s??'').replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
function _matchBacktickFenceLine(line){
const m=String(line||'').match(/^[ ]{0,3}(`{3,})([^`]*)$/);
if(!m) return null;
return {fence:m[1],len:m[1].length,info:(m[2]||'').trim()};
}
function _isBacktickFenceClose(line,minLen){
const m=String(line||'').match(/^[ ]{0,3}(`{3,})[ \t]*$/);
return !!(m&&m[1].length>=minLen);
}
/**
* Render fenced code blocks inside user messages.
* Extracts ```…``` fences, replaces them with placeholders,
@@ -62,9 +71,12 @@ function _renderUserFencedBlocks(text){
const stash=[];
let s=String(text||'');
// Extract fenced code blocks → stash, replace with null-token placeholder
// CommonMark line-anchored fence (fixes #1438): inner ``` inside content no longer truncates the block.
s=s.replace(/(^|\n)[ ]{0,3}```([a-zA-Z0-9_+-]*)\n(?:([\s\S]*?)\n)?[ ]{0,3}```(?=\n|$)/g,(_,lead,lang,code)=>{
lang=(lang||'').trim().toLowerCase();
// CommonMark §4.5 line-anchored fence: the closing run must use at least
// as many backticks as the opener, so inner triple-backtick fences remain content.
s=s.replace(/(^|\n)[ ]{0,3}(`{3,})([^\n`]*)\n(?:([\s\S]*?)\n)?[ ]{0,3}\2`*[ \t]*(?=\n|$)/g,(_,lead,_fence,info,code)=>{
const langInfo=(info||'').trim();
const langMatch=langInfo.match(/^(\w[\w+-]*)$/);
let lang=langMatch?(langMatch[1]||'').trim().toLowerCase():'';
code=code||'';
// Remove one trailing newline if present (the fence consumes its own)
if(code.endsWith('\n')) code=code.slice(0,-1);
@@ -1736,7 +1748,8 @@ function renderMd(raw){
s=(function _applyBlockquotes(input){
const lines=input.split('\n');
const out=[];
let inFence=false; // inside a non-blockquote ```...``` fence
let inFence=false; // inside a non-blockquote backtick fence
let fenceLen=0;
let bqStart=-1;
const flush=(end)=>{
if(bqStart<0) return;
@@ -1759,13 +1772,15 @@ function renderMd(raw){
const line=lines[i];
if(inFence){
out.push(line);
if(/^```/.test(line)) inFence=false;
if(_isBacktickFenceClose(line,fenceLen)){inFence=false;fenceLen=0;}
continue;
}
if(/^```/.test(line)){
const fenceOpen=_matchBacktickFenceLine(line);
if(fenceOpen){
flush(i);
out.push(line);
inFence=true;
fenceLen=fenceOpen.len;
continue;
}
if(/^>/.test(line)){
@@ -1809,14 +1824,16 @@ function renderMd(raw){
const _preBlock_stash=[];
const fence_stash=[];
// CommonMark §4.5: opening fence must start a line (with up to 3 spaces of indent)
// and closing fence must also start a line. Without line anchoring, a literal ``` inside
// a code block (e.g. a regex pattern with ``` in a lookbehind, a script that documents
// fences) terminates the outer block at the wrong place, leaking content into the
// markdown stream where bold/italic/inline-code passes corrupt it. Fixes #1438.
s=s.replace(/(^|\n)[ ]{0,3}```(?:([\s\S]*?)\n)?[ ]{0,3}```(?=\n|$)/g,(_,lead,raw)=>{
const m=raw.match(/^(\w[\w+-]*)\n?([\s\S]*)$/);
const lang=m?(m[1]||'').trim().toLowerCase():'';
const code=m?m[2]:raw.replace(/^\n?/,'');
// and closing fence must start a line with the same backtick char and at least
// as many backticks as the opener. Without line/fence-length anchoring, a literal
// ``` inside a code block (e.g. a nested markdown example) terminates the outer
// block at the wrong place, leaking content into the markdown stream where
// bold/italic/inline-code passes corrupt it. Fixes #1438 and #1696.
s=s.replace(/(^|\n)[ ]{0,3}(`{3,})([^\n`]*)\n(?:([\s\S]*?)\n)?[ ]{0,3}\2`*[ \t]*(?=\n|$)/g,(_,lead,_fence,info,code)=>{
const langInfo=(info||'').trim();
const langMatch=langInfo.match(/^(\w[\w+-]*)$/);
const lang=langMatch?(langMatch[1]||'').trim().toLowerCase():'';
code=code||'';
const codeLines=code.split('\n');
const firstCodeLine=codeLines.find(line=>line.trim())||'';
const firstMermaidLine=codeLines.map(line=>line.trim()).find(line=>line&&!line.startsWith('%%'))||'';
tests/test_1325_user_fenced_code.py
+33 -16
View File
@@ -7,23 +7,30 @@ UI_JS = os.path.join(os.path.dirname(__file__), '..', 'static', 'ui.js')
def _extract_js_functions():
"""Extract esc and _renderUserFencedBlocks from ui.js by line numbers."""
lines = open(UI_JS).read().split('\n')
# esc is on line 52 (0-indexed: 51)
esc_def = lines[51]
# _renderUserFencedBlocks starts at line 61 (0-indexed: 60)
# Find the end by matching closing brace at column 0
fn_lines = []
i = 60 # 0-indexed
depth = 0
while i < len(lines):
fn_lines.append(lines[i])
depth += lines[i].count('{') - lines[i].count('}')
if depth <= 0:
break
"""Extract esc, fence helpers, and _renderUserFencedBlocks from ui.js."""
src = open(UI_JS).read()
def extract_function(name):
start = src.find(f"function {name}(")
if start < 0:
raise AssertionError(f"{name} not found in ui.js")
i = src.find("{", start)
depth = 1
i += 1
fn_def = '\n'.join(fn_lines)
return esc_def, fn_def
while i < len(src) and depth:
if src[i] == "{":
depth += 1
elif src[i] == "}":
depth -= 1
i += 1
return src[start:i]
esc_line = next(line for line in src.split("\n") if line.startswith("const esc="))
helper_defs = "\n".join(
extract_function(name)
for name in ("_matchBacktickFenceLine", "_isBacktickFenceClose", "_renderUserFencedBlocks")
)
return esc_line, helper_defs
def _run_user_render(text_input):
@@ -116,6 +123,16 @@ class TestUserFencedBlocks:
assert '<a ' not in out
assert 'https://example.com' in out
def test_four_backtick_outer_fence_preserves_inner_triple_fence(self):
"""User-message code fences should follow CommonMark fence-length matching too."""
out = _run_user_render("````md\n```inner\nfoo\n```\n````")
assert out.count("<pre>") == 1
assert out.count("</pre>") == 1
assert '<div class="pre-header">md</div>' in out
assert "```inner" in out
assert "foo" in out
assert "<br>````" not in out
def test_inline_backticks_not_touched(self):
"""Inline backticks (single backtick, not fenced block) should remain escaped as text."""
out = _run_user_render("use `var x = 1` here")
tests/test_issue1154_fenced_code_leak.py
+2
View File
@@ -43,6 +43,8 @@ function extractFunc(name) {
}
return src.slice(start, i);
}
eval(extractFunc('_matchBacktickFenceLine'));
eval(extractFunc('_isBacktickFenceClose'));
eval(extractFunc('renderMd'));
let buf = '';
tests/test_issue1438_fence_anchoring.py
+11 -17
View File
@@ -199,23 +199,15 @@ def test_inline_code_after_fence():
def test_renderMd_fence_regex_is_line_anchored():
"""The fence regex in renderMd must include `(^|\\n)` opener and `(?=\\n|$)` closer.
Pattern: (^|\\n)[ ]{0,3}```(?:([\\s\\S]*?)\\n)?[ ]{0,3}```(?=\\n|$)
The `(?:...\\n)?` makes the body optional so empty fences (```\\n```) still match.
"""
assert re.search(
r"s=s\.replace\(/\(\^\|\\n\)\[ \]\{0,3\}```\(\?:\(\[\\s\\S\]\*\?\)\\n\)\?\[ \]\{0,3\}```\(\?=\\n\|\$\)/g",
UI_JS,
), "renderMd fence regex is not line-anchored — regression of #1438"
"""The fence regex in renderMd must keep line anchoring and fence-length matching."""
pattern = r"s=s.replace(/(^|\n)[ ]{0,3}(`{3,})([^\n`]*)\n(?:([\s\S]*?)\n)?[ ]{0,3}\2`*[ \t]*(?=\n|$)/g"
assert pattern in UI_JS, "renderMd fence regex lost line anchoring or #1696 fence-length matching"
def test_renderUserFencedBlocks_fence_regex_is_line_anchored():
"""The fence regex in _renderUserFencedBlocks must also be line-anchored."""
assert re.search(
r"s=s\.replace\(/\(\^\|\\n\)\[ \]\{0,3\}```\(\[a-zA-Z0-9_\+\-\]\*\)\\n\(\?:\(\[\\s\\S\]\*\?\)\\n\)\?\[ \]\{0,3\}```\(\?=\\n\|\$\)/g",
UI_JS,
), "_renderUserFencedBlocks fence regex is not line-anchored — regression of #1438"
pattern = r"s=s.replace(/(^|\n)[ ]{0,3}(`{3,})([^\n`]*)\n(?:([\s\S]*?)\n)?[ ]{0,3}\2`*[ \t]*(?=\n|$)/g"
assert UI_JS.count(pattern) >= 2, "render/user fence regexes lost line anchoring or #1696 fence-length matching"
def test_stripForTTS_fence_regex_is_line_anchored():
@@ -274,8 +266,10 @@ def test_diff_fence_with_inner_backticks_in_content():
# Pattern explanation: ui.js source contains literal backslash-n in regex literals
# (ONE backslash + 'n'). In a Python raw string, r"\\n" compiles to a regex pattern
# matching ONE literal backslash followed by 'n'.
matches = re.findall(r"```\(\?=\\n\|\$\)", UI_JS)
assert len(matches) >= 3, (
f"all 3 fence sites (renderMd, _renderUserFencedBlocks, _stripForTTS) "
f"must have line-anchored close fence; found {len(matches)} occurrences"
new_matches = UI_JS.count(r"[ ]{0,3}\2`*[ \t]*(?=\n|$)")
old_tts_matches = re.findall(r"```\(\?=\\n\|\$\)", UI_JS)
assert new_matches >= 2 and len(old_tts_matches) >= 1, (
f"renderMd/_renderUserFencedBlocks must have fence-length-aware line-anchored "
f"closers and _stripForTTS must keep a line-anchored closer; found "
f"new={new_matches}, tts={len(old_tts_matches)}"
)
tests/test_issue1446_glued_heading_lift.py
+2
View File
@@ -208,6 +208,8 @@ function extractFunc(name) {
}
return src.slice(start, i);
}
eval(extractFunc('_matchBacktickFenceLine'));
eval(extractFunc('_isBacktickFenceClose'));
eval(extractFunc('renderMd'));
let buf = '';
tests/test_issue1618_yaml_json_diff_newline_preserve.py
+2
View File
@@ -145,6 +145,8 @@ function extractFunc(name) {
}
return src.slice(start, i);
}
eval(extractFunc('_matchBacktickFenceLine'));
eval(extractFunc('_isBacktickFenceClose'));
eval(extractFunc('renderMd'));
let buf = '';
tests/test_renderer_js_behaviour.py
+45
View File
@@ -54,6 +54,8 @@ function extractFunc(name) {
}
return src.slice(start, i);
}
eval(extractFunc('_matchBacktickFenceLine'));
eval(extractFunc('_isBacktickFenceClose'));
eval(extractFunc('renderMd'));
let buf = '';
@@ -285,6 +287,49 @@ class TestBugFencedCodeInBlockquote:
assert "x = 1" in out
class TestFencedCodeFenceLength:
"""CommonMark §4.5 requires the closer to be at least as long as the opener."""
def test_five_backtick_outer_fence_preserves_inner_triple_fence(self, driver_path):
src = (
"- optionally also support fenced code blocks\n\n"
"`````md\n"
"`md\n"
"```novelcrafter\n"
"{#if novel.hasSeries}\n"
"...\n"
"{#endif}\n"
"```\n"
"`````\n\n"
"That is much more correct than pretending"
)
out = _render(driver_path, src)
assert out.count("<pre>") == 1
assert out.count("</pre>") == 1
assert '<div class="pre-header">md</div>' in out
assert "```novelcrafter" in out
assert "{#if novel.hasSeries}" in out
assert "That is much more correct than pretending" in out
assert "<p>`````" not in out
assert "<br>`````" not in out
def test_four_backtick_outer_fence_preserves_inner_triple_fence(self, driver_path):
out = _render(driver_path, "````md\n```inner\nfoo\n```\n````\n")
assert out.count("<pre>") == 1
assert out.count("</pre>") == 1
assert '<div class="pre-header">md</div>' in out
assert "```inner" in out
assert "foo" in out
assert "<p>````" not in out
def test_three_backtick_fence_still_renders_language_class(self, driver_path):
out = _render(driver_path, "```js\nconsole.log('ok')\n```")
assert out.count("<pre>") == 1
assert '<div class="pre-header">js</div>' in out
assert 'class="language-js"' in out
assert "console.log(&#39;ok&#39;)" in out
class TestBugBlankContinuationInBlockquote:
"""Bug 2: blank > lines between paragraphs fragmented the blockquote into
separate elements with literal > characters between them."""
tests/test_sprint16.py
+52 -5
View File
@@ -61,8 +61,8 @@ def render_md(raw):
fence_stash.append(m.group())
return "\x00F" + str(len(fence_stash) - 1) + "\x00"
# Fence regex line-anchored to match JS fix for #1438 (allows empty fence)
s = re.sub(r"(?:^|\n)[ ]{0,3}```(?:[\s\S]*?\n)?[ ]{0,3}```(?=\n|$)|`[^`\n]+`", stash, s)
# Fence regex line-anchored to match JS fix for #1438 and fence-length fix for #1696
s = re.sub(r"(?:^|\n)[ ]{0,3}(`{3,})[^\n`]*\n(?:[\s\S]*?\n)?[ ]{0,3}\1`*(?=\n|$)|`[^`\n]+`", stash, s)
s = re.sub(r"<strong>([\s\S]*?)</strong>", lambda m: "**" + m.group(1) + "**", s, flags=re.I)
s = re.sub(r"<b>([\s\S]*?)</b>", lambda m: "**" + m.group(1) + "**", s, flags=re.I)
s = re.sub(r"<em>([\s\S]*?)</em>", lambda m: "*" + m.group(1) + "*", s, flags=re.I)
@@ -77,11 +77,12 @@ def render_md(raw):
# Fenced code blocks
def fenced(m):
lang, code = m.group(1), (m.group(2) or "").rstrip("\n")
info, code = (m.group(2) or "").strip(), (m.group(3) or "").rstrip("\n")
lang = info.lower() if re.match(r"^\w[\w+-]*$", info) else ""
h = f'<div class="pre-header">{esc(lang)}</div>' if lang else ""
return h + "<pre><code>" + esc(code) + "</code></pre>"
# Fenced code blocks (line-anchored, fixes #1438; allows empty fence)
s = re.sub(r"(?:^|\n)[ ]{0,3}```([\w+-]*)\n(?:([\s\S]*?)\n)?[ ]{0,3}```(?=\n|$)", fenced, s)
# Fenced code blocks (line-anchored, fixes #1438; fence-length matching fixes #1696)
s = re.sub(r"(?:^|\n)[ ]{0,3}(`{3,})([^\n`]*)\n(?:([\s\S]*?)\n)?[ ]{0,3}\1`*(?=\n|$)", fenced, s)
s = re.sub(r"`([^`\n]+)`", lambda m: "<code>" + esc(m.group(1)) + "</code>", s)
# Inline formatting (top-level, outside list items)
@@ -358,6 +359,52 @@ def test_render_md_fenced_code_protects_html(cleanup_test_sessions):
"Fenced code content was lost after stash/restore"
def test_render_md_fenced_code_with_five_backtick_outer_preserves_inner_triples(cleanup_test_sessions):
"""CommonMark §4.5: a 5-backtick fence must not close at an inner triple fence."""
src = (
"- optionally also support fenced code blocks\n\n"
"`````md\n"
"`md\n"
"```novelcrafter\n"
"{#if novel.hasSeries}\n"
"...\n"
"{#endif}\n"
"```\n"
"`````\n\n"
"That is much more correct than pretending"
)
out = render_md(src)
assert out.count("<pre>") == 1
assert out.count("</pre>") == 1
assert '<div class="pre-header">md</div>' in out
assert "```novelcrafter" in out
assert "{#if novel.hasSeries}" in out
assert "That is much more correct than pretending" in out
assert "<p>`````" not in out
assert "<br>`````" not in out
def test_render_md_fenced_code_with_four_backtick_outer_preserves_inner_triples(cleanup_test_sessions):
"""A 4-backtick outer fence should also require a 4+ backtick closer."""
src = "````md\n```inner\nfoo\n```\n````\n"
out = render_md(src)
assert out.count("<pre>") == 1
assert out.count("</pre>") == 1
assert '<div class="pre-header">md</div>' in out
assert "```inner" in out
assert "foo" in out
assert "<p>````" not in out
def test_render_md_fenced_code_three_backtick_path_still_renders_language(cleanup_test_sessions):
"""The common 3-backtick path must keep rendering a single language-tagged block."""
src = "```js\nconsole.log('ok')\n```"
out = render_md(src)
assert out.count("<pre>") == 1
assert '<div class="pre-header">js</div>' in out
assert "console.log(&#39;ok&#39;)" in out or "console.log(&#x27;ok&#x27;)" in out
# ── Security: XSS must be blocked ─────────────────────────────────────────────
def test_render_md_xss_img_tag_escaped(cleanup_test_sessions):