Merge PR #7727 (contributor head bbdfbae2f7)
This commit is contained in:
commit
e5839b1a20
2 files changed
+47
-4
No files matched your search
+18
-4
@@ -4379,9 +4379,10 @@ def _title_language_mismatch(user_text: str, title: str) -> bool:
|
||||
short and frequently embed a borrowed Latin technical term (e.g. a CJK
|
||||
title containing the word "Python"), the title side uses a proportion
|
||||
threshold (>=35% of the title's alphabetic characters in a non-start
|
||||
script, min 2 chars) rather than a strict majority -- so a CJK title with
|
||||
one English word still trips, while an English title with a single
|
||||
foreign place-name does not.
|
||||
script, min 2 chars) rather than a strict majority. CJK titles with
|
||||
borrowed Latin terms are allowed when the title also contains CJK
|
||||
characters (#7693), but pure-Latin titles for CJK conversations are
|
||||
still rejected.
|
||||
2. The legacy German-start → English-title heuristic, preserved verbatim so
|
||||
the original behavior keeps working for same-script (latin) drift that
|
||||
the script check can't see.
|
||||
@@ -4391,13 +4392,26 @@ def _title_language_mismatch(user_text: str, title: str) -> bool:
|
||||
return False
|
||||
|
||||
# (1) Cross-script mismatch — language-agnostic.
|
||||
# CJK text routinely borrows Latin product/technical terms (e.g. "WeChat
|
||||
# Pay", "Python", "ProRes RAW"), so when the user writes in CJK, Latin
|
||||
# characters in the title are acceptable as long as the title also
|
||||
# contains CJK — i.e. the title is genuinely mixed, not pure drift.
|
||||
# A pure-Latin title for a CJK conversation is still rejected.
|
||||
# Unrelated scripts (Cyrillic, Arabic, Greek …) are always flagged.
|
||||
user_script = _dominant_script(user_text)
|
||||
if user_script:
|
||||
title_counts = _script_counts(candidate)
|
||||
title_total = sum(title_counts.values())
|
||||
if title_total >= 2:
|
||||
for script, n in title_counts.items():
|
||||
if script != user_script and n >= 2 and (n / title_total) >= 0.35:
|
||||
if script == user_script:
|
||||
continue
|
||||
# When user writes in CJK, Latin in the title is a borrowed
|
||||
# term as long as the title also contains CJK characters.
|
||||
if user_script == 'cjk' and script == 'latin':
|
||||
if title_counts.get('cjk', 0) >= 2:
|
||||
continue
|
||||
if n >= 2 and (n / title_total) >= 0.35:
|
||||
return True
|
||||
|
||||
# (2) Legacy same-script German→English heuristic.
|
||||
|
||||
@@ -71,6 +71,35 @@ def test_cjk_start_english_title_is_rejected():
|
||||
assert _title_language_mismatch("如何修复这个错误问题", "Fixing the Bug") is True
|
||||
|
||||
|
||||
def test_cjk_start_mixed_cjk_latin_title_is_allowed():
|
||||
"""CJK conversations frequently embed English product/technical terms in
|
||||
titles (e.g. 'WeChat Pay 回调失败排查'). As long as the title also
|
||||
contains CJK characters, Latin borrowed terms should not trigger rejection.
|
||||
Regression test for #7693."""
|
||||
from api.streaming import _title_language_mismatch
|
||||
|
||||
# Pure CJK user, CJK title with English product name
|
||||
assert _title_language_mismatch(
|
||||
"微信支付回调一直失败怎么办", "WeChat Pay 回调失败排查"
|
||||
) is False
|
||||
# CJK user, CJK title with English tech term
|
||||
assert _title_language_mismatch(
|
||||
"如何修复这个错误问题", "Python 代码修复"
|
||||
) is False
|
||||
# Mixed CJK+Latin user, mixed title
|
||||
assert _title_language_mismatch(
|
||||
"prores raw是否能选择压缩?", "ProRes RAW 压缩选项与 BRAW 对比"
|
||||
) is False
|
||||
|
||||
|
||||
def test_cjk_start_pure_latin_title_still_rejected():
|
||||
"""A purely Latin title for a CJK conversation is still a genuine drift."""
|
||||
from api.streaming import _title_language_mismatch
|
||||
|
||||
assert _title_language_mismatch("如何修复这个错误问题", "Fixing the Bug") is True
|
||||
assert _title_language_mismatch("如何修复这个错误问题", "Python Error Guide") is True
|
||||
|
||||
|
||||
# ── regression guards: legitimate same-script titles must NOT be rejected ───
|
||||
|
||||
def test_english_start_english_title_allowed():
|
||||
|
||||
Reference in new issue
Block a user