"""Behavioural tests that drive the actual `_applyReasoningChip()` from static/ui.js via node, not just a regex over the source. The static checks in test_reasoning_chip_btw_fixes.py confirm the *shape* of the function (no `display='none'`, the right toggle call exists, etc.) but they pass even if a runtime detail is wrong — e.g. if `inactive` were inverted, or `_normalizeReasoningEffort` mishandled whitespace, or the label fell through to a wrong value for an unknown input. This file pins the actual rendered output for every effort state so the chip's None/Default visibility cannot silently regress. """ import os import shutil import subprocess from pathlib import Path import pytest REPO_ROOT = Path(__file__).parent.parent.resolve() UI_JS_PATH = REPO_ROOT / "static" / "ui.js" NODE = shutil.which("node") pytestmark = pytest.mark.skipif(NODE is None, reason="node not on PATH") _DRIVER_SRC = r""" const fs = require('fs'); const src = fs.readFileSync(process.argv[2], 'utf8'); function makeEl() { return { style: {}, _attrs: {}, setAttribute(k, v){this._attrs[k] = String(v)}, getAttribute(k){return this._attrs[k]}, classList: { _set: new Set(), add(c){this._set.add(c)}, remove(c){this._set.delete(c)}, toggle(c, on){ const want = on === undefined ? !this._set.has(c) : Boolean(on); if (want) this._set.add(c); else this._set.delete(c); }, contains(c){return this._set.has(c)}, }, dataset: {}, title: '', textContent: '', querySelectorAll(){return []}, }; } const els = { composerReasoningWrap: makeEl(), composerReasoningLabel: makeEl(), composerReasoningChip: makeEl(), composerReasoningDropdown: makeEl(), }; els.composerReasoningWrap.style.display = 'none'; // mirrors the HTML default global.window = {}; global.document = { createElement: () => makeEl(), addEventListener: () => {}, querySelectorAll: () => [], querySelector: () => null, }; global.$ = id => els[id] || null; global.api = () => ({ then: () => ({ catch: () => {} }), catch: () => {} }); var _profileTransitionReasoningContext = null; function extractFunc(name) { const re = new RegExp('function\\s+' + name + '\\s*\\('); const start = src.search(re); if (start < 0) throw new Error(name + ' not found'); let i = src.indexOf('{', start); let depth = 1; i++; while (depth > 0 && i < src.length) { if (src[i] === '{') depth++; else if (src[i] === '}') depth--; i++; } return src.slice(start, i); } eval(extractFunc('_normalizeReasoningEffort')); eval(extractFunc('_formatReasoningEffortLabel')); eval(extractFunc('_highlightReasoningOption')); eval(extractFunc('_applyReasoningChip')); const input = JSON.parse(process.argv[3]); _applyReasoningChip(input); const result = { display: els.composerReasoningWrap.style.display, label: els.composerReasoningLabel.textContent, inactive: els.composerReasoningChip.classList.contains('inactive'), title: els.composerReasoningChip.title, }; process.stdout.write(JSON.stringify(result)); """ @pytest.fixture(scope="module") def driver_path(tmp_path_factory): p = tmp_path_factory.mktemp("reasoning_driver") / "driver.js" p.write_text(_DRIVER_SRC, encoding="utf-8") return str(p) def _apply(driver_path, value): """Run _applyReasoningChip(value) against the actual ui.js.""" import json as _json result = subprocess.run( [NODE, driver_path, str(UI_JS_PATH), _json.dumps(value)], capture_output=True, text=True, timeout=30, ) if result.returncode != 0: raise RuntimeError(f"node driver failed: {result.stderr}") return _json.loads(result.stdout) # ───────────────────────────────────────────────────────────────────────────── # The chip MUST stay visible for every effort state (issue #1068). This used # to be hidden for !eff and 'none', and the source-regex tests in # test_reasoning_chip_btw_fixes.py verify the literal `display='none'` is gone # — but only a behavioural check confirms the wrap actually receives `''`. # ───────────────────────────────────────────────────────────────────────────── class TestChipAlwaysVisible: def test_empty_string_shows_chip_with_default_label(self, driver_path): out = _apply(driver_path, "") assert out["display"] == "", f"empty effort must show the chip: {out}" assert out["label"] == "Default" assert out["inactive"] is True def test_null_shows_chip_with_default_label(self, driver_path): out = _apply(driver_path, None) assert out["display"] == "" assert out["label"] == "Default" assert out["inactive"] is True def test_none_shows_chip_with_none_label(self, driver_path): """The bug from #1068 — 'none' must NOT hide the chip.""" out = _apply(driver_path, "none") assert out["display"] == "", ( f"'none' must show the chip (the regression that started #1068): {out}" ) assert out["label"] == "None" assert out["inactive"] is True def test_low_shows_chip_active(self, driver_path): out = _apply(driver_path, "low") assert out["display"] == "" assert out["label"] == "Low" assert out["inactive"] is False def test_high_shows_chip_active(self, driver_path): out = _apply(driver_path, "high") assert out["display"] == "" assert out["label"] == "High" assert out["inactive"] is False class TestNormalizationEdgeCases: """Pin the input-normalisation contract so it can't silently shift.""" def test_uppercase_normalises(self, driver_path): # Even though the API and slash command use lowercase, defensive # normalisation matters — copy/paste of an uppercase value or a # mis-cased server response shouldn't break the chip. out = _apply(driver_path, "NONE") assert out["label"] == "None" assert out["inactive"] is True def test_whitespace_trimmed(self, driver_path): out = _apply(driver_path, " none ") assert out["label"] == "None" assert out["inactive"] is True def test_unknown_value_falls_through_visible(self, driver_path): # Defensive: unknown effort still shows the chip rather than hiding. out = _apply(driver_path, "banana") assert out["display"] == "" assert out["label"] == "Banana" assert out["inactive"] is False class TestTitleAttributeAccessibility: """The chip's `title` is the hover tooltip and a screen-reader hint — confirm it always carries the current state in human-readable form.""" def test_title_has_default_label_for_unset(self, driver_path): out = _apply(driver_path, "") assert out["title"] == "Reasoning effort: Default" def test_title_has_none_label_for_none(self, driver_path): out = _apply(driver_path, "none") assert out["title"] == "Reasoning effort: None" def test_title_has_active_label_for_high(self, driver_path): out = _apply(driver_path, "high") assert out["title"] == "Reasoning effort: High" # ───────────────────────────────────────────────────────────────────────────── # supports_thinking_toggle (ZAI GLM-4.5–5.1): the chip must STAY visible when # supported_efforts is empty but the model still accepts the thinking on/off # toggle. Without this, the gate regresses GLM-4.5/4.6/5.0/5.1 users who today # can toggle thinking on/off (#6219 round-2 review). GLM-4.7 (forced thinking) # passes supports_thinking_toggle:false and the chip hides as intended. # ───────────────────────────────────────────────────────────────────────────── _DRIVER_META_SRC = r""" const fs = require('fs'); const src = fs.readFileSync(process.argv[2], 'utf8'); function makeEl() { return { style: {}, _attrs: {}, setAttribute(k, v){this._attrs[k] = String(v)}, getAttribute(k){return this._attrs[k]}, classList: { _set: new Set(), add(c){this._set.add(c)}, remove(c){this._set.delete(c)}, toggle(c, on){ const want = on === undefined ? !this._set.has(c) : Boolean(on); if (want) this._set.add(c); else this._set.delete(c); }, contains(c){return this._set.has(c)}, }, dataset: {}, title: '', textContent: '', querySelectorAll(){return []}, }; } const els = { composerReasoningWrap: makeEl(), composerReasoningLabel: makeEl(), composerReasoningChip: makeEl(), composerReasoningDropdown: makeEl(), }; els.composerReasoningWrap.style.display = 'none'; global.window = {}; global.document = { createElement: () => makeEl(), addEventListener: () => {}, querySelectorAll: () => [], querySelector: () => null, }; global.$ = id => els[id] || null; global.api = () => ({ then: () => ({ catch: () => {} }), catch: () => {} }); var _profileTransitionReasoningContext = null; function extractFunc(name) { const re = new RegExp('function\\s+' + name + '\\s*\\('); const start = src.search(re); if (start < 0) throw new Error(name + ' not found'); let i = src.indexOf('{', start); let depth = 1; i++; while (depth > 0 && i < src.length) { if (src[i] === '{') depth++; else if (src[i] === '}') depth--; i++; } return src.slice(start, i); } eval(extractFunc('_normalizeReasoningEffort')); eval(extractFunc('_formatReasoningEffortLabel')); eval(extractFunc('_highlightReasoningOption')); eval(extractFunc('_applyReasoningOptions')); eval(extractFunc('_applyReasoningChip')); const input = JSON.parse(process.argv[3]); _applyReasoningChip(input.effort, input.meta); const result = { display: els.composerReasoningWrap.style.display, label: els.composerReasoningLabel.textContent, }; process.stdout.write(JSON.stringify(result)); """ @pytest.fixture(scope="module") def driver_meta_path(tmp_path_factory): p = tmp_path_factory.mktemp("reasoning_meta_driver") / "driver_meta.js" p.write_text(_DRIVER_META_SRC, encoding="utf-8") return str(p) def _apply_meta(driver_meta_path, effort, meta): """Run _applyReasoningChip(effort, meta) against the actual ui.js.""" import json as _json payload = _json.dumps({"effort": effort, "meta": meta}) result = subprocess.run( [NODE, driver_meta_path, str(UI_JS_PATH), payload], capture_output=True, text=True, timeout=30, ) if result.returncode != 0: raise RuntimeError(f"node meta driver failed: {result.stderr}") return _json.loads(result.stdout) class TestSupportsThinkingToggleVisibility: """#6219 round-2: empty effort ladder must NOT hide the chip when the model still accepts the thinking on/off toggle (GLM-4.5–5.1 on native zai).""" def test_empty_efforts_with_toggle_stays_visible(self, driver_meta_path): """GLM-4.6 (thinking-only tier): empty ladder + toggle=True → chip stays.""" out = _apply_meta( driver_meta_path, "", {"supported_efforts": [], "supports_thinking_toggle": True}, ) assert out["display"] == "", ( "GLM-4.5–5.1 accept thinking:{type:enabled|disabled} per Z.AI docs; " f"the chip must stay visible for the On/None control: {out}" ) def test_empty_efforts_forced_thinking_hides_chip(self, driver_meta_path): """GLM-4.7 (forced thinking): empty ladder + toggle=False → chip hides.""" out = _apply_meta( driver_meta_path, "", {"supported_efforts": [], "supports_thinking_toggle": False}, ) assert out["display"] == "none", ( "glm-4.7 forces thinking on — neither toggle nor ladder should be " f"offered; chip must hide: {out}" ) def test_effort_ladder_stays_visible_regardless_of_toggle(self, driver_meta_path): """GLM-5.2 (full ladder): effort ladder present → chip stays visible whether or not the toggle flag is set (effort implies toggle).""" out = _apply_meta( driver_meta_path, "max", {"supported_efforts": ["minimal", "low", "medium", "high", "xhigh", "max"], "supports_thinking_toggle": True}, ) assert out["display"] == "" out2 = _apply_meta( driver_meta_path, "max", {"supported_efforts": ["minimal", "low", "medium", "high", "xhigh", "max"], "supports_thinking_toggle": False}, ) assert out2["display"] == "", ( "effort ladder alone is sufficient to show the chip regardless of toggle" ) def test_default_toggle_undefined_keeps_prior_behavior(self, driver_meta_path): """When supports_thinking_toggle is absent (legacy responses), the chip visibility must follow the prior contract: empty efforts → hide.""" out = _apply_meta(driver_meta_path, "", {"supported_efforts": []}) # No supports_thinking_toggle in meta → undefined → treated as true → # chip stays visible. This is the conservative default: a missing flag # must not newly hide the chip for older response shapes. assert out["display"] == "", ( "absent supports_thinking_toggle defaults to visible (prior behavior " f"for legacy responses without the field): {out}" ) # ───────────────────────────────────────────────────────────────────────────── # Round-3 #6219: the thinking toggle must be TWO-WAY for GLM-4.5–5.1 (thinking # tier). The dropdown must expose both Default (on = clear override) and None # (off). The earlier round-2 fix kept the chip visible but the only rendered # option was "None" — so a user could turn thinking OFF but never back ON. # These tests drive the actual _applyReasoningOptions against a simulated # dropdown containing all 8 options from static/index.html (Default/none/ # minimal/low/medium/high/xhigh/max) and pin which are visible per tier. # ───────────────────────────────────────────────────────────────────────────── _DRIVER_OPTIONS_SRC = r""" const fs = require('fs'); const src = fs.readFileSync(process.argv[2], 'utf8'); function makeOption(effort) { return { style: {display: ''}, dataset: {effort: effort}, classList: { _set: new Set(), add(c){this._set.add(c)}, remove(c){this._set.delete(c)}, toggle(c, on){ const w = on === undefined ? !this._set.has(c) : Boolean(on); if(w) this._set.add(c); else this._set.delete(c); }, contains(c){return this._set.has(c)}, }, _attrs: {}, setAttribute(k, v){this._attrs[k] = String(v)}, getAttribute(k){return this._attrs[k]}, }; } function makeEl() { return { style: {}, _attrs: {}, setAttribute(k, v){this._attrs[k] = String(v)}, getAttribute(k){return this._attrs[k]}, classList: { _set: new Set(), add(c){this._set.add(c)}, remove(c){this._set.delete(c)}, toggle(c, on){ const w = on === undefined ? !this._set.has(c) : Boolean(on); if(w) this._set.add(c); else this._set.delete(c); }, contains(c){return this._set.has(c)}, }, dataset: {}, title: '', textContent: '', }; } // Options mirror static/index.html's composerReasoningDropdown exactly. const options = ['', 'none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'].map(makeOption); const optionEls = options; const els = { composerReasoningWrap: makeEl(), composerReasoningLabel: makeEl(), composerReasoningChip: makeEl(), composerReasoningDropdown: { querySelectorAll(sel){ return sel === '.reasoning-option' ? optionEls : []; } }, }; els.composerReasoningWrap.style.display = 'none'; global.window = {}; global.document = { createElement: () => makeEl(), addEventListener: () => {}, querySelectorAll: () => [], querySelector: () => null, }; global.$ = id => els[id] || null; global.api = () => ({ then: () => ({ catch: () => {} }), catch: () => {} }); var _profileTransitionReasoningContext = null; function extractFunc(name) { const re = new RegExp('function\\s+' + name + '\\s*\\('); const start = src.search(re); if (start < 0) throw new Error(name + ' not found'); let i = src.indexOf('{', start); let depth = 1; i++; while (depth > 0 && i < src.length) { if (src[i] === '{') depth++; else if (src[i] === '}') depth--; i++; } return src.slice(start, i); } eval(extractFunc('_normalizeReasoningEffort')); eval(extractFunc('_formatReasoningEffortLabel')); eval(extractFunc('_highlightReasoningOption')); eval(extractFunc('_applyReasoningOptions')); eval(extractFunc('_applyReasoningChip')); const input = JSON.parse(process.argv[3]); _applyReasoningChip(input.effort, input.meta); const visible = optionEls .filter(o => o.style.display !== 'none') .map(o => o.dataset.effort === '' ? 'Default' : o.dataset.effort); process.stdout.write(JSON.stringify({visible: visible})); """ @pytest.fixture(scope="module") def driver_options_path(tmp_path_factory): p = tmp_path_factory.mktemp("reasoning_options_driver") / "driver_options.js" p.write_text(_DRIVER_OPTIONS_SRC, encoding="utf-8") return str(p) def _visible_options(driver_options_path, effort, meta): """Return the list of visible dropdown labels after _applyReasoningChip.""" import json as _json payload = _json.dumps({"effort": effort, "meta": meta}) result = subprocess.run( [NODE, driver_options_path, str(UI_JS_PATH), payload], capture_output=True, text=True, timeout=30, ) if result.returncode != 0: raise RuntimeError(f"node options driver failed: {result.stderr}") return _json.loads(result.stdout)["visible"] class TestTwoStateToggleControl: """#6219 round-3: GLM-4.5–5.1 (thinking tier) must have BOTH Default (on) and None (off) — the toggle is two-way, not off-only.""" def test_thinking_tier_offers_both_default_and_none(self, driver_options_path): """GLM-4.6: empty ladder + toggle=True → Default + None both visible.""" visible = _visible_options( driver_options_path, "", {"supported_efforts": [], "supports_thinking_toggle": True}, ) assert "Default" in visible, ( f"Default (re-enable thinking) must be shown for thinking-tier models; " f"got {visible}" ) assert "none" in visible, ( f"None (disable thinking) must be shown; got {visible}" ) # No effort levels for the thinking-only tier. for level in ["minimal", "low", "medium", "high", "xhigh", "max"]: assert level not in visible, ( f"thinking-tier must not show effort level {level}; got {visible}" ) def test_effort_tier_offers_default_none_and_ladder(self, driver_options_path): """GLM-5.2: full ladder + toggle → Default, None, AND all 6 levels.""" visible = _visible_options( driver_options_path, "high", {"supported_efforts": ["minimal", "low", "medium", "high", "xhigh", "max"], "supports_thinking_toggle": True}, ) assert "Default" in visible assert "none" in visible for level in ["minimal", "low", "medium", "high", "xhigh", "max"]: assert level in visible, f"effort tier must show {level}; got {visible}" def test_off_then_on_round_trip_options(self, driver_options_path): """Simulate the user flow: Default → click None → click Default. At every step, BOTH Default and None must remain visible.""" meta_toggle = {"supported_efforts": [], "supports_thinking_toggle": True} # Start: Default selected (effort=''), chip shows Default+None v1 = _visible_options(driver_options_path, "", meta_toggle) # User clicks None (effort='none') v2 = _visible_options(driver_options_path, "none", meta_toggle) # User clicks Default again (effort='') v3 = _visible_options(driver_options_path, "", meta_toggle) for i, v in enumerate([v1, v2, v3], 1): assert "Default" in v and "none" in v, ( f"step {i}: both Default and None must remain visible throughout " f"the round trip; got {v}" )