From 18845fd00091fba06167341f7832dd1dfbc1ed98 Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Sat, 11 Jul 2026 03:05:43 +0000 Subject: [PATCH] fix(#4732): keep mobile composer-mic dictation active through pauses (salvage of #5294) Salvaged from #5294 by @brianmmaina. Rebuilt fresh on master with the maintainer gate fix-spec: continuous+restart gated to MOBILE (desktop stays one-shot), wake-lock async race serialized, error-path toasts + abort-loop bound. Co-authored-by: brianmmaina --- static/boot.js | 136 +++++++++++- ...t_issue5294_mobile_dictation_continuity.py | 203 ++++++++++++++++++ 2 files changed, 336 insertions(+), 3 deletions(-) create mode 100644 tests/test_issue5294_mobile_dictation_continuity.py diff --git a/static/boot.js b/static/boot.js index a05b8306..0314bc0c 100644 --- a/static/boot.js +++ b/static/boot.js @@ -675,6 +675,17 @@ function _micToastKeyForRecognitionError(error){ let _finalText=''; let _prefix=''; let _isRecording=false; + // #5294 salvage — mobile composer-mic dictation continuity. + // _speechStopRequested distinguishes an intentional stop (send/toggle) from a + // natural pause so onend only auto-restarts on real pauses. _micWakeLock keeps + // the screen awake while dictating; _micWakeLockOp serializes acquire/release + // so rapid start/stop/visibility churn can't leak a lock. _micRestartCount is + // bounded by _micMaxRestarts to stop a tight loop if the audio session is stolen. + let _speechStopRequested=false; + let _micWakeLock=null; + let _micWakeLockOp=null; + let _micRestartCount=0; + const _micMaxRestarts=20; let _micHoldTimer=null; let _micHoldActive=false; let _micPointerDown=false; @@ -802,6 +813,78 @@ function _micToastKeyForRecognitionError(error){ } } + // Gate continuous dictation to MOBILE so desktop stays one-shot (single + // utterance). An explicit hermes-mic-continuous flag wins in both directions, + // mirroring the hermes-voice-continuous pattern: 'true' opts a desktop in, + // 'false' opts a mobile out. Absent a flag, coarse-pointer (touch) devices get + // continuity and everything else stays single-utterance. + function _micDictationContinuous(){ + try{ + const flag=localStorage.getItem('hermes-mic-continuous'); + if(flag==='true') return true; + if(flag==='false') return false; + }catch(_){} + try{ return window.matchMedia('(pointer:coarse)').matches; }catch(_){ return false; } + } + + // Only auto-restart the composer-mic session on a natural pause: continuity + // must be enabled (mobile/opt-in), the session must still be active speech, + // the stop must not have been requested, and we must be under the restart cap. + function _micShouldRestartDictation(){ + return _micDictationContinuous() + && !_speechStopRequested + && !!window._micActive + && _activeCaptureMode==='speech' + && _micRestartCount<_micMaxRestarts; + } + + // Screen Wake Lock while dictating. All acquire/release ops are chained onto a + // single in-flight promise (_micWakeLockOp) so rapid start/stop/visibility + // churn runs strictly in order and can't interleave to leak a lock or null + // _micWakeLock mid-request. + function _acquireMicWakeLock(){ + if(!navigator.wakeLock) return Promise.resolve(); + _micWakeLockOp=Promise.resolve(_micWakeLockOp).then(async()=>{ + if(_micWakeLock) return; + if(!window._micActive||_activeCaptureMode!=='speech') return; + try{ + const lock=await navigator.wakeLock.request('screen'); + // If the session ended while awaiting, don't hold a stale lock. + if(!window._micActive||_activeCaptureMode!=='speech'){ + try{ await lock.release(); }catch(_){} + return; + } + _micWakeLock=lock; + _micWakeLock.addEventListener?.('release',()=>{ _micWakeLock=null; },{once:true}); + }catch(_){ + _micWakeLock=null; + } + }); + return _micWakeLockOp; + } + + function _releaseMicWakeLock(){ + _micWakeLockOp=Promise.resolve(_micWakeLockOp).then(async()=>{ + const lock=_micWakeLock; + _micWakeLock=null; + if(!lock) return; + try{ await lock.release(); }catch(_){} + }); + return _micWakeLockOp; + } + + // The OS drops a screen wake lock when the tab is hidden; reacquire on return + // if we're still dictating (release on hide is a no-op if none is held). + document.addEventListener('visibilitychange',()=>{ + if(document.visibilityState==='hidden'){ + void _releaseMicWakeLock(); + return; + } + if(window._micActive&&_activeCaptureMode==='speech'){ + void _acquireMicWakeLock(); + } + }); + function _stopMic(){ _micStartSeq+=1; _isRecording=false; @@ -811,6 +894,7 @@ function _micToastKeyForRecognitionError(error){ // which would otherwise make us stop the wrong backend and orphan the other // (#3169 Codex review). _activeCaptureMode is pinned at start. if(recognition && _activeCaptureMode==='speech'){ + _speechStopRequested=true; recognition.stop(); return; } @@ -826,7 +910,9 @@ function _micToastKeyForRecognitionError(error){ function _ensureSpeechRecognition(){ if(!SpeechRecognition) return null; const sr=recognition||new SpeechRecognition(); - sr.continuous=false; + // Desktop dictation stays one-shot (single utterance); mobile / opt-in + // devices run continuous so a natural pause doesn't end the session (#5294). + sr.continuous=_micDictationContinuous(); sr.interimResults=true; sr.lang=(typeof _locale!=='undefined'&&_locale._speech)||'en-US'; @@ -850,9 +936,32 @@ function _micToastKeyForRecognitionError(error){ ? _prefix+' '+_finalText.trimStart() : _prefix+_finalText) : ta.value; - _setRecording(false); ta.value=committed; autoResize(); + // Mobile / opt-in continuity: a natural pause ends this recognition run but + // the user is still dictating, so restart to keep the session alive. Desktop + // (one-shot) and intentional stops (_speechStopRequested) skip this and + // finalize. Bounded by _micMaxRestarts so a stolen audio session can't loop. + if(_micShouldRestartDictation()){ + _prefix=committed&&!committed.endsWith(' ')&&!committed.endsWith('\n') + ? committed+' ' + : committed; + _finalText=''; + _micRestartCount++; + try{ + sr.start(); + return; + }catch(err){ + // Restart failed (e.g. the audio session was taken by another app). + // Surface it instead of silently dropping to idle (Greptile P2). + showToast(t('mic_error')+String((err&&err.message)||'restart')); + } + } + _speechStopRequested=false; + _isRecording=false; + _micRestartCount=0; + void _releaseMicWakeLock(); + _setRecording(false); if(window._micPendingSend){ window._micPendingSend=false; send(); @@ -861,10 +970,25 @@ function _micToastKeyForRecognitionError(error){ }; sr.onerror=(event)=>{ + // While dictating with continuity on, a no-speech/aborted error is a normal + // pause or transient audio-session hiccup — swallow it and let onend restart + // (bounded by _micMaxRestarts). Desktop one-shot still surfaces the toast. + if((event.error==='no-speech'||event.error==='aborted') + && _micDictationContinuous() + && window._micActive + && _activeCaptureMode==='speech' + && !_speechStopRequested + && _micRestartCount<_micMaxRestarts){ + return; + } + _speechStopRequested=false; _setRecording(false); window._micPendingSend=false; _isRecording=false; - if(event.error==='network'||event.error==='not-allowed'){ + _micRestartCount=0; + void _releaseMicWakeLock(); + if(event.error==='network'||event.error==='not-allowed' + ||event.error==='service-not-allowed'||event.error==='audio-capture'){ // Persist SR failure: next reload will skip SpeechRecognition localStorage.setItem(_micForceMediaRecorderKey,'1'); _forceMediaRecorder=true; @@ -960,8 +1084,14 @@ function _micToastKeyForRecognitionError(error){ } if(recognition && !_forceMediaRecorder && !_rawAudioMode){ _activeCaptureMode='speech'; + _speechStopRequested=false; + _micRestartCount=0; + // Refresh continuity gate at start so a settings/orientation change takes + // effect for this session (desktop stays one-shot, mobile stays continuous). + recognition.continuous=_micDictationContinuous(); recognition.lang=(typeof _locale!=='undefined'&&_locale._speech)||'en-US'; recognition.start(); + void _acquireMicWakeLock(); _setRecording(true); return; } diff --git a/tests/test_issue5294_mobile_dictation_continuity.py b/tests/test_issue5294_mobile_dictation_continuity.py new file mode 100644 index 00000000..40e80909 --- /dev/null +++ b/tests/test_issue5294_mobile_dictation_continuity.py @@ -0,0 +1,203 @@ +"""Behavioural regression for #5294 (salvage of the composer-mic dictation fix). + +The composer microphone (SpeechRecognition inside `_ensureSpeechRecognition`) +must keep a MOBILE dictation session alive across a natural pause, while DESKTOP +dictation stays one-shot (a single utterance that finalizes on the first pause). + +Rather than regex the source, this drives the ACTUAL `_ensureSpeechRecognition` +(and its gate helpers `_micDictationContinuous` / `_micShouldRestartDictation`) +from static/boot.js via node with a fake SpeechRecognition, then simulates a +final result followed by a pause (`onend`) and observes whether the session +restarts. It pins both directions so the mobile-gate can't silently regress into +either "desktop keeps restarting" or "mobile dies on every pause". +""" +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parent.parent.resolve() +BOOT_JS_PATH = REPO_ROOT / "static" / "boot.js" + +NODE = shutil.which("node") +pytestmark = pytest.mark.skipif(NODE is None, reason="node not on PATH") + + +_DRIVER_SRC = r""" +const fs = require('fs'); +const src = fs.readFileSync(process.argv[2], 'utf8'); +const input = JSON.parse(process.argv[3]); + +// ── Extract a named function body from boot.js by brace-matching ──────────── +function extractFunc(name) { + const re = new RegExp('function\\s+' + name + '\\s*\\('); + const start = src.search(re); + if (start < 0) throw new Error(name + ' not found'); + let i = src.indexOf('{', start); + let depth = 1; i++; + while (depth > 0 && i < src.length) { + if (src[i] === '{') depth++; + else if (src[i] === '}') depth--; + i++; + } + return src.slice(start, i); +} + +// ── Fake browser + closure environment ────────────────────────────────────── +const _store = Object.assign({}, input.store || {}); +global.localStorage = { + getItem: (k) => (k in _store ? _store[k] : null), + setItem: (k, v) => { _store[k] = String(v); }, +}; +global.navigator = { wakeLock: null }; +global.window = { + _micActive: false, + _micPendingSend: false, + matchMedia: (q) => ({ matches: q.indexOf('pointer:coarse') >= 0 ? !!input.coarse : false }), +}; +global.document = { addEventListener: () => {}, visibilityState: 'visible' }; + +// Closure state that the extracted functions read/mutate at top level. +let SpeechRecognition = function () { + this.continuous = undefined; + this.interimResults = undefined; + this.lang = undefined; + this.onstart = null; + this.onresult = null; + this.onend = null; + this.onerror = null; + this.startCalls = 0; + this.stopCalls = 0; + this.start = function () { this.startCalls++; }; + this.stop = function () { this.stopCalls++; }; +}; +let recognition = null; +let _finalText = ''; +let _prefix = ''; +let _isRecording = false; +let _speechStopRequested = false; +let _micWakeLock = null; +let _micWakeLockOp = null; +let _micRestartCount = 0; +const _micMaxRestarts = 20; +let _activeCaptureMode = null; +let _forceMediaRecorder = false; +const _micForceMediaRecorderKey = 'mic_force_mediarecorder'; +const _locale = { _speech: 'en-US' }; +let ta = { value: '' }; +const toasts = []; + +function autoResize() {} +function send() {} +function showToast(m) { toasts.push(m); } +function t(k) { return k; } +function _applyDeferredServerSttFlip() {} +function _micToastKeyForRecognitionError() { return null; } +function _setRecording(on) { + window._micActive = on; + if (!on) { _finalText = ''; _prefix = ''; } +} +function _releaseMicWakeLock() { _micWakeLock = null; return Promise.resolve(); } +function _acquireMicWakeLock() { return Promise.resolve(); } + +// ── The real functions under test ─────────────────────────────────────────── +eval(extractFunc('_micDictationContinuous')); +eval(extractFunc('_micShouldRestartDictation')); +eval(extractFunc('_ensureSpeechRecognition')); + +// ── Drive a dictation session: start → final result → natural pause ───────── +recognition = _ensureSpeechRecognition(); +const continuousAfterEnsure = recognition.continuous; + +// Simulate _startMicCapture's speech branch entering an active session. +_activeCaptureMode = 'speech'; +_speechStopRequested = !!input.stopRequested; +_micRestartCount = 0; +window._micActive = true; +recognition.start(); // initial start (mimics _startMicCapture) +const startCallsAfterStart = recognition.startCalls; + +// A final result lands, then the recognizer ends on a natural pause. +recognition.onstart(); +recognition.onresult({ + resultIndex: 0, + results: [Object.assign([{ transcript: 'hello world' }], { isFinal: true, length: 1 })], +}); +recognition.onend(); + +const out = { + continuousAfterEnsure: continuousAfterEnsure, + restarted: recognition.startCalls > startCallsAfterStart, + micActiveAfterPause: !!window._micActive, + taValue: ta.value, + toasts: toasts, +}; +process.stdout.write(JSON.stringify(out)); +""" + + +@pytest.fixture(scope="module") +def driver_path(tmp_path_factory): + p = tmp_path_factory.mktemp("mic_dictation_driver") / "driver.js" + p.write_text(_DRIVER_SRC, encoding="utf-8") + return str(p) + + +def _run(driver_path, *, coarse=False, store=None, stop_requested=False): + payload = {"coarse": coarse, "store": store or {}, "stopRequested": stop_requested} + result = subprocess.run( + [NODE, driver_path, str(BOOT_JS_PATH), json.dumps(payload)], + capture_output=True, text=True, timeout=30, + ) + if result.returncode != 0: + raise RuntimeError(f"node driver failed: {result.stderr}") + return json.loads(result.stdout) + + +# ───────────────────────────────────────────────────────────────────────────── +# DESKTOP must stay ONE-SHOT: continuous=false and no auto-restart on a pause. +# ───────────────────────────────────────────────────────────────────────────── +class TestDesktopStaysOneShot: + def test_desktop_recognition_is_not_continuous(self, driver_path): + out = _run(driver_path, coarse=False) + assert out["continuousAfterEnsure"] is False + + def test_desktop_does_not_restart_on_natural_pause(self, driver_path): + out = _run(driver_path, coarse=False) + assert out["restarted"] is False, "desktop dictation must finalize on the first pause" + assert out["micActiveAfterPause"] is False, "desktop session must end after the pause" + + def test_desktop_optout_flag_forces_one_shot_even_on_coarse_pointer(self, driver_path): + # An explicit 'false' opt-out must win even on a touch/coarse device. + out = _run(driver_path, coarse=True, store={"hermes-mic-continuous": "false"}) + assert out["continuousAfterEnsure"] is False + assert out["restarted"] is False + + +# ───────────────────────────────────────────────────────────────────────────── +# MOBILE must keep the session ALIVE through a pause via restart-on-onend. +# ───────────────────────────────────────────────────────────────────────────── +class TestMobileKeepsSessionAlive: + def test_mobile_recognition_is_continuous(self, driver_path): + out = _run(driver_path, coarse=True) + assert out["continuousAfterEnsure"] is True + + def test_mobile_restarts_on_natural_pause(self, driver_path): + out = _run(driver_path, coarse=True) + assert out["restarted"] is True, "mobile dictation must restart to survive a natural pause" + assert out["micActiveAfterPause"] is True, "mobile session must remain active after the pause" + + def test_mobile_intentional_stop_does_not_restart(self, driver_path): + # _speechStopRequested (set by _stopMic on send/toggle) must suppress the + # restart so an intentional stop still finalizes on mobile. + out = _run(driver_path, coarse=True, stop_requested=True) + assert out["restarted"] is False + assert out["micActiveAfterPause"] is False + + def test_desktop_optin_flag_enables_continuity(self, driver_path): + # An explicit 'true' opt-in makes a non-coarse (desktop) device continuous. + out = _run(driver_path, coarse=False, store={"hermes-mic-continuous": "true"}) + assert out["continuousAfterEnsure"] is True + assert out["restarted"] is True