diff --git a/.env.example b/.env.example index 3a3d4c13..a96b87f5 100644 --- a/.env.example +++ b/.env.example @@ -132,6 +132,13 @@ # Extra sources for the notes drawer (see docs for the expected format). # HERMES_WEBUI_EXTERNAL_NOTES_SOURCES= +# Sidebar recency window: how many recent sessions are listed (default: 20). +# Also caps how many delegated subagent children can nest at once; raise it +# for wide fan-outs. Invalid or non-positive values fall back to 20. +# Values above 200 are clamped. Resolved before profile init, so a profile +# .env cannot override this server-wide bound. +# HERMES_WEBUI_VISIBLE_SESSION_LIMIT=20 + # Directory scanned for WebUI plugins (default: ~/.hermes/plugins). # HERMES_WEBUI_PLUGINS_DIR=~/.hermes/plugins diff --git a/.github/workflows/conversation-lifecycle.yml b/.github/workflows/conversation-lifecycle.yml index e952d34f..24ede84b 100644 --- a/.github/workflows/conversation-lifecycle.yml +++ b/.github/workflows/conversation-lifecycle.yml @@ -12,7 +12,9 @@ on: - 'api/**.py' - 'server.py' - 'tests/browser_conversation_lifecycle.py' + - 'tests/sidebar_terminal_barrier_server.py' - 'tests/browser_historical_transcript_hydration.py' + - 'tests/browser_reconnect_scene_redraw.py' - 'requirements*.txt' - 'pyproject.toml' - '.github/workflows/conversation-lifecycle.yml' @@ -23,7 +25,9 @@ on: - 'api/**.py' - 'server.py' - 'tests/browser_conversation_lifecycle.py' + - 'tests/sidebar_terminal_barrier_server.py' - 'tests/browser_historical_transcript_hydration.py' + - 'tests/browser_reconnect_scene_redraw.py' - 'requirements*.txt' - 'pyproject.toml' - '.github/workflows/conversation-lifecycle.yml' @@ -32,7 +36,8 @@ jobs: live-to-final: runs-on: ubuntu-latest timeout-minutes: 10 - continue-on-error: true + # Keep existing proof rows informational; reconnect regressions fail CI. + continue-on-error: ${{ matrix.name != 'reconnect-scene-redraw' }} strategy: fail-fast: false matrix: @@ -45,6 +50,9 @@ jobs: scenario: terminal-error - name: historical-transcript-hydration script: tests/browser_historical_transcript_hydration.py + - name: reconnect-scene-redraw + script: tests/browser_reconnect_scene_redraw.py + browsers: chromium steps: - uses: actions/checkout@v4 @@ -69,6 +77,7 @@ jobs: LIFECYCLE_ARTIFACT_DIR: lifecycle-artifacts HISTORICAL_HYDRATION_ARTIFACT_DIR: lifecycle-artifacts LIFECYCLE_SCENARIO: ${{ matrix.scenario }} + BROWSERS: ${{ matrix.browsers }} run: python ${{ matrix.script }} - name: Run settlement frame proof @@ -79,6 +88,14 @@ jobs: LIFECYCLE_TEST_BITE: settle-worklog-frame-proof run: python tests/browser_conversation_lifecycle.py + - name: Recover a missing terminal frame + if: matrix.scenario == 'normal' + env: + LIFECYCLE_ARTIFACT_DIR: lifecycle-artifacts/missing-terminal + LIFECYCLE_SCENARIO: normal + LIFECYCLE_TEST_BITE: sidebar-idle-missing-terminal + run: python tests/browser_conversation_lifecycle.py + - name: Upload failure artifacts if: failure() uses: actions/upload-artifact@v4 diff --git a/AGENTS.md b/AGENTS.md index d1b785bc..713ebedb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,6 +67,39 @@ Follow that checklist's safety rules: the manual verification performed. - For runtime, streaming, recovery, replay, compression, or sidebar metadata changes, name the state layer being mutated and prove the relevant invariant. +- Active-run Steer resolves the stream-bound agent with explicit stream and + worker ownership before consulting the reusable session cache. Compression + may rotate the agent identity; steering must never evict or close an agent. + Keep HTTP response writes outside runtime registry locks. A Gateway-owned + active run must resolve to the Gateway outcome before any local cache fallback, + even when no in-process worker is registered for that stream. Stop publishes + cancellation and detaches stream/agent entries under the same stream lock + (STREAMS_LOCK -> ACTIVE_RUNS_LOCK); interrupt and session persistence remain + outside it. The retained cache-only path (no registered worker) revalidates + stream membership, owner, and active-run session/backend/phase and enqueues + agent.steer() under that same lock edge, so a Stop that claims cancellation + never strands guidance an earlier Steer response reported as accepted. Test + both Stop/Steer orderings — registered and cache-only — with deterministic + barriers. Initial active-run publication and its cancel flag must share that + lock edge with Stop; do not recreate the flag after journal setup. Worker + registration must also check its retained cancel event and live stream + membership: Stop can remove CANCEL_FLAGS during initialization. + Do not re-register a cancelled worker; finalize outside the stream lock. + Local Steer accepts only explicit starting/running phases. Close admission + by publishing finalizing under STREAMS_LOCK before the last pending-steer + drain; earlier accepted guidance is drained, later guidance is rejected with + `not_running` while the owned stream is still live (not `stream_dead`). Use + one idempotent terminal settlement before done/error/end and final cleanup, + covering returned errors, exceptions and self-heal, not only the success path. + Merge Agent-returned `pending_steer` with the registered worker's final slot + drain and emit leftovers before terminal events, outside registry locks. + Test both drain/Steer orderings, including compression-rotated identities. +- Inactive-session recovery is separate from live Steer. Resolve durable + compression lineage in the session's profile database, read-only, even when + the WebUI sidecar has no snapshot flag. Never reopen a sealed parent. Reject + stale chat POSTs before workspace/model/pending-state mutation; the browser + loads the continuation and preserves the draft without automatic replay. + Explicit closures and unknown terminal reasons do not authorize a redirect. - For Docker build changes in `docker_init.bash`, mirror directory exclusions in both the `rsync` and `cp -a` paths — `/opt/hermes` may contain subdirectories with restricted permissions (e.g. `.playwright/`). diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 72be8bc8..2ce017cf 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -237,6 +237,29 @@ Session is a plain Python class (not a dataclass, not SQLAlchemy): title_from(): takes messages list, finds first user message, returns first 64 chars. Called after run_conversation() completes to set the session title retroactively. +#### Session transcript reconciliation with `state.db` + +`reconciled_state_db_messages_for_session()` uses +`merge_session_messages_append_only()` to combine a WebUI sidecar or context +projection with active Agent `state.db` rows. Its explicit +`incoming_provenance="state_db"` fence permits a state-only row whose timestamp +predates the sidecar tail to use a safe chronological slot. Paginated +`msg_limit` consumers rely on this merged order directly rather than applying a +later timestamp sort. + +The same merge helper also stitches ordered child sidecars onto archived +compression parents. Those calls leave incoming provenance unverified, so their +stable message sequence remains append-only even when parent rows carry later, +restamped timestamps. Timestamp alone is never authority to move a continuation +inside its parent transcript. + +If an older row could only be placed before the first surviving sidecar/context +row, the insertion helper declines it to avoid resurrecting compacted history. +Reconciliation then appends that row instead of dropping it; rows without a +usable timestamp and rows at or after the sidecar tail also append normally. +The fallback therefore preserves an accepted state-only row when exact ordering +is ambiguous, while safely placeable recovery rows remain chronological. + #### Imported `state.db` sidebar projection `api.models.get_cli_sessions()` projects conversations from the active Hermes diff --git a/CHANGELOG.md b/CHANGELOG.md index 172494cc..e0bfaa58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,256 @@ ## [Unreleased] +### Added + +- **Full-session resolve concurrency is configurable.** `HERMES_WEBUI_MAX_SESSION_RESOLVE` sets how + many full-transcript session resolves may run at once (default 2, a positive integer up to 64; + zero, negative, non-numeric or out-of-range values fall back to 2). It is process-wide, so a + profile's `.env` cannot override it. (#7421, #7656 by @happy5318) +- **The sidebar's recent-session window is configurable.** `HERMES_WEBUI_VISIBLE_SESSION_LIMIT` sets + how many recent sessions the sidebar lists (default 20). It also bounds how many delegated subagent + children can nest at once, so raise it for wide fan-outs. Invalid or non-positive values fall back to + 20, values above 200 are clamped, and it is resolved before profile init so a profile `.env` cannot + override it. (#7631 by @carlotestor) + +### Performance + +- **Opening a session while its task is still running is much faster.** Rebuilding the live + snapshot from the run journal parsed the journal twice, walked every metering row and grew the + reasoning text with repeated string concatenation, which is quadratic on long runs. The journal + is now parsed once, metering rows are skipped (their timestamp watermark is kept) and reasoning + deltas are joined once per segment. On a 15 MB / 22.7k-event journal the author measured the + rebuild going from 4.34 s to 0.49 s with a byte-identical snapshot. The interim-echo check now matches a + compact-equivalent suffix without a fixed window, so an echo stretched by interior whitespace is + no longer shown twice. (#7310, #7569 by @happy5318) + ### Fixed +- **A session deleted during a restart no longer produces a spurious recovery warning.** When + WebUI startup recovery re-attached background processes, a session that had vanished between + enumeration and rebind raised a `KeyError` that was logged as a warning. It now follows the + existing skip path, confined to the session lookup, so the vanished owner is skipped, live owners + still rebind, and registry errors still warn. (#7753, #7774 by @happy5318) + +- **Waiting on the Agent's session lease is shown as a warning instead of looking stuck.** When + another Hermes process (gateway, CLI or cron) holds the session's turn lease, the Agent's + "another Hermes process is using this session" notices now reach the chat as a warning status + instead of being dropped, and the status clears when the run ends. Classification keys on the Agent + status kind (`lifecycle` / `warn`), so user-authored text can never be promoted to a warning. + (#7760 by @ruizanthony) +- **A stale in-flight projection can no longer reach a gateway watcher after its last subscriber + leaves.** Final unsubscribe and queue eviction now invalidate the cache and fence projections that + were already in flight, without holding the lock across database reads or SSE writes, so a client + that re-subscribes gets a fresh snapshot instead of a stale one. (#7761 by @ruizanthony) +- **A burst of "session busy" refusals no longer drops a background-task completion.** When + `start_session_turn()` refused a completion wake-up with a transient 409 (Agent runtime stale, + process wake-ups paused, or the session busy with another turn), the bridge released the durable + claim as a plain failure, so a few refusals in a row could terminally drop a completion whose + session was alive and waiting. Transient refusals now return the claim as retryable, while hard + failures still use up the attempt budget. (#7758 by @ruizanthony) +- **A first visit now uses the browser's language.** The server stores "no preference" as `null` + instead of defaulting to `"en"`, so a first-time visitor gets `navigator.language` while an + explicitly saved language (including English) still wins, and legacy `settings.json` files that + already hold `"en"` keep English. Reading the browser language is guarded, so an environment where + `navigator` throws falls back cleanly. (#7622, #7730 by @happy5318) +- **The live model list for a custom provider respects its `models:` allowlist, without treating + per-model metadata as one.** `/api/models/live` filters a custom provider's live catalog to its + configured `models:` when that is a list, or when it is a mapping with `discover_models: false`. + A mapping of per-model settings (the shape `hermes setup` writes) keeps the full live catalog, as + the Agent does. (#7165 by @happy5318) +- **`MEDIA:` links wrapped in inline code no longer 404.** Every `MEDIA:` capture site (renderer, + streaming parser, TTS stripper, session-media authorization and snapshot capture, seven in all) + swallowed the closing backtick of `` `MEDIA:/path` `` into the path, so the file lookup and the + session allowlist both missed. Backtick-wrapped refs are now normalized first, while bare paths that + genuinely contain a backtick keep their full name. (#7359, #7708 by @happy5318) +- **A turn's Worklog no longer vanishes when the sidebar reports idle before the final frame.** + `/api/sessions` could say a session was idle before the chat stream's terminal frame reached the + page, and three sidebar paths (idle reconciliation, the INFLIGHT purge and optimistic-row + retirement) then erased the pane's Worklog and stream id, so the late `done` was rejected as stale + and the settled scene was never saved. The cleanup now waits briefly for the pane's own open + stream, then checks `/api/chat/stream/status` once, and falls back to the existing + interrupted-stream recovery if that check fails. (#7749 by @franksong2702) +- **A session's run journal can no longer be written out of order.** The journal writer reserved a + sequence number under the per-path lock, released it, and then appended, so two concurrent writers + could land sequence N+1 on disk before N and the replay reader would stop at the gap + (`replay_noncontiguous`). Sequence allocation and the physical append now happen under the same + existing lock. (#7751 by @franksong2702) +- **A stopped chat can no longer publish or reuse its agent after Stop.** A worker that was + cancelled could still publish its cached Agent or invoke it after Stop landed, and its late + cleanup could close or evict the same Agent object a successor turn had just picked up. The initial + path and both credential self-heal paths now take one Stop admission, the cancellation event, live + stream and exact Agent are rechecked immediately before invocation (no registry lock is held across + provider or tool execution), and cache/lifecycle handles are retired only while the surviving owner + still matches. (#7748 by @franksong2702) +- **Reconnecting to a running session no longer redraws every tool card over and over.** After a + reconnect restored the live activity scene from the run journal, the client also replayed its older + cached in-flight tool list on top, so a turn with N tool cards redrew them N times. When the + journal-backed scene restores successfully the replay is now skipped (newer rows still arrive + through the reattached stream); the replay stays for legacy restores and for a failed or + unavailable scene. (#7436, @atchisonbrent) + +- **A settled assistant answer is no longer shown twice (#2051).** Two client-side paths could put a + second copy of a finished answer on screen after a turn settled: a stale live-turn node that was + still treated as live, and a settled rebuild branch that appended a turn instead of replacing it. + The same fix stops an interrupted turn's notice from appearing twice, and stops the composer's + model picker from listing one configured custom-provider model twice. The stored transcript was + always correct; only the rendering duplicated it. (#7374, @Thireus) + +- **A hidden browser tab stops polling a session that was deleted.** When a tab is in the + background, its stream poll kept asking for a session every few seconds after it was deleted or + archived away, forever. A `404`/`410` for the polled session now stops that poll; `503` and network + errors keep retrying, a late `404` for one session cannot stop another session's poll, and making + the tab visible again reopens the live stream. (#7299, #7716 by @happy5318) +- **The OpenRouter setup no longer offers a model OpenRouter doesn't serve.** Onboarding offered + `z-ai/glm-4.5-flash`, which is not in OpenRouter's catalog, so picking it failed on the first + message. That slot now offers `z-ai/glm-4.5-air`, Z.AI's nearest light model. The direct Z.AI + setup still offers GLM-4.5 Flash. (#7520, #7734 by @MuhammadUsamaMX) +- **Opening the sidebar can no longer stall the agent's writes to `state.db`.** Several WebUI + paths that only read the agent's `state.db` could quietly become writers. The read-only opener + fell back to a writable connection when `mode=ro` failed. The session listing self-healed a + missing `idx_messages_session` with `CREATE INDEX` through its own writable connection, which + holds the SQLite writer lock for minutes on a large `messages` table. The cron sidebar, insights + and deep-health checks opened the database with a bare `sqlite3.connect()`. Every one of these + readers now opens strictly read-only (`file:...?mode=ro`) and never creates an index. A missing + index falls back to the existing pre-aggregated listing. The gateway watcher also no longer polls + the database when no client is subscribed. Operators with an older agent can create the read + indexes in a drained maintenance window with `scripts/ensure_state_db_read_indexes.py`; see + `docs/troubleshooting.md`. (#7445 by @ruizanthony) + +- **A failed chat launch no longer leaves the session stuck "running".** If the worker thread + couldn't start, its stream ownership and the session's pending-stream fields were never cleared, + so the session could look permanently active and the registries grew. Launch failure now clears + exactly that stream's records, after releasing the chat-start lock, and never evicts a successor + stream. (#6869, #6937 by @jbdrak) +- **A compressed conversation no longer shows up twice in the sidebar.** When context + compression started the continuation a few milliseconds before the parent was marked ended, the + continuation wasn't recognised. The sidebar then showed a duplicate same-title row plus a + spurious child-session entry, and opening the conversation didn't stitch the transcript. Both the + sidebar and the transcript stitcher now accept a bounded (2 s) early start when every other + lineage signal agrees: same source, direct parent link, a compression (or `cli_close`) end reason and not a fork. + (#6931, #7021 by @webtecnica) +- **The profile switcher no longer 500s in a two-container Docker setup.** With the agent + source not mounted (`HERMES_WEBUI_CHAT_BACKEND=gateway`), `GET /api/profiles` fell through to + a skills-stats fallback that imported `agent.skill_utils` unguarded, so the missing module + surfaced as an error on every profile-list load. The import is now guarded and the skill + stats report as unknown in that case, so the picker stays usable (the skills line is simply + omitted). Thanks @webtecnica. (#7305, #7312) + +- **A title that mixes Chinese, Japanese or Korean with English terms is no longer rejected.** + The cross-script guard that stops a generated title from drifting into the wrong language + treated the borrowed Latin words in a CJK title as drift, so a valid title like + `WeChat Pay 回调失败排查` or `Python 代码修复` was thrown away and the session kept its + fallback title. Latin terms are now accepted when the title also contains CJK text, while + an all-Latin title for a CJK conversation is still rejected. Thanks @MuhammadUsamaMX. + (#7693, #7727) + +- **A late-arriving prompt no longer renders below the reply it asked for.** When a message + reached the transcript from `state.db` after the sidecar had already been merged⟪HERMES-CONTEXT-COMPRESSION: 809 of 1,009 chars omitted here by Hermes's context compressor. This is NOT part of the original tool call and must never be reproduced in new output — always write full, untruncated content.⟫- **A rejected request no longer poisons the next one on the same connection.** `server.py` + is a raw HTTP/1.1 handler where `rfile` is the socket itself, so answering a request + before reading its body left those bytes queued. The next request on a keep-alive + connection was then parsed starting mid-body, and the client got + `400 Bad request syntax ('{"a": "b"}GET /api/health HTTP/1.1')` — an error naming a + request it never sent, which is expensive to diagnose from the client side. Every + reject path now arms `Connection: close` when a body is still pending, and framing is + validated strictly (RFC 9110 `1*DIGIT`, duplicate and comma-combined `Content-Length` + reconciled, every `Transfer-Encoding` refused since nothing here decodes one). A + genuinely bodyless rejection keeps its keep-alive, so healthy pooled connections are + not dropped. Thanks @rodrigogs. (#7550, #6658) + +- **Opening a session that belongs to another profile now offers to switch to it instead of + looking deleted.** Several cross-profile guards answered `404 Session not found`, which the + front-end treats as a missing session and self-heals by clearing the URL and local storage — + so a valid deep link into another profile's session destroyed its own way back. Those guards + now return the same `409 session_profile_mismatch` the detail-load endpoint has used since + #5419, while a genuinely missing session still 404s and still self-heals. The clarify card, + compression-recovery card and manual-compression flow were each treating *every* `409` as + their own "stale" signal and are now scoped so a cross-profile refusal no longer hides a live + prompt or reports a false compression failure. Thanks @happy5318. (#7710, #7714) + +- **An OIDC-only or passkey-only deployment no longer shows a dead password prompt.** The + `/login` page rendered the password input, submit button and passkey control + unconditionally, so an instance with native OIDC configured and no + `HERMES_WEBUI_PASSWORD` displayed a password form that silently 401'd every submit. The + controls are now gated on the auth methods actually configured, with the passkey button + kept for passwordless-passkey instances where it is the only login affordance. Thanks + @happy5318. (#7056, #7715) + +- **A model whose native id contains a slash no longer poisons its configured-model badge.** + The badge builder synthesised a `{provider}/{model}` alias alongside the bare id, so a + model already carrying a slash (for example `commandcode` + `deepseek/deepseek-v4-flash`) + produced `commandcode/deepseek/deepseek-v4-flash` — a non-functional id that leaked into + the badge map, persisted into session state and returned `HTTP 400` from the agent. Only + the bare id and the `@provider:model` form are emitted now. Thanks @happy5318. (#7290, #7709) + +- **A conversation whose history carries an explicit `null` tool-call list no longer breaks + the tool-call summary.** `_extract_tool_calls_from_messages` used `.get('tool_calls', [])`, + which returns the default only when the key is *absent* — when a stored message carried the + key with a `null` value, the code tried to iterate `None` and raised `TypeError`. Reading + such a session now skips the empty entry and still summarises the tool calls that follow it. + Thanks @KayZz69. (#7265) + +- **The opencode-go provider lists its real models again instead of a frozen snapshot.** Model + discovery had fallen back to a hard-coded catalog, so models added or removed upstream never + appeared. The provider now queries live, with the lookup scoped to opencode-go alone: a slow + or unreachable endpoint falls back to the static catalog rather than blocking the picker for + anyone else, older Agent builds that lack the discovery API keep the previous behaviour, and a + configured model allowlist still wins over whatever discovery returns. Thanks + @shameez-struggles-to-commit. (#7220) + +- **Denying a gateway approval now retires the request instead of leaving the run waiting.** + When an approval routed through the gateway was denied, the deny reached the agent but the + local producer was never retired, so the run could sit waiting on a decision that had already + been made. Denial now settles the producer atomically, scoped to the exact + `(session_id, run_id, approval_id)` that was answered — sibling approvals from the same run + and approvals from other runs are left untouched — and repeated or late responses are bounded + no-ops rather than resurrecting a settled request. Thanks @snoyberg. (#7570) + +- **The approval card's "Skip all this session" button no longer shows two lightning bolts.** + The button rendered its ⚡ twice — once from the icon span in the markup and again from the + translated label, which carried its own leading glyph in 14 of 15 locales. The icon now comes + only from the markup, matching every sibling button on the card (Allow once / Allow session / + Always allow / Deny all pair an icon element with a glyph-free label), and translators no + longer carry the symbol in their strings. (#7701) + +- **The gateway watcher no longer polls `state.db` around the clock with nobody listening.** + Its poll loop re-fingerprinted the gateway state database every few seconds whether or not + any SSE client was attached, and slept in 0.1s increments — roughly 10 wakeups a second, all + day, on an idle server. The loop now parks on an event when there are no subscribers and is + woken by the first one, so an idle instance does no polling work at all; while subscribed it + waits on a single timer that still returns immediately on shutdown. Connecting a client + remains prompt — the first subscriber unparks the loop rather than waiting out the poll + interval. Thanks @DevNexsler. (#7694) + +- **A long-running turn no longer replays a stale token count after a reload.** The run-journal + recorded live metering frames, so reattaching to a stream — or reloading a tab mid-turn — + could replay a snapshot from earlier in the same run and briefly show token/TPS figures that + had already been superseded. Metering is now live-only and never journaled, and replayed + frames no longer carry an event id that could advance the client's resume cursor past real + content. Reattach and reload now show the current numbers for the turn in progress. Thanks + @laitekin. (#7291) + +- **Steering a conversation works again after the context is compressed.** When compression + rotated `agent.session_id`, a steer could no longer find the active worker: it was either + silently dropped or accepted and never delivered. Steers now resolve the owning worker + directly rather than relying on a cache lookup that compression invalidates, and every + terminal exit — normal completion, returned error, raised exception and self-heal — passes + through one idempotent settle boundary, so guidance can no longer be stranded between the + final drain and teardown. Two related defects are fixed alongside it: **Stop now actually + stops** a run that had already passed preflight (previously the worker could consult a + removed registry entry, miss the cancellation, and continue to completion), and a live + `finalizing` run is no longer reported as `stream_dead`, which had caused the client to tear + down its state while the stream was still alive. Compressed sessions from the CLI, TUI, + Desktop and ACP now resume correctly, and installations running an older Agent keep their + existing sidecar recovery instead of being left unresumable. Thanks @ruizanthony. (#7546) + +- **Slash-command autocomplete stops offering commands the WebUI cannot run.** The composer's + `/` menu announced all 51 registered commands, but many are CLI-only — picking one produced + a command that went nowhere. The menu now announces only the WebUI-dispatchable subset (16), + so the difference shows up where it matters: typing a prefix like `/a` or `/re` no longer + fills the list with dead options. Filtering is confined to the suggestion list — manually + typing a CLI-only command behaves exactly as before, and plugin commands plus the native + `/moa`, `/sessions`, `/resume` and `/pet` remain available. Thanks @webtecnica. + - **Concurrent background completions stop burning the async-delegation delivery budget.** When several delegated runs finished at once and idle-woke the same session together, each wakeup consumed a delivery attempt before discovering the others, so the budget could be @@ -46,6 +294,7 @@ ### Changed +- **Two flaky tests are stable again.** Three `get_available_models()` cache-metadata tests raced the 4-second live-rebuild budget on a loaded CI host and could time out onto the stale-cache path. They now pin the budget to `0`, which is the documented setting for the legacy unbounded synchronous rebuild, so they still exercise the real rebuild (#7735). A source-text oracle that string-matched `delete_cli_session`'s source to prove it opens a writable connection is removed. The behavioral test in `test_issue1494_state_db_fd_leak.py` still guards that contract: it fails with `attempt to write a readonly database` if the delete path is ever switched to a read-only connection (#7726). Test-only; no runtime change. Thanks @webtecnica. (#7744, #7742) - **The chat composer grows natively instead of being resized by JavaScript on every keystroke.** `autoResize()` measured `scrollHeight` and wrote `style.height` on each input event — a forced synchronous reflow on the most-typed-in surface in the app. Browsers that support CSS `field-sizing: content` (Chromium today; also Firefox 152 and Safari 26.2) now own the geometry directly, gated on `CSS.supports()`, and the existing JavaScript path is untouched for every other engine. Measured behaviour is identical across both paths: 44px resting height, no jump when the first character is typed or the last deleted, growth to the 200px ceiling, then internal scrolling. Because `field-sizing` deliberately includes placeholder text in content sizing, `:placeholder-shown` pins fixed sizing while the composer is empty so a long placeholder can't inflate it. Thanks @starship-s. (#6760, #5514) ### Fixed diff --git a/README.md b/README.md index 7e73b9f1..76429aea 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,7 @@ Full list of environment variables: | `HERMES_HOME` | Windows: `%LOCALAPPDATA%\hermes`; POSIX: `~/.hermes` | Base directory for Hermes state (affects all paths) | | `HERMES_CONFIG_PATH` | `$HERMES_HOME/config.yaml` | Path to Hermes config file | | `HERMES_WEBUI_SERVER_CWD` | *(unset)* | Working directory for the server process. Defaults to the agent dir; point it at a writable workspace when the agent dir is read-only so fallback relative writes land somewhere writable | +| `HERMES_WEBUI_VISIBLE_SESSION_LIMIT` | `20` | Size of the sidebar's interactive recency window (how many recent non-cron/webhook sessions are listed). Also bounds how many delegated subagent children can nest at once, since a child only renders when its row wins a slot in the window — raise it for wide fan-outs. Non-integer or non-positive values fall back to the default. Values above 200 are clamped. Resolved before profile init, so a profile `.env` cannot override it | | `HERMES_WEBUI_AGENT_CACHE_MAX` | `25` | Max live agent instances kept warm in the in-memory LRU. Each pins a full conversation transcript, so this is the dominant lever on resident memory — lower it on installs with many long sessions to cap RAM (at the cost of more cold reloads) | | `HERMES_WEBUI_SESSIONS_MAX` | `100` | Legacy operator override for the max compact `Session` objects held in the in-memory LRU. Prefer the `webui.sessions_cache_max` key in `config.yaml` (which takes precedence); this env var remains a fallback. Bounds resident memory so long-running installs cannot accumulate every session ever touched and eventually crash (#4765/#2233/#4633). Eviction only ever drops clean, persisted, non-active sessions; an evicted session lazily reloads from its JSON sidecar on next access | diff --git a/TESTING.md b/TESTING.md index af111974..f96a4f88 100644 --- a/TESTING.md +++ b/TESTING.md @@ -154,12 +154,35 @@ HISTORICAL_HYDRATION_TEST_BITE=break-tool-link \ python tests/browser_historical_transcript_hydration.py ``` -The dedicated `Conversation lifecycle (informational)` workflow runs the current -proof rows (`normal`, `terminal-error`, and `historical-transcript-hydration`) and -stays non-blocking while the public -matrix expands to additional behavior rows. The maintainer's private QA harness -remains broader; later public slices will add session switching, reconnect/replay, -cancellation, compression, and recovery. +The dedicated `Conversation lifecycle (informational)` workflow keeps the existing +proof rows (`normal`, `terminal-error`, and `historical-transcript-hydration`) +non-blocking while the public matrix expands. The Chromium +`reconnect-scene-redraw` row does **not** allow failures: it exercises the real +`loadSession` reconnect path and fails the workflow on a regression. Required +merge checks remain a maintainer-controlled repository setting. +The maintainer's private QA harness remains broader; later public slices will +add cancellation, compression, and recovery coverage. + +### Active-session reconnect redraw gate + +`tests/browser_reconnect_scene_redraw.py` is an opt-in deterministic browser +gate for reconnecting to a running session with a large Anchor activity scene. +It uses an isolated temporary server/home and fixture SSE events—no agent, +provider credentials, or production state. By default it checks Chromium and +WebKit, desktop and 390px viewports, and all three activity display modes: + +```bash +pip install playwright +python -m playwright install chromium webkit +python tests/browser_reconnect_scene_redraw.py +``` + +`TOOL_COUNT`, `BROWSERS`, and `MODES` narrow the matrix. `MEASURE_BASELINE=1` +records an unoptimized baseline without enforcing the redraw ceiling; +`METRICS_FILE` saves JSON metrics and `SCREENSHOT_DIR` saves generated-fixture +screenshots. This gate tests page reconstruction, journal-cursor resume, and +subsequent fixture SSE updates. It does not test a real provider/network or PWA +service-worker cache behavior. ### Streaming reader intent diff --git a/api/agent_sessions.py b/api/agent_sessions.py index 97086795..ed3ab538 100644 --- a/api/agent_sessions.py +++ b/api/agent_sessions.py @@ -1,8 +1,12 @@ """Shared helpers for reading Hermes Agent sessions from state.db.""" +import json import logging +import os import sqlite3 +import sys from contextlib import closing -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath +from urllib.parse import quote, quote_from_bytes logger = logging.getLogger(__name__) @@ -18,36 +22,71 @@ logger = logging.getLogger(__name__) _SOURCE_COLUMN_WARNED_DB_PATHS: set[str] = set() +def state_db_file_uri(db_path, platform: str | None = None) -> str: + """Build the ``file:`` URI (no query string) for an absolute ``state.db`` path. + + ``Path.as_uri()`` is not usable here: for a UNC share it emits + ``file://server/share/state.db`` and SQLite rejects any non-local URI + authority unless compiled with ``SQLITE_ALLOW_URI_AUTHORITY`` (the + CPython 3.11-3.13 Windows builds are not). SQLite instead accepts the + *empty*-authority spelling ``file:////server/share/state.db``, whose path + ``//server/share/...`` the Windows VFS opens as the UNC name. Drive-letter + paths keep the documented ``file:///C:/...`` form and POSIX paths are + unchanged. Only ``/`` survives unescaped, so ``?`` and ``#`` in path + components cannot leak into the query string. + + ``platform`` defaults to the running interpreter; tests pass it explicitly + so the Windows shapes are checked from any host. + """ + platform = platform or sys.platform + if platform == "win32": + win = PureWindowsPath(str(db_path)) + drive = win.drive + if drive.startswith("\\\\?\\"): + # Extended-length prefix: "\\?\C:" or "\\?\UNC\server\share". + drive = drive[4:] + if drive.upper().startswith("UNC\\"): + drive = "\\\\" + drive[4:] + parts = win.parts[1:] # drop the anchor, keep the path components + if drive.startswith("\\\\"): + host_share = drive[2:].replace("\\", "/") + posix_path = "//" + "/".join((host_share, *parts)) + else: + posix_path = "/" + "/".join((drive, *parts)) + # ``:`` stays literal so the drive letter keeps SQLite's documented + # ``file:///C:/...`` shape (``Path.as_uri()`` leaves it unescaped too). + return "file://" + quote(posix_path, safe="/:") + posix_path = PurePosixPath(str(db_path)).as_posix() + # Quote the filesystem BYTES, not the str: a POSIX path component that is + # not valid UTF-8 is carried in the str as ``surrogateescape`` code points, + # which ``quote(str)`` rejects with UnicodeEncodeError (the caller then + # treats the db as unreadable and every agent-backed session vanishes). + # ``Path.as_uri()`` — what master used — percent-encodes os.fsencode() + # bytes; this keeps that behavior. + return "file://" + quote_from_bytes(os.fsencode(posix_path), safe="/") + + +def state_db_readonly_uri(db_path, platform: str | None = None) -> str: + """Strict read-only (``mode=ro``) form of :func:`state_db_file_uri`.""" + return state_db_file_uri(db_path, platform=platform) + "?mode=ro" + + def open_state_db_readonly(db_path: Path, log: logging.Logger | None = None) -> sqlite3.Connection: """Open the live agent ``state.db`` read-only for a pure-read projection. Same rationale as the session-listing path (#5455): a write-capable handle on the multi-GB, WAL ``state.db`` while the agent streams into it adds needless checkpoint/lock surface. The read-only ``file:...?mode=ro`` URI - avoids that. Falls back to a writable connection (and warns) if the - read-only open fails, so callers never lose data on exotic filesystems. + avoids that. Read failures propagate; a reader never upgrades to a writer. The caller must ensure ``db_path`` exists — this raises ``FileNotFoundError`` - for a missing path rather than letting the writable fallback below create an - empty, writable ``state.db`` there (a ghost DB in the agent's HOME). The - fallback is only for an *existing* DB whose read-only open fails on an exotic - filesystem, so a real read never loses data. + for a missing path rather than creating a ghost database. Callers own the returned connection (wrap it in ``contextlib.closing``). """ - log = log or logger if not db_path.exists(): raise FileNotFoundError(f"agent state.db not found: {db_path}") - read_only_uri = f"{db_path.resolve().as_uri()}?mode=ro" - try: - return sqlite3.connect(read_only_uri, uri=True) - except sqlite3.Error as exc: - log.warning( - "agent state.db read-only open failed for %s; falling back to writable connection: %s", - db_path, - exc, - ) - return sqlite3.connect(str(db_path)) + return sqlite3.connect(state_db_readonly_uri(db_path.resolve()), uri=True) MESSAGING_SOURCES = { @@ -65,6 +104,15 @@ MESSAGING_SOURCES = { CLI_MIN_UNTITLED_MESSAGE_COUNT = 6 CLI_MIN_UNTITLED_USER_MESSAGE_COUNT = 2 +# Sub-second scheduling/write-order races during a compression/cli_close +# handoff can persist the continuation row with ``started_at`` a few +# milliseconds BEFORE the parent's ``ended_at`` lands (#6931). The tolerance +# below lets those rows still classify as the next segment; the fork / +# model_config branch-marker / cross-source / end_reason guards in +# ``_is_continuation_session`` already rule out unrelated rows, so a small +# window cannot collapse genuinely concurrently started children. +CONTINUATION_STARTED_AT_TOLERANCE_SECONDS = 2.0 + SOURCE_LABELS = { 'acp': 'ACP', 'api_server': 'API', @@ -318,6 +366,30 @@ def is_cli_session_row_visible(row: dict) -> bool: return _count_user_turns(row) >= CLI_MIN_UNTITLED_USER_MESSAGE_COUNT +def _branch_markers(row: dict | None) -> tuple[str | None, str | None]: + """Return ``(_branched_from, _delegate_from)`` from a row's ``model_config``. + + Hermes Agent stamps explicit branches and delegate/subagent runs in the + ``model_config`` JSON column (``_branched_from`` / ``_delegate_from``); + ``session_source='fork'`` alone only covers WebUI-created forks. Missing, + empty, or unparsable ``model_config`` degrades to ``(None, None)``. + """ + if not row: + return None, None + raw = row.get('model_config') + if isinstance(raw, dict): + config = raw + elif isinstance(raw, str) and raw.strip(): + try: + parsed = json.loads(raw) + except (TypeError, ValueError): + parsed = None + config = parsed if isinstance(parsed, dict) else {} + else: + config = {} + return config.get('_branched_from'), config.get('_delegate_from') + + def _is_continuation_session(parent: dict | None, child: dict | None) -> bool: """Return True when ``child`` is the next segment of the same conversation. @@ -325,17 +397,36 @@ def _is_continuation_session(parent: dict | None, child: dict | None) -> bool: by ``hermes -c`` also records a new child session; for sidebar projection it should continue the same visible conversation rather than becoming a separate child-session row. Plain parent/child links that started before the - parent's ended boundary remain child sessions. + parent's ended boundary remain child sessions. A small scheduling/write-order + overlap between the handoff boundary timestamps is tolerated (#6931). Do not collapse lineage across raw sources. A WebUI session that continues from a Telegram/CLI/etc. parent must remain visible as its own surface-owned conversation; otherwise the tip inherits the root's title/source metadata and can disappear under messaging/sidebar policies. + + Explicit branches/delegates never continue: a child whose ``model_config`` + ``_branched_from`` / ``_delegate_from`` points at this parent is a fork of + the conversation and must stay visible. Beyond the bounded tolerance window + there is no reliable continuation signal — titles are user-controlled and + non-unique, so they are never used to widen the window (#7021 re-gate). """ if not parent or not child: return False if str(child.get('session_source') or '').strip().lower() == 'fork': return False + # Real Agent branches/delegates are marked in model_config (not + # session_source, which only WebUI-created forks carry). The marker must + # reference THIS parent: compression continuations inherit the rotated + # agent's model_config verbatim, so presence alone (a delegate's + # continuation still carries the delegate's own ``_delegate_from``) would + # misclassify real continuations. + child_branched_from, child_delegate_from = _branch_markers(child) + parent_id = parent.get('id') + if parent_id and ( + child_branched_from == parent_id or child_delegate_from == parent_id + ): + return False parent_source = str(parent.get('source') or '').strip().lower() child_source = str(child.get('source') or '').strip().lower() if parent_source and child_source and parent_source != child_source: @@ -349,9 +440,16 @@ def _is_continuation_session(parent: dict | None, child: dict | None) -> bool: # continuations when no boundary timestamp is available. return True try: - return float(child.get('started_at') or 0) >= float(ended_at) + child_started = float(child.get('started_at') or 0) + parent_ended = float(ended_at) except (TypeError, ValueError): return False + if child_started >= parent_ended - CONTINUATION_STARTED_AT_TOLERANCE_SECONDS: + return True + # Beyond the bounded early-side window the child is a genuine concurrent + # session — an exact title match is not evidence (titles are user-controlled + # and non-unique, so it cannot extend the tolerance). + return False def _continuation_root_id(rows_by_id: dict[str, dict], session_id: str | None) -> str | None: @@ -549,21 +647,7 @@ def read_importable_agent_session_rows( return [] log = log or logger - # Open read-only for this projection/listing path: it is a pure read, and - # holding a write-capable handle on the live (multi-GB, WAL) state.db while - # the agent streams into it adds needless checkpoint/lock surface (#5455). - # The defensive index self-heal below still runs, but through a separate - # short-lived writable connection on the rare missing-index path only. - read_only_uri = f"{db_path.resolve().as_uri()}?mode=ro" - try: - conn = sqlite3.connect(read_only_uri, uri=True) - except sqlite3.Error as exc: - log.warning( - "agent session listing read-only open failed for %s; falling back to writable connection: %s", - db_path, - exc, - ) - conn = sqlite3.connect(str(db_path)) + conn = open_state_db_readonly(db_path, log=log) with closing(conn): conn.row_factory = sqlite3.Row cur = conn.cursor() @@ -588,6 +672,7 @@ def read_importable_agent_session_rows( parent_expr = _optional_col('parent_session_id', session_cols) session_source_expr = _optional_col('session_source', session_cols) + model_config_expr = _optional_col('model_config', session_cols) ended_expr = _optional_col('ended_at', session_cols) end_reason_expr = _optional_col('end_reason', session_cols) user_id_expr = _optional_col('user_id', session_cols) @@ -613,11 +698,7 @@ def read_importable_agent_session_rows( use_messages_join = messages_has_session_id count_col = 'id' if 'id' in message_cols else 'session_id' - # Defensive index prime (#3887). The normal candidate-ordering shape uses - # the agent's standard ``idx_messages_session ON messages(session_id, - # timestamp)`` index; without it, large cron-only scans degrade badly. - # Writable dbs self-heal by recreating the index. Read-only or locked dbs - # fall back to the pre-aggregated cron-only path below instead of failing. + # Index creation belongs to explicit drained maintenance, never a read. messages_index_present = False if messages_has_session_id and messages_has_timestamp: try: @@ -625,21 +706,7 @@ def read_importable_agent_session_rows( messages_index_present = any(str(row[1]) == "idx_messages_session" for row in cur.fetchall()) except sqlite3.Error: messages_index_present = False - if not messages_index_present: - # Self-heal via a separate writable connection so the common - # (index-present) path keeps its read-only handle. On a truly - # read-only/locked db this fails and we degrade to the - # pre-aggregated cron-only path below, exactly as before. - try: - with closing(sqlite3.connect(str(db_path))) as _heal: - _heal.execute( - "CREATE INDEX IF NOT EXISTS idx_messages_session " - "ON messages(session_id, timestamp)" - ) - _heal.commit() - messages_index_present = True - except sqlite3.Error: - pass # read-only db / locked / older schema — degrade gracefully + if use_messages_join: actual_count_expr = f"COUNT(m.{count_col})" @@ -665,7 +732,6 @@ def read_importable_agent_session_rows( where_clauses = ["s.source IS NOT NULL"] params: list[object] = [] - included = () if include_sources: included = tuple(str(source) for source in include_sources if source) if included: @@ -679,10 +745,16 @@ def read_importable_agent_session_rows( where_clauses.append(f"s.source NOT IN ({placeholders})") params.extend(excluded) + # Without ``idx_messages_session`` the correlated ``MAX(mx.timestamp)`` + # candidate ordering rescans ``messages`` once per session (seconds on + # a few thousand sessions). Every missing-index projection — default + # sidebar, gateway watcher, cron/webhook/kanban views — orders through + # one read-only pre-aggregation pass instead; the listing never creates + # the index itself (that is drained maintenance, see + # ``scripts/ensure_state_db_read_indexes.py``). use_preaggregated_candidate_order = ( use_messages_join and messages_has_timestamp - and included == ("cron",) and not messages_index_present ) if use_preaggregated_candidate_order: @@ -709,6 +781,7 @@ def read_importable_agent_session_rows( SELECT s.id, s.title, s.model, s.message_count, s.started_at, s.source, {session_source_expr}, + {model_config_expr}, {user_id_expr}, {chat_id_expr}, {chat_type_expr}, @@ -791,6 +864,11 @@ def read_importable_agent_session_rows( params, ) projected = _project_agent_session_rows([dict(row) for row in cur.fetchall()]) + # model_config is selected only so the continuation classifier can + # read the real _branched_from/_delegate_from markers; it is internal + # agent state and must not leak into the /api/sessions response. + for row in projected: + row.pop('model_config', None) projected = [_with_normalized_source(row) for row in projected] projected = [row for row in projected if is_cli_session_row_visible(row)] if limit is None: @@ -893,6 +971,7 @@ def read_session_lineage_report(db_path: Path, session_id: str | None, max_hops: source_expr = _optional_col('source', session_cols) session_source_expr = _optional_col('session_source', session_cols) + model_config_expr = _optional_col('model_config', session_cols) title_expr = _optional_col('title', session_cols) started_expr = _optional_col('started_at', session_cols, '0') ended_expr = _optional_col('ended_at', session_cols) @@ -907,6 +986,7 @@ def read_session_lineage_report(db_path: Path, session_id: str | None, max_hops: SELECT s.id, {source_expr}, {session_source_expr}, + {model_config_expr}, {title_expr}, {started_expr}, {parent_expr}, @@ -953,6 +1033,7 @@ def read_session_lineage_report(db_path: Path, session_id: str | None, max_hops: SELECT s.id, {source_expr}, {session_source_expr}, + {model_config_expr}, {title_expr}, {started_expr}, {parent_expr}, @@ -1030,6 +1111,7 @@ def read_session_lineage_metadata(db_path: Path, session_ids: list[str] | set[st return {} session_source_expr = _optional_col('session_source', session_cols) source_expr = _optional_col('source', session_cols) + model_config_expr = _optional_col('model_config', session_cols) message_count_expr = _optional_col('message_count', session_cols, '0') # Scoped fetch via PRIMARY KEY + idx_sessions_parent rather than a # full table scan. The sessions table grows unbounded over time @@ -1066,7 +1148,7 @@ def read_session_lineage_metadata(db_path: Path, session_ids: list[str] | set[st placeholders = ','.join('?' * len(chunk)) cur.execute( f""" - SELECT s.id, {source_expr}, {session_source_expr}, s.title, s.started_at, s.parent_session_id, s.ended_at, s.end_reason, {message_count_expr} + SELECT s.id, {source_expr}, {session_source_expr}, {model_config_expr}, s.title, s.started_at, s.parent_session_id, s.ended_at, s.end_reason, {message_count_expr} FROM sessions s WHERE s.id IN ({placeholders}) """, @@ -1095,7 +1177,7 @@ def read_session_lineage_metadata(db_path: Path, session_ids: list[str] | set[st placeholders = ','.join('?' * len(chunk)) cur.execute( f""" - SELECT s.id, {source_expr}, {session_source_expr}, s.title, s.started_at, s.parent_session_id, s.ended_at, s.end_reason, {message_count_expr} + SELECT s.id, {source_expr}, {session_source_expr}, {model_config_expr}, s.title, s.started_at, s.parent_session_id, s.ended_at, s.end_reason, {message_count_expr} FROM sessions s WHERE s.parent_session_id IN ({placeholders}) """, diff --git a/api/auth.py b/api/auth.py index 846c9945..c61ee669 100644 --- a/api/auth.py +++ b/api/auth.py @@ -17,6 +17,7 @@ import time from pathlib import Path from api.config import STATE_DIR, get_config, load_settings +from api.helpers import request_declares_body logger = logging.getLogger(__name__) @@ -1207,6 +1208,42 @@ def check_auth(handler, parsed) -> bool: return False +def check_auth_or_close(handler, parsed) -> bool: + """Check auth; when rejected, close so an unread body can't poison HTTP/1.1 reuse. + + The flag is armed BEFORE check_auth() writes its 401/302, so end_headers() + can advertise ``Connection: close``; success restores the prior flag so an + authenticated request keeps its keep-alive. + + Armed ONLY when the request's framing declares a body still queued in + ``rfile`` (see ``request_declares_body()``), which is the same rule the other + reject-before-read sites apply through ``arm_connection_close_if_body_pending()``. + Arming unconditionally was the mirror image of the over-close this PR already + fixed on the sidecar path: a body-less POST that failed auth answered + ``401`` WITH ``Connection: close`` and dropped the client's pipelined + follow-up, killing a healthy keep-alive connection for no framing reason. + Verified on the wire, pipelined down one socket against the production + handler: ``POST /api/session/new`` with no ``Content-Length`` (and with + ``Content-Length: 0``) answered ``401`` + ``Connection: close`` and the + following ``GET /api/auth/status`` was never served. The helper cannot be + swapped in directly here because the arming has to happen before + ``check_auth()`` writes its response and be undone if it succeeds. + + ``close_connection`` only exists once BaseHTTPRequestHandler has parsed a + request line, so partially-built handler stubs may not have it at all. Read + it defensively and only arm/restore when it was really there: inventing the + attribute on a stub would leave a bogus keep-alive verdict behind. + """ + if not hasattr(handler, 'close_connection') or not request_declares_body(handler): + return check_auth(handler, parsed) + prior_close = handler.close_connection + handler.close_connection = True + if check_auth(handler, parsed): + handler.close_connection = prior_close + return True + return False + + def _is_loopback(addr: str) -> bool: """Return True if *addr* is a loopback address (127.x.x.x, ::1, or ::ffff:127.x.x.x).""" import ipaddress as _ipaddress diff --git a/api/background_process.py b/api/background_process.py index 2a3c581f..eea15ebb 100644 --- a/api/background_process.py +++ b/api/background_process.py @@ -1020,6 +1020,29 @@ def _record_async_delegation_accepted( ) +def _wakeup_refusal_is_transient(resp: dict | None, status: int) -> bool: + """True when the target refused the wake-up for a reason it will recover from. + + Such refusals are not delivery failures of the completion itself, so they + must not consume its bounded durable delivery budget. Otherwise a burst of + refusals (e.g. ``agent_runtime_stale`` answers while the Agent runtime is + being replaced) terminally drops a completion whose origin session is alive: + + * any payload that explicitly says ``retryable: True`` (stale runtime); + * the paused-wakeup contract (``process_wakeup_paused``); + * a session busy with another turn (``active_stream_id`` on a 409). + """ + if not isinstance(resp, dict): + return False + if resp.get("retryable") is True: + return True + if status != 409: + return False + if resp.get("error") == "process_wakeup_paused": + return True + return bool(resp.get("active_stream_id")) + + def _start_async_delegation_wakeup_turn( session_id: str, wakeup_prompt: str, @@ -1059,7 +1082,11 @@ def _start_async_delegation_wakeup_turn( ) return - release_async_delegation_delivery(evt, claim) + transient = _wakeup_refusal_is_transient(resp, status) + if transient: + release_async_delegation_delivery(evt, claim, retryable=True) + else: + release_async_delegation_delivery(evt, claim) _retry_unclaimed_async_delegation_event( process_registry, evt, keep_legacy_retrying=True ) @@ -1068,6 +1095,14 @@ def _start_async_delegation_wakeup_turn( "async delegation wakeup paused for session %s; delivery remains retryable", session_id, ) + elif transient: + logger.info( + "async delegation wakeup refused transiently for session %s " + "(status=%s err=%r); attempt refunded, durable retry scheduled", + session_id, + status, + (resp or {}).get("error"), + ) else: logger.debug( "async delegation wakeup not accepted for session %s: " @@ -1848,7 +1883,22 @@ def recover_processes_for_webui(process_registry=None, get_session_fn=None) -> i try: proc_session = process_registry.get(process_id) session_key = str(getattr(proc_session, "session_key", "") or "") - if not session_key or get_session_fn(session_key, metadata_only=True) is None: + if not session_key: + continue + # The session resolver (``api.models._resolve_session_once``) + # raises ``KeyError(sid)`` for a missing session — that is its + # documented contract for the "owner gone" outcome, not a + # fault. Treat it identically to a ``None`` return: skip the + # process without logging, and let a neighbouring live + # process still rebind. The narrow ``except KeyError`` lives + # INSIDE the outer try so an unrelated ``KeyError`` from + # ``process_registry.get()`` (or any routing-index work) still + # surfaces in the existing warning path. + try: + resolved = get_session_fn(session_key, metadata_only=True) + except KeyError: + continue + if resolved is None: continue except Exception: logger.warning( diff --git a/api/compression_continuation.py b/api/compression_continuation.py new file mode 100644 index 00000000..64bc7dca --- /dev/null +++ b/api/compression_continuation.py @@ -0,0 +1,80 @@ +"""Read-only routing hints for externally compressed WebUI sessions. + +SQLite owns compression lineage; sidecar snapshot flags may predate a +Desktop/CLI rotation. Never reopen a sealed parent or mutate Agent state here. +""" +import inspect +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def _is_local_interactive_persisted_source(source): + return isinstance(source, str) and source in { + "webui", "tui", "cli", "desktop", "acp", + } + + +def durable_compression_continuation(session): + """Return (sealed, resumable tip), without making a recovery write. + + A known sealed parent without a safe tip stays sealed (no sidecar fallback). + Older Agent installations without this read API retain legacy behavior. + """ + from api.profiles import _PROFILE_ID_RE, _resolve_profile_home_for_name + + sid = str(getattr(session, "session_id", "") or "") + profile = str(getattr(session, "profile", None) or "default") + if not sid or (profile != "default" and not _PROFILE_ID_RE.fullmatch(profile)): + return False, None + db = None + sealed = False + try: + from hermes_state import SessionDB + + path = Path(_resolve_profile_home_for_name(profile)) / "state.db" + if not path.is_file(): + return False, None + db = SessionDB(path, read_only=True) + # Establish the complete read API before accepting SQLite authority. + # Old Agents must retain legacy sidecar recovery, not a sealed null tip. + for name in ('get_session', 'get_compression_tip'): + method = getattr(db, name, None) + if not callable(method): + return False, None + try: + inspect.signature(method).bind(sid) + except (TypeError, ValueError): + return False, None + parent = db.get_session(sid) + if not parent or parent.get("end_reason") != "compression": + return False, None + sealed = True + tip = db.get_compression_tip(sid) + if not tip or tip == sid: + return True, None + child = db.get_session(tip) + if not child or not _is_local_interactive_persisted_source(child.get("source")): + return True, None + for row in (parent, child): + if row.get("profile_name") not in (None, "", profile): + return True, None + # Only the observed automatic idle closure is resumable here. Explicit + # resets/closures and unknown future reasons must not become redirects. + reason = child.get("end_reason") + if reason not in (None, "", "idle_timeout"): + return True, None + if child.get("ended_at") is not None and reason != "idle_timeout": + return True, None + if db.get_compression_tip(sid) != tip: + return True, None + return True, tip + except (ImportError, AttributeError, TypeError): + return sealed, None + except Exception: + logger.debug("Could not resolve durable compression continuation", exc_info=True) + return sealed, None + finally: + if db is not None: + db.close() diff --git a/api/config.py b/api/config.py index 05094e48..1650865d 100644 --- a/api/config.py +++ b/api/config.py @@ -68,6 +68,21 @@ def _env_int(name: str, default: int, *, minimum: int = 1) -> int: return default return value if value >= minimum else default + +def _env_int_clamped(name: str, default: int, *, minimum: int = 1, maximum: int) -> int: + """Like ``_env_int``, then clamp a valid override to ``maximum``.""" + value = _env_int(name, default, minimum=minimum) + if not str(os.getenv(name) or "").strip(): + return value + return min(value, maximum) + + +# Sidebar recency window. Resolved here, before profile init, so a profile +# .env cannot override a server-wide resource bound. Clamped at 200. +CLI_VISIBLE_SESSION_LIMIT = _env_int_clamped( + "HERMES_WEBUI_VISIBLE_SESSION_LIMIT", 20, maximum=200, +) + # ── TLS/HTTPS config (optional, env-overridable) ──────────────────────────── TLS_CERT = os.getenv("HERMES_WEBUI_TLS_CERT", "").strip() or None TLS_KEY = os.getenv("HERMES_WEBUI_TLS_KEY", "").strip() or None @@ -1914,28 +1929,44 @@ _PROVIDER_MODELS = { {"id": "big-pickle", "label": "Big Pickle"}, ], # OpenCode Go — flat-rate models via opencode.ai/go ($10/month). - # Synced 2026-07-08 from the public Go docs and documented models endpoint. - # Keep preview/free-only Zen models out of this Go picker snapshot. + # Fallback only: the live Hermes CLI catalog (Go-specific + # /zen/go/v1/models probe, core v0.20.5+) leads (#1240, #5311). + # Mirrors Hermes core's curated opencode-go list + # (hermes_cli/models_catalog_static.py, core main as of 2026-09-10). + # Core's 2026-09-09 sync dropped `ox-alpha-free` (Go relay delisted it: + # GET /zen/go/v1/models omits it, POST → 401) and added `glm-5.3-flash` + # and `muse-spark-1.3-contributor`. Core owns the sync duty against the + # live endpoint; WebUI mirrors. "opencode-go": [ - {"id": "minimax-m3", "label": "MiniMax M3"}, - {"id": "minimax-m2.7", "label": "MiniMax M2.7"}, - {"id": "minimax-m2.5", "label": "MiniMax M2.5"}, - {"id": "kimi-k2.7-code", "label": "Kimi K2.7 Code"}, - {"id": "kimi-k2.6", "label": "Kimi K2.6"}, - {"id": "kimi-k2.5", "label": "Kimi K2.5"}, - {"id": "glm-5.2", "label": "GLM-5.2"}, - {"id": "glm-5.1", "label": "GLM-5.1"}, - {"id": "glm-5", "label": "GLM-5"}, - {"id": "deepseek-v4-pro", "label": "DeepSeek V4 Pro"}, - {"id": "deepseek-v4-flash","label": "DeepSeek V4 Flash"}, - {"id": "qwen3.7-max", "label": "Qwen3.7 Max"}, - {"id": "qwen3.7-plus", "label": "Qwen3.7 Plus"}, - {"id": "qwen3.6-plus", "label": "Qwen3.6 Plus"}, - {"id": "qwen3.5-plus", "label": "Qwen3.5 Plus"}, - {"id": "mimo-v2-pro", "label": "MiMo V2 Pro"}, - {"id": "mimo-v2-omni", "label": "MiMo V2 Omni"}, - {"id": "mimo-v2.5-pro", "label": "MiMo V2.5 Pro"}, - {"id": "mimo-v2.5", "label": "MiMo V2.5"}, + {"id": "kimi-k3", "label": "Kimi K3"}, + {"id": "kimi-k2.7-code", "label": "Kimi K2.7 Code"}, + {"id": "kimi-k2.6", "label": "Kimi K2.6"}, + {"id": "kimi-k2.5", "label": "Kimi K2.5"}, + {"id": "gpt-5.6-luna", "label": "GPT 5.6 Luna"}, + {"id": "grok-4.5", "label": "Grok 4.5"}, + {"id": "glm-5.3", "label": "GLM-5.3"}, + {"id": "glm-5.3-flash", "label": "GLM-5.3 Flash"}, + {"id": "glm-5.2", "label": "GLM-5.2"}, + {"id": "glm-5.1", "label": "GLM-5.1"}, + {"id": "glm-5", "label": "GLM-5"}, + {"id": "mimo-v2.5-pro", "label": "MiMo V2.5 Pro"}, + {"id": "mimo-v2.5", "label": "MiMo V2.5"}, + {"id": "mimo-v2-pro", "label": "MiMo V2 Pro"}, + {"id": "mimo-v2-omni", "label": "MiMo V2 Omni"}, + {"id": "minimax-m3", "label": "MiniMax M3"}, + {"id": "minimax-m2.7", "label": "MiniMax M2.7"}, + {"id": "minimax-m2.5", "label": "MiniMax M2.5"}, + {"id": "deepseek-v4-pro", "label": "DeepSeek V4 Pro"}, + {"id": "deepseek-v4-flash", "label": "DeepSeek V4 Flash"}, + {"id": "qwen3.8-max", "label": "Qwen3.8 Max"}, + {"id": "qwen3.7-max", "label": "Qwen3.7 Max"}, + {"id": "qwen3.7-plus", "label": "Qwen3.7 Plus"}, + {"id": "qwen3.6-plus", "label": "Qwen3.6 Plus"}, + {"id": "qwen3.5-plus", "label": "Qwen3.5 Plus"}, + {"id": "hy3", "label": "HY3"}, + {"id": "hy3-preview", "label": "HY3 Preview"}, + {"id": "muse-spark-1.2-contributor", "label": "Muse Spark 1.2 Contributor"}, + {"id": "muse-spark-1.3-contributor", "label": "Muse Spark 1.3 Contributor"}, ], # 'gemini' is the hermes_cli provider ID for Google AI Studio # Model IDs are bare — sent directly to: @@ -6937,8 +6968,8 @@ def _configured_model_badges_from_static_catalog( for entry in configured_entries: provider = entry["provider"] model = entry["model"] - raw_candidates = [] - for candidate in (model, f"{provider}/{model}", f"@{provider}:{model}"): + raw_candidates: list[str] = [] + for candidate in (model, f"@{provider}:{model}"): if candidate and candidate not in raw_candidates: raw_candidates.append(candidate) @@ -8475,6 +8506,26 @@ def _read_live_provider_model_ids(provider_id: str) -> list[str]: return [] +def _hermes_cli_supports_opencode_go_live_catalog() -> bool: + """Whether the installed Agent has Go-specific model discovery. + + Hermes core versions before 0.20.5 route ``opencode-go`` through a + generic public catalog. That lookup can return a convincing non-empty + list containing models the Go relay rejects with 404, so absence or an + unparseable/prerelease version must fail closed to WebUI's static Go list. + """ + try: + import hermes_cli + + version = str(getattr(hermes_cli, "__version__", "")).strip() + except Exception: + return False + match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)", version) + if not match: + return False + return tuple(int(part) for part in match.groups()) >= (0, 20, 5) + + def _models_from_live_provider_ids(provider_id: str, live_ids: list[str]) -> list[dict]: """Convert Hermes CLI model ids into WebUI picker model entries.""" formatter = _format_ollama_label if provider_id in ("ollama", "ollama-cloud") else None @@ -8683,10 +8734,9 @@ def get_available_models(*, prefer_cache: bool = False, force_refresh: bool = Fa for entry in configured_entries: provider = entry["provider"] model = entry["model"] - raw_candidates = [] + raw_candidates: list[str] = [] for candidate in ( model, - f"{provider}/{model}", f"@{provider}:{model}", ): if candidate and candidate not in raw_candidates: @@ -9872,12 +9922,14 @@ def get_available_models(*, prefer_cache: bool = False, force_refresh: bool = Fa if not raw_models: if pid == "moa": raw_models = _moa_preset_models_from_config(cfg) - elif pid == "opencode-go": - # Skip live /v1/models probe for OpenCode Go — it - # returns models from the public catalog that are - # not enabled on the Go tier, causing 404 when - # selected. Use the curated static list only. (#5311) - pass + elif ( + pid == "opencode-go" + and not _hermes_cli_supports_opencode_go_live_catalog() + ): + # Before core v0.20.5 this resolver returned the + # generic public catalog, including models that + # 404 on the Go tier. Use the curated Go fallback. + raw_models = [] else: raw_models = _models_from_live_provider_ids( pid, @@ -11532,7 +11584,13 @@ _SETTINGS_DEFAULTS = { "hidden_tabs": [], # sidebar tab panel names hidden by user (e.g. ["tasks","kanban"]); chat and settings are always visible "tab_order": [], # user-defined sidebar/rail tab order for reorderable tabs; chat/settings stay fixed "composer_control_order": [], # user-defined composer footer control order; invalid/duplicate keys are ignored - "language": "en", # UI locale code; must match a key in static/i18n.js LOCALES + # #7622 (round 3): language is intentionally absent from the defaults so + # `load_settings()` reports `None` for a fresh install. This lets the + # client distinguish "no preference" from an explicit saved choice, + # so a user who genuinely picked English (and has `language: "en"` + # persisted on disk) is no longer overridden by the browser hint on + # their first hydration. When a user picks a locale in the Settings + # modal the value is written here and the field is set explicitly. "bot_name": os.getenv( "HERMES_WEBUI_BOT_NAME", "Hermes" ), # display name for the assistant @@ -11772,6 +11830,16 @@ def load_settings() -> dict: settings["default_model_provider"] = str(model_cfg.get("provider")) except Exception: logger.debug("Failed to resolve default model provider for settings") + # #7622/#7730 (round 4): keep the tri-state signal explicit. `language` + # is intentionally absent from `_SETTINGS_DEFAULTS` (see above), so + # without this line a fresh install's returned dict would OMIT the key + # entirely and the API payload would carry no `language` field. Emit an + # explicit `None` (serialized as JSON `null`) so the client receives the + # three-way signal it trusts: `null` = no preference, "en" = explicitly + # saved English, any other code = explicitly chosen locale. Stored + # values (including a legacy English pick written before this change) + # win via the merge above and are never touched here. + settings.setdefault("language", None) return settings @@ -11779,6 +11847,15 @@ _SETTINGS_ALLOWED_KEYS = set(_SETTINGS_DEFAULTS.keys()) - { "password_hash", "default_model", "simplified_tool_calling", +} | { + # #7622 (round 3): `language` is intentionally absent from + # `_SETTINGS_DEFAULTS` so a fresh install returns `None` for + # `settings["language"]` and the client can distinguish "no + # preference" from an explicit saved choice. But the user + # MUST still be able to pick a locale in the Settings modal, + # so we add it back to the explicit allow-list here. The + # existing BCP-47 validation at save-time still applies. + "language", } _SETTINGS_ENUM_VALUES = { "send_key": {"enter", "ctrl+enter", "shift+enter"}, diff --git a/api/gateway_chat.py b/api/gateway_chat.py index 35ced6fd..c66175d5 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -153,6 +153,9 @@ _WEBUI_GATEWAY_BASE_URL_ENV = "HERMES_WEBUI_GATEWAY_BASE_URL" _WEBUI_GATEWAY_API_KEY_ENV = "HERMES_WEBUI_GATEWAY_API_KEY" _WEBUI_GATEWAY_USE_RUNS_API_ENV = "HERMES_WEBUI_GATEWAY_USE_RUNS_API" _GATEWAY_CHAT_BACKENDS = {"gateway", "api_server", "api-server"} +# Backend tag of the in-process WebUI runtime. Local workers register their +# active run with it; cache-only Steer only enqueues on this explicit value. +WEBUI_LOCAL_CHAT_BACKEND = "legacy" def _gateway_model_field(model: str | None) -> str: @@ -272,7 +275,7 @@ def webui_chat_backend_mode(config_data=None, environ: dict[str, str] | None = N ).strip().lower() if raw in _GATEWAY_CHAT_BACKENDS: return "gateway" - return "legacy" + return WEBUI_LOCAL_CHAT_BACKEND def webui_gateway_chat_enabled(config_data=None, environ: dict[str, str] | None = None) -> bool: @@ -738,8 +741,12 @@ def _run_gateway_runs_api_streaming( sse_event = "message" continue if payload_event == "run.completed": - from api.route_approvals import retire_gateway_pending_mirror - retire_gateway_pending_mirror(session_id, run_id=run_id) + from api.route_approvals import settle_gateway_pending_run + settle_gateway_pending_run( + session_id, + run_id, + reason="Gateway run completed before approval resolution", + ) if payload.get("error"): raise RuntimeError(str(payload["error"])) output = str(payload.get("output") or "") @@ -751,12 +758,20 @@ def _run_gateway_runs_api_streaming( sse_event = "message" continue if payload_event == "run.failed": - from api.route_approvals import retire_gateway_pending_mirror - retire_gateway_pending_mirror(session_id, run_id=run_id) + from api.route_approvals import settle_gateway_pending_run + settle_gateway_pending_run( + session_id, + run_id, + reason="Gateway run failed before approval resolution", + ) raise RuntimeError(str(payload.get("error") or "Gateway run failed")) if payload_event == "run.cancelled": - from api.route_approvals import retire_gateway_pending_mirror - retire_gateway_pending_mirror(session_id, run_id=run_id) + from api.route_approvals import settle_gateway_pending_run + settle_gateway_pending_run( + session_id, + run_id, + reason="Gateway run was cancelled before approval resolution", + ) put_gateway_event("cancel", {"message": "Cancelled by gateway"}) return None, usage reasoning_delta = _gateway_sse_reasoning_delta(payload) @@ -978,7 +993,7 @@ def _run_gateway_chat_streaming( except Exception: logger.debug("Failed to note gateway event_id %s for stream %s", event_id, stream_id, exc_info=True) try: - queue_item = (event, data, event_id) if event_id and hasattr(q, "subscribe_with_snapshot") else (event, data) + queue_item = (event, data, event_id) if hasattr(q, "subscribe_with_snapshot") else (event, data) q.put_nowait(queue_item) except Exception: logger.debug("Failed to put gateway event to queue") @@ -1464,10 +1479,14 @@ def _run_gateway_chat_streaming( mapped_run_id = str(_STREAM_RUN_IDS.get(stream_id) or "").strip() if mapped_run_id: try: - from api.route_approvals import retire_gateway_pending_mirror - retire_gateway_pending_mirror(session_id, run_id=mapped_run_id) + from api.route_approvals import settle_gateway_pending_run + settle_gateway_pending_run( + session_id, + mapped_run_id, + reason="Gateway run ended during teardown before approval resolution", + ) except Exception: - logger.debug("Failed to retire gateway pending mirrors during teardown", exc_info=True) + logger.debug("Failed to settle gateway pending approvals during teardown", exc_info=True) if s is not None: try: with _get_session_agent_lock(session_id): diff --git a/api/gateway_watcher.py b/api/gateway_watcher.py index b52f5b0c..69b1bbb6 100644 --- a/api/gateway_watcher.py +++ b/api/gateway_watcher.py @@ -203,7 +203,11 @@ class GatewayWatcher: ): self._subscribers: list[queue.Queue] = [] self._sub_lock = threading.Lock() + # Final removal invalidates any projection admitted by an earlier cohort. + self._subscriber_epoch = 0 self._stop_event = threading.Event() + # Wakes a poll loop parked because nobody is subscribed (subscribe/stop). + self._idle_wake = threading.Event() self._thread: threading.Thread | None = None self._hermes_home = Path(hermes_home).expanduser().resolve() if hermes_home else None self._state_db_path = ( @@ -244,6 +248,7 @@ class GatewayWatcher: def stop(self): """Stop the watcher thread.""" self._stop_event.set() + self._idle_wake.set() # unpark if waiting for the first subscriber # Wake up any subscribers with self._sub_lock: for q in self._subscribers: @@ -255,6 +260,11 @@ class GatewayWatcher: self._thread.join(timeout=3) self._thread = None + def _has_subscribers(self) -> bool: + """Return True when at least one SSE client is attached.""" + with self._sub_lock: + return bool(self._subscribers) + def subscribe(self) -> queue.Queue: """Subscribe to change events. Returns a queue.Queue. Events are dicts: {'type': 'sessions_changed', 'sessions': [...]} @@ -273,23 +283,37 @@ class GatewayWatcher: q.put_nowait(None) except Exception: logger.debug("Failed to send stop sentinel to late subscriber") + # Wake a poll loop parked with zero subscribers so the first SSE client + # gets a prompt initial projection instead of waiting out POLL_INTERVAL. + self._idle_wake.set() return q - def unsubscribe(self, q: queue.Queue): - """Remove a subscriber queue.""" - with self._sub_lock: - try: - self._subscribers.remove(q) - except ValueError: - pass + def _remove_subscriber_locked(self, q: queue.Queue) -> bool: + """Remove a known queue under _sub_lock; fence polls on the last removal.""" + try: + self._subscribers.remove(q) + except ValueError: + return False + if not self._subscribers: + self._subscriber_epoch += 1 + self._last_cheap_fp = '' + self._last_full_projection_at = None + return True - def _notify_subscribers(self, sessions: list): + def unsubscribe(self, q: queue.Queue): + """Remove a subscriber queue, invalidating projection on the last removal.""" + with self._sub_lock: + self._remove_subscriber_locked(q) + + def _notify_subscribers(self, sessions: list, *, epoch: int | None = None): """Push change event to all subscribers.""" event = { 'type': 'sessions_changed', 'sessions': sessions, } with self._sub_lock: + if epoch is not None and epoch != self._subscriber_epoch: + return # An old poll must not notify a new subscriber cohort. dead = [] for q in self._subscribers: try: @@ -299,10 +323,7 @@ class GatewayWatcher: except Exception: dead.append(q) for q in dead: - try: - self._subscribers.remove(q) - except ValueError: - pass + self._remove_subscriber_locked(q) # Send a None sentinel so the SSE handler unblocks, closes, # and lets the browser's EventSource auto-reconnect. try: @@ -317,6 +338,12 @@ class GatewayWatcher: protects projection fields (notably role-derived CLI visibility) that the agent's existing index cannot see. """ + with self._sub_lock: + has_subscribers = bool(self._subscribers) + admission_epoch = self._subscriber_epoch + if not has_subscribers: + # Final removal already invalidated the cache under the same lock. + return False db_path = self._state_db_path # A watcher may start before the agent has created state.db. Publishing an # empty first snapshot would make an already-rendered sidebar disappear; @@ -344,29 +371,48 @@ class GatewayWatcher: if sessions is None: return False current_hash = _snapshot_hash(sessions) - if cheap_fp is not None: - self._last_cheap_fp = cheap_fp - self._last_full_projection_at = current_time - - if current_hash != self._last_hash: - self._last_hash = current_hash - self._last_sessions = sessions - self._notify_subscribers(sessions) + with self._sub_lock: + if admission_epoch != self._subscriber_epoch or not self._subscribers: + return False # Never restore a cache invalidated during the DB read. + if cheap_fp is not None: + self._last_cheap_fp = cheap_fp + self._last_full_projection_at = current_time + if current_hash != self._last_hash: + changed = True + self._last_hash = current_hash + self._last_sessions = sessions + else: + changed = False + if changed: + self._notify_subscribers(sessions, epoch=admission_epoch) return True def _poll_loop(self): - """Main polling loop. Runs in a daemon thread.""" + """Main polling loop. Runs in a daemon thread. + + With no SSE subscribers there is nobody to notify, so the loop parks on + ``_idle_wake`` instead of re-fingerprinting ``state.db`` every few + seconds (maint #3035). While subscribed it blocks once on + ``_stop_event.wait(POLL_INTERVAL)`` — a single timer sleep that returns + immediately when ``stop()`` sets the event, replacing the previous + 10 wakeups/sec ``time.sleep(0.1)`` spin. + """ while not self._stop_event.is_set(): + if not self._has_subscribers(): + self._idle_wake.clear() + # subscribe() may have raced between the check and clear; if so + # the flag is already set again and we must not park. + if self._has_subscribers() or self._stop_event.is_set(): + continue + self._idle_wake.wait() + continue + try: self._poll_once() except Exception: logger.debug("Error in gateway watcher poll loop", exc_info=True) - # Sleep in small increments so we can stop promptly - for _ in range(self.POLL_INTERVAL * 10): - if self._stop_event.is_set(): - return - time.sleep(0.1) + self._stop_event.wait(self.POLL_INTERVAL) # ── Module-level watcher registry ────────────────────────────────────────── diff --git a/api/helpers.py b/api/helpers.py index 6e658f0d..1984bf28 100644 --- a/api/helpers.py +++ b/api/helpers.py @@ -202,6 +202,248 @@ def _security_headers(handler): ) +def arm_connection_close(handler) -> None: + """Mark the connection dead BEFORE a reject-before-read response is written. + + A rejection that answers without consuming the request body leaves those + bytes queued in ``rfile``; reusing the connection makes the next HTTP/1.1 + request parse mid-body (the classic ``{}GET ... -> 501``). Arming must + happen before the response is flushed so ``advertise_connection_close()`` + can serialize it into the response headers. + + Best-effort: handler stubs and read-only doubles may reject the attribute, + and a failure here must never turn a 4xx into a 500. + """ + try: + handler.close_connection = True + except Exception: + pass + + +# What ``unsupported_transfer_encoding()`` reports for a header that declares no +# coding at all (``Transfer-Encoding:``). A placeholder rather than the raw '' so +# the return value is never a falsy non-``None`` — callers test ``is not None``, +# and an empty string would also render as a hole in the rejection message. +_BLANK_TRANSFER_CODING = '(blank)' + +# The only whitespace a field value may be padded with: RFC 9110 OWS is SP/HTAB +# and nothing else. ``str.strip()`` would also erase U+00A0 and U+0085 -- both +# reachable over the wire, because request lines decode as latin-1 -- and +# ``Content-Length: \xa00`` would then read back as a genuine ``0`` while its +# payload sat unread on the socket. Padding a value with a character the grammar +# does not allow makes it unreadable, not zero. +_FIELD_VALUE_OWS = ' \t' + + +def _framing_header_values(handler, name: str) -> list[str]: + """Every value the wire carried for one framing header, tolerating stubs. + + ALL values, not just the first: ``Message.get()`` returns only the first + occurrence, so a request carrying ``Content-Length: 0`` followed by + ``Content-Length: 42`` reads back as length zero. Its 42 body bytes are then + never consumed and get parsed as the next request line -- the same poisoning + this module exists to prevent, reached by duplicating a header rather than by + omitting one. Conflicting framing is only visible if every value is. + + A real handler carries an ``email.message.Message`` (case-insensitive + ``get_all()``, like the wire); the suite's ``SimpleNamespace(headers={...})`` + doubles are plain dicts, which are not, so fall back to a case-insensitive + scan for those. A missing or odd ``headers`` must never raise: every caller + is on a rejection path where an exception becomes a spurious 500. + """ + headers = getattr(handler, 'headers', None) + if headers is None: + return [] + try: + get_all = getattr(headers, 'get_all', None) + if callable(get_all): + values = get_all(name) + return [] if values is None else [str(v) for v in values if v is not None] + if isinstance(headers, dict): + lowered = name.lower() + return [ + str(v) for k, v in headers.items() + if str(k).lower() == lowered and v is not None + ] + value = headers.get(name) + return [] if value is None else [str(value)] + except Exception: + return [] + + +def _declared_content_lengths(handler) -> set[int] | None: + """Distinct declared ``Content-Length`` values, or ``None`` if any is unreadable. + + ``None`` means the framing cannot be trusted at all (a value that will not + parse); an empty set means no length was declared. + + A BLANK or whitespace-only value is unreadable, NOT absent. ``Content-Length:`` + with payload bytes behind it declares a body whose size the header refuses to + state; skipping the empty value reported "no length declared", so the + rejection kept the connection alive and those bytes were parsed as the next + request line (403 then ``400 Bad request syntax ('{...}GET /api/... HTTP/1.1')``). + Absence is something only the wire can say, and ``_framing_header_values()`` + already reports a header the request never carried as no value at all -- so + anything that *is* here and does not parse has to be treated as pending bytes, + exactly like ``banana`` or two lengths that disagree. Note ``Message`` collapses + ``Content-Length:`` and ``Content-Length: `` to the same ``''``. + + "Parses" means RFC 9110's ``Content-Length = 1*DIGIT``: an unsigned run of + ASCII digits, nothing else. ``int()`` is far more generous than the grammar -- + it accepts a leading sign, PEP 515 underscores and non-ASCII digits -- and + every such spelling of ZERO used to read back as an honest ``0`` and take the + keep-alive branch below with its payload still queued: ``Content-Length: +0`` + (also ``-0``, ``0_0``, ``+00``, ``\\xa00``) answered 403 with no + ``Connection: close``, then ``400 Bad request syntax ('{...}GET /api/...')`` + -- the same trace as a blank value. A sign is only invisible on a NON-zero + value, where the wrong parse happens to close anyway. Anything the grammar + rejects is unreadable framing, which means a body may be pending. + + The ``int()`` guard stays even though ``isdigit()`` has already vetted every + character: a digit run longer than ``sys.get_int_max_str_digits()`` (4300) + raises ``ValueError`` too, and this runs on rejection paths where an + exception becomes a spurious 500. + """ + parsed: set[int] = set() + for raw in _framing_header_values(handler, 'Content-Length'): + # RFC 9110 §5.3: a list-valued field may arrive as repeated lines OR as + # one comma-combined line, and the two spellings mean the same thing. + # So "0, 0" is the same bodyless request as two "Content-Length: 0" + # headers and must keep its keep-alive; only a DISAGREEING list (caught + # below by len(parsed) > 1) is unframeable. Without the split the comma + # fails isdigit() and every agreeing list over-closed a healthy socket. + for member in raw.split(','): + stripped = member.strip(_FIELD_VALUE_OWS) + if not (stripped.isascii() and stripped.isdigit()): + return None + try: + parsed.add(int(stripped)) + except ValueError: + return None + return parsed + + +def unreadable_content_length(handler) -> bool: + """True when ``Content-Length`` cannot be trusted to frame the body. + + Either a value that will not parse (``banana``, ``0x5``, or a BLANK one), or + two or more values that disagree. None of them can tell a reader how many + bytes to drain, so such a request can only be rejected -- with the connection + armed for close, since bytes may still be queued behind it. + """ + parsed = _declared_content_lengths(handler) + return parsed is None or len(parsed) > 1 + + +def request_declares_body(handler) -> bool: + """True when the request's framing declares a body that is still queued. + + Keyed on the wire, not on the method: ``Transfer-Encoding`` present (chunked + framing hides the length until the terminating chunk is read) or + ``Content-Length`` present and non-zero. The method proves nothing in either + direction -- a GET may carry a declared body and a POST/PUT/DELETE may carry + none -- and a static per-method flag therefore both misses body-bearing + "read" requests (whose unread bytes poison the next pooled request) and + kills healthy keep-alive on body-less writes. + + A blank, unparseable or self-contradicting ``Content-Length`` counts as a + declared body, as does ANY ``Transfer-Encoding`` -- ``identity`` and a blank + value included: a value we cannot interpret does not prove the body's absence, + and the safe assumption for connection reuse is that bytes are pending. Only + framing that positively says "no body" -- no framing header at all, or every + declared length agreeing on zero -- keeps the connection alive. + """ + if unsupported_transfer_encoding(handler) is not None: + return True + parsed = _declared_content_lengths(handler) + if parsed is None: + return True + return any(length != 0 for length in parsed) + + +def unsupported_transfer_encoding(handler) -> str | None: + """Return the transfer coding this server cannot decode, or ``None``. + + EVERY present value counts, with no exemption for any coding. + ``BaseHTTPRequestHandler`` decodes none of them: it hands every reader a raw + ``rfile``, so a chunked body reads back as its literal chunk framing while a + ``Content-Length``-based reader sees length 0 and consumes nothing at all. + RFC 9112 section 6.3 says the same thing normatively -- a request whose final + transfer coding is not ``chunked`` has a body length the recipient cannot + determine, so it must be refused and the connection closed. Such a request + can therefore only be rejected, and only with the connection armed for close, + because its payload bytes are still queued on the socket. + + ``identity`` used to be exempt as a coding that "frames nothing". It is not: + it names no framing either, and RFC 7230 removed it from the transfer codings + altogether. A sidecar rejection then answered 403 with keep-alive and the + payload was parsed as the next request line + (``400 Bad request syntax ('{...}GET /api/health/agent HTTP/1.1')``), and all + four upload handlers fell through to a length-0 read, answered 400 "No file + field in request" and left the multipart bytes on the socket + (``400 Bad request syntax ('--x')``). Nothing legitimate is lost: no client + sends a transfer coding this server could have honoured. + + A header present but EMPTY (``Transfer-Encoding:``) is reported through + ``_BLANK_TRANSFER_CODING`` rather than as ``''``, so callers testing + ``is not None`` and the 411 message both read right. Only an ABSENT header + yields no values at all -- absence is something only the wire can say. + + The first value is enough precisely because no value is acceptable; it is + reported as-is (SP/HTAB trimmed, the one padding RFC 9110 allows) purely so + the rejection message names what arrived. + """ + values = _framing_header_values(handler, 'Transfer-Encoding') + if not values: + return None + return values[0].strip(_FIELD_VALUE_OWS) or _BLANK_TRANSFER_CODING + + +def arm_connection_close_if_body_pending(handler) -> bool: + """Arm close for a reject-before-read, but only if a body was really declared. + + Returns whether the connection was armed. Use this instead of + ``arm_connection_close()`` wherever the rejection can also fire on a request + with no body: closing then would drop a pooled client's healthy connection + for no framing reason. + """ + if not request_declares_body(handler): + return False + arm_connection_close(handler) + return True + + +def advertise_connection_close(handler) -> None: + """Serialize ``close_connection`` into exactly one ``Connection: close`` header. + + ``BaseHTTPRequestHandler`` tracks ``close_connection`` internally but never + writes it to the wire, so a server-side close is invisible to the client: a + pooled client reuses the socket and fails with BrokenPipeError instead of + opening a fresh connection. Call this from ``end_headers()``, before the + header buffer is flushed. + + Dedup is derived from the pending header buffer rather than a private flag, + because the SSE endpoints send their own ``Connection: close`` (and + ``send_header`` itself flips ``close_connection`` when they do) — keying off + a flag we set ourselves would emit the header twice on every stream. + + Both attribute reads are defensive: partially-built handler stubs in the + test suite carry neither ``close_connection`` nor ``_headers_buffer``, and + header emission must not depend on their presence. The buffer is also + type-checked before it is scanned -- a stub whose ``_headers_buffer`` is a + truthy non-iterable (a bare ``Mock``) would otherwise raise ``TypeError`` + from inside ``end_headers()``, which ``_handle_write`` would swallow into a + spurious 500. + """ + if not getattr(handler, 'close_connection', False): + return + buffered = getattr(handler, '_headers_buffer', None) + for raw in buffered if isinstance(buffered, (list, tuple)) else (): + if raw[:11].lower() == b'connection:': + return + handler.send_header('Connection', 'close') + + def flush_pending_auth_cookies(handler) -> None: pending = getattr(handler, '_pending_set_cookies', None) if not pending: diff --git a/api/media_snapshots.py b/api/media_snapshots.py index 929c1431..e54c34d8 100644 --- a/api/media_snapshots.py +++ b/api/media_snapshots.py @@ -505,6 +505,13 @@ def annotate_media_snapshots( if allowed_predicate is None: allowed_predicate = media_capture_allowed media_re = _re.compile(r"MEDIA:([^\s\)\]]+)") + # #7680 re-gate (9/22): two-pass scan. First strip backtick + # wrappers (`` `MEDIA:path` `` → ``MEDIA:path``) so the bare + # class below does not consume the closing backtick as part of + # the path. Then the bare class (no backtick in the exclusion + # set) captures the full filename even when the path itself + # contains a backtick (e.g. ``report`final.png``). + backtick_re = _re.compile(r"`MEDIA:([^`\s]+)`") captured = 0 for msg in messages or []: if not isinstance(msg, dict) or msg.get("role") != "assistant": @@ -530,6 +537,7 @@ def annotate_media_snapshots( if not text_parts: continue text = "\n".join(text_parts) + text = backtick_re.sub(lambda m: f"MEDIA:{m.group(1)}", text) refs = media_re.findall(text) if not refs: continue diff --git a/api/models.py b/api/models.py index 84a91b9c..0322a97b 100644 --- a/api/models.py +++ b/api/models.py @@ -48,7 +48,11 @@ from api.agent_sessions import ( from api.process_event_utils import stamp_message_source logger = logging.getLogger(__name__) -CLI_VISIBLE_SESSION_LIMIT = 20 +# Size of the interactive sidebar recency window. Also bounds how many +# delegated subagent children can be rendered at once, since a child only +# nests when it wins a slot in this window. Resolved in api.config before +# profile init; override with HERMES_WEBUI_VISIBLE_SESSION_LIMIT (clamped 1–200). +CLI_VISIBLE_SESSION_LIMIT = _cfg.CLI_VISIBLE_SESSION_LIMIT # How many messageful cron sessions to surface in the project-chip layer. # Needs to exceed CLI_VISIBLE_SESSION_LIMIT so older cron runs stay # addressable even when many newer non-cron sessions dominate the default @@ -4784,7 +4788,41 @@ class _FullSessionResolveRequired(RuntimeError): """Internal signal that a full sidecar load must enter the bounded path.""" -_FULL_SESSION_RESOLVE_MAX_CONCURRENT = 2 +def _read_max_session_resolve_concurrent() -> int: + """#7421: env override for the heavy session-resolve cap. The + hardcoded literal 2 is too low for high-concurrency + deployments where several parallel active sessions all need + a full-transcript resolve; operators can set + ``HERMES_WEBUI_MAX_SESSION_RESOLVE`` to a positive int to + raise the cap without code changes. A bad or missing value + falls back to the previous default of 2, so an operator who + sets ``HERMES_WEBUI_MAX_SESSION_RESOLVE=0`` or a non-numeric + value does not regress to a blocked or unbounded state. + + The cap is a *safety bound* (per-resolve cost tracked in + #7310), not a user preference — a typo like ``=999999`` + must fall back to the safe default, never to the permissive + extreme. Every other bad input in this function falls back + to ``2``; out-of-range values do too. The previously + accepted upper bound (64) is reachable only via an explicit + in-range operator value; a profile's ``.env`` cannot reach + it at all (see ``_PROTECTED_ENV_KEYS``). + """ + raw = (os.getenv("HERMES_WEBUI_MAX_SESSION_RESOLVE") or "").strip() + if not raw: + return 2 + try: + n = int(raw) + except (TypeError, ValueError): + return 2 + if n < 1: + return 2 + if n > 64: + return 2 # #7656 round-3: typo-safe fallback, not clamp-up-to-64 + return n + + +_FULL_SESSION_RESOLVE_MAX_CONCURRENT = _read_max_session_resolve_concurrent() _FULL_SESSION_RESOLVE_SLOTS = threading.BoundedSemaphore( _FULL_SESSION_RESOLVE_MAX_CONCURRENT ) @@ -8581,9 +8619,26 @@ def get_state_db_session_messages( cur.execute("PRAGMA table_info(sessions)") session_cols = {str(row['name']) for row in cur.fetchall()} if {'parent_session_id', 'end_reason', 'started_at', 'source'}.issubset(session_cols): + # session_source/model_config carry the fork/delegate boundary + # identity consumed by _is_continuation_session (#6931/#7021). + # Select them when present so the branch-marker guard applies + # to the stitch path too; older schemas degrade to NULL. + identity_cols = [] + if 'session_source' in session_cols: + identity_cols.append('session_source') + else: + identity_cols.append('NULL AS session_source') + if 'model_config' in session_cols: + identity_cols.append('model_config') + else: + identity_cols.append('NULL AS model_config') + lineage_select = ( + "id, source, started_at, parent_session_id, ended_at, end_reason" + + ", " + ", ".join(identity_cols) + ) cur.execute( - """ - SELECT id, source, started_at, parent_session_id, ended_at, end_reason + f""" + SELECT {lineage_select} FROM sessions WHERE id = ? """, @@ -8601,8 +8656,8 @@ def get_state_db_session_messages( if not parent_id or parent_id in seen: break cur.execute( - """ - SELECT id, source, started_at, parent_session_id, ended_at, end_reason + f""" + SELECT {lineage_select} FROM sessions WHERE id = ? """, @@ -10382,16 +10437,22 @@ def _insert_state_message_chronologically(messages: list, msg: dict) -> bool: # preceding assistant's tool_calls, not just the first — a multi-tool # turn has several adjacent tool results, and inserting between any of # them splits the block (assistant, tool, , tool). - if ( - idx < len(messages) - and messages[idx].get("role") == "tool" - and idx > 0 - and messages[idx - 1].get("role") == "assistant" - and messages[idx - 1].get("tool_calls") - ): - while idx < len(messages) and messages[idx].get("role") == "tool": - idx += 1 - advanced = True + if idx < len(messages) and messages[idx].get("role") == "tool": + # Walk back over any contiguous tool rows already emitted for + # this block, so the owning assistant is found even when idx + # lands on the SECOND result of a multi-tool turn (where + # messages[idx - 1] is another tool row, not the assistant). + owner = idx - 1 + while owner >= 0 and messages[owner].get("role") == "tool": + owner -= 1 + if ( + owner >= 0 + and messages[owner].get("role") == "assistant" + and messages[owner].get("tool_calls") + ): + while idx < len(messages) and messages[idx].get("role") == "tool": + idx += 1 + advanced = True # (b) Skip past an equal-timestamp run whose left neighbour shares # this message's role — inserting there would re-order an # already-matched same-role turn (user, , assistant). @@ -10420,6 +10481,7 @@ def merge_session_messages_append_only( *, truncation_watermark=None, truncation_boundary=None, + incoming_provenance: Literal["unverified", "state_db"] = "unverified", ) -> list: """Merge sidecar/context and state.db messages without deleting local rows. @@ -10433,6 +10495,7 @@ def merge_session_messages_append_only( state_messages, truncation_watermark=truncation_watermark, truncation_boundary=truncation_boundary, + incoming_provenance=incoming_provenance, ) finally: _STRUCTURED_IDENTITY_MEMO.reset(token) @@ -10444,6 +10507,7 @@ def _merge_session_messages_append_only_impl( *, truncation_watermark=None, truncation_boundary=None, + incoming_provenance=None, ) -> list: """Merge sidecar/context and state.db messages without deleting local rows. @@ -11038,7 +11102,21 @@ def _merge_session_messages_append_only_impl( seen_dedup_keys.add(dedup_key) seen_content_keys.add(content_key) seen_visible_keys.add(visible_key) - merged_messages.append(msg) + # This terminal path is shared by state.db reconciliation and by ordered + # sidecar stitching (notably compression continuations). Only the caller + # that read state.db may authorize timestamp-based recovery placement; + # stable child-sidecar sequence remains authoritative even when an + # archived parent was restamped later. + if ( + incoming_provenance == "state_db" + and max_sidecar_timestamp is not None + and timestamp is not None + and timestamp < max_sidecar_timestamp + ): + if not _insert_state_message_chronologically(merged_messages, msg): + merged_messages.append(msg) + else: + merged_messages.append(msg) _remember_merged_message(msg, source="state") return merged_messages @@ -11158,6 +11236,7 @@ def reconciled_state_db_messages_for_session( state_messages, truncation_watermark=getattr(session, "truncation_watermark", None), truncation_boundary=getattr(session, "truncation_boundary", None), + incoming_provenance="state_db", ) return _state_db_session_messages_result( reconciled_messages, diff --git a/api/onboarding.py b/api/onboarding.py index f09651c4..44ecabfe 100644 --- a/api/onboarding.py +++ b/api/onboarding.py @@ -71,7 +71,9 @@ _SUPPORTED_PROVIDER_SETUPS = { "default_model": "anthropic/claude-sonnet-4.6", "requires_base_url": False, "models": [ - {"id": _to_openrouter_namespace(model["id"]), "label": model["label"]} + {"id": "z-ai/glm-4.5-air", "label": "GLM-4.5 Air"} + if model["id"] == "zai/glm-4.5-flash" + else {"id": _to_openrouter_namespace(model["id"]), "label": model["label"]} for model in _FALLBACK_MODELS ], "category": "easy_start", diff --git a/api/process_event_utils.py b/api/process_event_utils.py index 2d9872f6..f6970d6d 100644 --- a/api/process_event_utils.py +++ b/api/process_event_utils.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections import OrderedDict from dataclasses import dataclass +import inspect import logging import math import re @@ -451,16 +452,46 @@ def complete_async_delegation_delivery( _mark_legacy_async_delivery_complete(claim.delegation_id) +def _release_accepts_retryable(release_fn: Any) -> bool: + """Whether the core ``release_event_delivery`` supports ``retryable=``. + + Older Agent cores predate the keyword; calling them with it would raise + ``TypeError`` and skip the release entirely. + """ + try: + params = inspect.signature(release_fn).parameters + except (TypeError, ValueError): + return False + if "retryable" in params: + return params["retryable"].kind in ( + inspect.Parameter.KEYWORD_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ) + return any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()) + + def release_async_delegation_delivery( evt: Any, claim: AsyncDelegationDeliveryClaim, + *, + retryable: bool = False, ) -> None: - """Release a failed claim so a later WebUI consumer can retry it.""" + """Release a failed claim so a later WebUI consumer can retry it. + + ``retryable=True`` tells the core the refusal was transient, so the claimed + attempt is refunded instead of counting against the bounded delivery + budget (``tools.async_delegation.release_event_delivery(..., + retryable=True)``). A core that predates the keyword gets a plain release: + the attempt is then consumed as before, never an exception. + """ try: if claim.durable: from tools.async_delegation import release_event_delivery - release_event_delivery(evt, claim.claim_id) + if retryable and _release_accepts_retryable(release_event_delivery): + release_event_delivery(evt, claim.claim_id, retryable=True) + else: + release_event_delivery(evt, claim.claim_id) except Exception: logger.warning( "Failed to release durable async delegation delivery for %s", diff --git a/api/profiles.py b/api/profiles.py index 1250899e..b7fd9297 100644 --- a/api/profiles.py +++ b/api/profiles.py @@ -198,7 +198,19 @@ def _unwrap_profile_home_to_base(home: Path) -> Path: # are operator/deployment-level postures, not per-profile toggles. Letting a # profile .env set HERMES_WEBUI_ISOLATED_PROFILE=0 would let a contained user # escape isolation (#4589). -_PROTECTED_ENV_KEYS = frozenset({'HERMES_WEBUI_ISOLATED_PROFILE'}) +_PROTECTED_ENV_KEYS = frozenset({ + 'HERMES_WEBUI_ISOLATED_PROFILE', + # Server-wide sidebar window; a profile .env must not widen the per-request query. + 'HERMES_WEBUI_VISIBLE_SESSION_LIMIT', + # #7656 round-3: HERMES_WEBUI_MAX_SESSION_RESOLVE is a process-wide resource + # cap (controls the BoundedSemaphore that gates full-transcript resolves). + # A profile's .env is per-profile by design; allowing it to set a + # server-wide limit (and a first-load-wins asymmetry) would let a + # single profile drag the rest into a tighter or looser posture than + # the operator intended. Same shape as the isolated-profile key: only + # the operator/launcher env at startup can set it. + 'HERMES_WEBUI_MAX_SESSION_RESOLVE', +}) def _isolated_profile_opt_in() -> bool: @@ -934,6 +946,13 @@ _BLOCKED_RUNTIME_ENV_KEYS = { # #4589: operator/deployment isolation posture — never overridable by a # profile's own env on any runtime/gateway-parity path. 'HERMES_WEBUI_ISOLATED_PROFILE', + 'HERMES_WEBUI_VISIBLE_SESSION_LIMIT', + # #7656 round-3: process-wide resolve cap. Filter out a profile's + # .env override the same way we filter the isolated-profile key, so + # the cap stays at the operator/launcher value across profile + # switches rather than re-sizing to whatever the first-loaded + # profile env set. + 'HERMES_WEBUI_MAX_SESSION_RESOLVE', } @@ -1866,7 +1885,19 @@ def _compute_profile_skills_stats(profile_dir: Path) -> tuple[int, int]: except Exception: pass - from agent.skill_utils import iter_skill_index_files, parse_frontmatter, skill_matches_platform + try: + from agent.skill_utils import iter_skill_index_files, parse_frontmatter, skill_matches_platform + except ImportError as exc: + logger.debug("agent.skill_utils unavailable; reporting skill stats as unknown: %s", exc) + # agent source not mounted (two-container Docker, + # HERMES_WEBUI_CHAT_BACKEND=gateway): this must never 500 GET + # /api/profiles (#7305). Report the skill stats as unknown — a stable + # (0, 0) — instead of keeping a partial shadow of agent.skill_utils + # here: a local re-implementation cannot preserve the index walk's + # exclusions, frontmatter-name identity or platform filtering, so any + # count it produced would be inaccurate. The UI omits the skills line + # when the total is 0, so the profile picker stays fully usable. + return (0, 0) seen_names = set() enabled_count = 0 diff --git a/api/route_approvals.py b/api/route_approvals.py index eb2761a0..da3a27d9 100644 --- a/api/route_approvals.py +++ b/api/route_approvals.py @@ -2,6 +2,24 @@ State-extraction prelude to the routes.py split tracked in #1907. Extracts approval state, not handlers, by design. + +Gateway approvals carry two pieces of state with one lifecycle: + +* the **producer** entry in ``tools.approval._gateway_queues`` — the queued + approval itself, which owns the lifecycle; and +* the **mirror** entry in ``_pending`` — this module's projection of that + producer for the UI. + +A producer is consumed only by an explicit settlement operation, which assigns +its result and signals its waiter before fallible projection work begins. +``retire_gateway_pending_mirror`` cleans up the UI projection; it must not +silently delete a producer, because that would strand the agent thread parked +on the producer's event. For a provider-qualified response, +``resolve_gateway_pending_run`` matches the exact ``(run_id, approval_id)`` +pair, settles that producer, and preserves same-run siblings and other runs. +When a run becomes terminal, ``settle_gateway_pending_run`` fail-closes every +producer owned by that run with ``deny`` before retiring its mirrors; producers +owned by another run remain live. """ import queue import threading @@ -505,21 +523,26 @@ def release_gateway_approval_relay_owner(session_key: str, run_id: str, approval _gateway_relay_owners.pop(key, None) +def _settle_gateway_entry(entry, choice: str, reason: str | None = None) -> None: + """Complete a removed producer before any fallible projection work.""" + entry.result = choice + if reason: + entry.reason = reason + entry.event.set() + + def retire_gateway_pending_mirror( session_key: str, approval_id: str = "", run_id: str = "", mirror_token: str = "", ) -> bool: - """Retire one approval, or every mirror for a terminal run.""" + """Retire one approval mirror, or every mirror for a terminal run.""" with _lock: reconcile_gateway_pending_mirror_locked(session_key) queue = _pending.get(session_key) entries = queue if isinstance(queue, list) else [queue] if queue else [] normalized_run_id = str(run_id or "").strip() - gateway_queue = _gateway_queues.get(session_key) or [] - retained_gateway_queue = gateway_queue - gateway_queue_changed = False if approval_id: match = _gateway_pending_mirror_locked( session_key, @@ -542,15 +565,7 @@ def retire_gateway_pending_mirror( if _is_gateway_mirror_entry(entry) and not str(entry.get("run_id") or "").strip() ] - if normalized_run_id: - retained_gateway_queue = [] - for entry in gateway_queue: - data = getattr(entry, "data", None) or {} - if str(data.get("run_id") or "").strip() == normalized_run_id: - gateway_queue_changed = True - continue - retained_gateway_queue.append(entry) - if not retired and not gateway_queue_changed: + if not retired: head, total, changed = reconcile_gateway_pending_mirror_locked(session_key) _approval_sse_notify_locked(session_key, head, total) if changed: @@ -558,11 +573,6 @@ def retire_gateway_pending_mirror( return changed for match in retired: entries.remove(match) - if normalized_run_id and not approval_id: - if retained_gateway_queue: - _gateway_queues[session_key] = retained_gateway_queue - else: - _gateway_queues.pop(session_key, None) if entries: _pending[session_key] = entries else: @@ -573,6 +583,60 @@ def retire_gateway_pending_mirror( return True +def settle_gateway_pending_run( + session_key: str, + run_id: str, + *, + reason: str, +) -> tuple[int, dict | None, int]: + """Fail-close every producer and mirror owned by a terminal gateway run.""" + run_id = str(run_id or "").strip() + if not run_id: + return 0, None, 0 + + targets = [] + with _lock: + gateway_queue = _gateway_queues.get(session_key) or [] + retained_gateway_queue = [] + for entry in gateway_queue: + data = getattr(entry, "data", None) or {} + if str(data.get("run_id") or "").strip() == run_id: + targets.append(entry) + else: + retained_gateway_queue.append(entry) + if retained_gateway_queue: + _gateway_queues[session_key] = retained_gateway_queue + else: + _gateway_queues.pop(session_key, None) + + # Removal and waiter settlement are one semantic operation. In + # particular, reconciliation and SSE notification below may raise. + for entry in targets: + _settle_gateway_entry(entry, "deny", reason) + + queue = _pending.get(session_key) + entries = queue if isinstance(queue, list) else [queue] if queue else [] + retained_pending = [ + entry + for entry in entries + if not ( + _is_gateway_mirror_entry(entry) + and str(entry.get("run_id") or "").strip() == run_id + ) + ] + mirrors_retired = len(retained_pending) != len(entries) + if retained_pending: + _pending[session_key] = retained_pending + else: + _pending.pop(session_key, None) + + head, total, changed = reconcile_gateway_pending_mirror_locked(session_key) + _approval_sse_notify_locked(session_key, head, total) + if targets or mirrors_retired or changed: + publish_session_list_changed("attention_resolved") + return len(targets), head, total + + def _gateway_mirrored_pending_run_id(session_key: str, approval_id: str) -> str | None: """Compatibility wrapper for exact run-backed lookup.""" approval_id = str(approval_id or "").strip() @@ -804,14 +868,50 @@ def resolve_gateway_pending_local( _gateway_queues[session_key] = gateway_queue else: _gateway_queues.pop(session_key, None) + if target is not None: + _settle_gateway_entry(target, choice, reason) + head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) + _approval_sse_notify_locked(session_key, head, total) + if target is None: + return 0, head, total + publish_session_list_changed("attention_resolved") + return 1, head, total + + +def resolve_gateway_pending_run( + session_key: str, + approval_id: str, + run_id: str, + choice: str, + reason: str | None = None, +) -> tuple[int, dict | None, int]: + """Resolve one exact run-backed producer and wake its parked waiter.""" + target = None + approval_id = str(approval_id or "").strip() + run_id = str(run_id or "").strip() + if not approval_id or not run_id: + return 0, None, 0 + with _lock: + gateway_queue = _gateway_queues.get(session_key) or [] + retained_gateway_queue = [] + for entry in gateway_queue: + data = getattr(entry, "data", None) or {} + entry_approval_id = str(data.get("approval_id") or "").strip() + entry_run_id = str(data.get("run_id") or "").strip() + if target is None and entry_approval_id == approval_id and entry_run_id == run_id: + target = entry + else: + retained_gateway_queue.append(entry) + if retained_gateway_queue: + _gateway_queues[session_key] = retained_gateway_queue + else: + _gateway_queues.pop(session_key, None) + if target is not None: + _settle_gateway_entry(target, choice, reason) head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) _approval_sse_notify_locked(session_key, head, total) if target is None: return 0, head, total - target.result = choice - if reason: - target.reason = reason - target.event.set() publish_session_list_changed("attention_resolved") return 1, head, total @@ -856,12 +956,9 @@ def resolve_gateway_pending_local_no_run_mirror( _pending[session_key] = entries else: _pending.pop(session_key, None) + _settle_gateway_entry(target, choice, reason) head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) _approval_sse_notify_locked(session_key, head, total) - target.result = choice - if reason: - target.reason = reason - target.event.set() publish_session_list_changed("attention_resolved") return True, 1, head, total @@ -904,14 +1001,11 @@ def resolve_gateway_pending_local_all( else: _pending.pop(session_key, None) + for entry in targets: + _settle_gateway_entry(entry, choice, reason) head, total, _changed = reconcile_gateway_pending_mirror_locked(session_key) _approval_sse_notify_locked(session_key, head, total) - for entry in targets: - entry.result = choice - if reason: - entry.reason = reason - entry.event.set() if targets or removed_pending: publish_session_list_changed("attention_resolved") return len(targets), head, total diff --git a/api/routes.py b/api/routes.py index fec6cc3d..1b459479 100644 --- a/api/routes.py +++ b/api/routes.py @@ -46,6 +46,7 @@ from api.agent_sessions import ( _looks_like_default_cli_title, is_cli_session_row, is_cli_session_row_visible, + open_state_db_readonly, read_session_lineage_report, ) from api.compression_anchor import visible_messages_for_anchor @@ -320,7 +321,7 @@ def _latest_cron_session_info_for_jobs( if not db_path or not Path(db_path).exists(): return {jid: {"session_id": "", "message_count": None} for jid in requested} try: - with closing(sqlite3.connect(str(db_path))) as conn: + with closing(open_state_db_readonly(db_path)) as conn: conn.row_factory = sqlite3.Row cur = conn.cursor() cur.execute("PRAGMA table_info(sessions)") @@ -593,7 +594,15 @@ def _request_session_visibility_exempt(method: str, path: str | None) -> bool: def _session_id_visible_to_request_profile(handler, sid, *, emit_error: bool = True) -> bool: - """Return whether ``sid`` belongs to the active profile.""" + """Return whether ``sid`` belongs to the active profile. + + On a profile mismatch, the helper mirrors the detail-load endpoint's + contract (#13043, #13493): return ``409 session_profile_mismatch`` for + a session owned by a KNOWN other profile, and keep ``404 Session + not found`` only for the unknown/legacy None-profile case so the + frontend's self-heal (clear stale URL + localStorage) keeps firing + for actually-missing sids. ``#7710``. + """ if not isinstance(sid, str) or not sid: return True if not is_safe_session_id(sid): @@ -602,9 +611,23 @@ def _session_id_visible_to_request_profile(handler, sid, *, emit_error: bool = T session = get_session(sid, metadata_only=True) except KeyError: return True - if not _session_visible_to_active_profile(getattr(session, "profile", None), handler): + session_profile = getattr(session, "profile", None) or None + if not _session_visible_to_active_profile(session_profile, handler): if emit_error: - bad(handler, "Session not found", 404) + if session_profile: + j(handler, { + "error": "Session belongs to a different profile", + "code": "session_profile_mismatch", + "session_id": sid, + "profile": session_profile, + }, status=409) + else: + # Unknown/legacy None-profile sidecar: keep the 404 so the + # frontend's self-heal still fires. _profiles_match coerces + # None->'default', so a truly missing/legacy session under a + # non-default active profile would otherwise emit a useless + # 409 with profile=null. + bad(handler, "Session not found", 404) return False return True @@ -2521,8 +2544,8 @@ def _build_session_list_cache_payload( ) if show_cli_sessions: diag_stage("cli_cap") - archived_scoped = _cap_recent_cli_sessions(archived_scoped, cli_cap=CLI_VISIBLE_SESSION_CAP) - visible_scoped = _cap_recent_cli_sessions(visible_scoped, cli_cap=CLI_VISIBLE_SESSION_CAP) + archived_scoped = _cap_recent_cli_sessions(archived_scoped) + visible_scoped = _cap_recent_cli_sessions(visible_scoped) if visible_only: archived_scoped = [ s for s in archived_scoped if _session_has_server_visible_messages(s) @@ -2952,6 +2975,7 @@ from api.helpers import ( require, bad, safe_resolve, + arm_connection_close_if_body_pending, j, t, read_body, @@ -3461,16 +3485,31 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | emit_error=False, ): return None - summary = find_run_summary(stream_id) - if not summary: - return None - session_id = str(summary.get("session_id") or "") - if not session_id: - return None + # Locate the run journal WITHOUT parsing it first. find_run_summary reads + # and parses the whole file (tens of thousands of rows on a long live + # run), and read_run_events below then parsed it AGAIN — two full passes + # cost ~0.6s per rebuild. Derive the durable summary from the single + # parse below instead; its last_seq / last_event_id are rebuilt from the + # same rows the snapshot consumes. + located = find_run_file(stream_id) + if located: + session_id, _journal_path = located + else: + # No journal file on disk for this run id: fall back to the + # historical summary-based lookup seam (defensive — a real miss + # returns None exactly like before; this is also the seam that + # long-standing tests stub with a handler-side summary). + fallback_summary = find_run_summary(stream_id) + if not fallback_summary: + return None + session_id = str(fallback_summary.get("session_id") or "") + if not session_id: + return None journal = read_run_events(session_id, stream_id) events = [event for event in (journal.get("events") or []) if isinstance(event, dict)] if not events: return None + summary = _summary_from_events(session_id, stream_id, events) event_run_ids: set[str] = set() malformed_envelope_run_id = False for event in events: @@ -3490,6 +3529,18 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | assistant_text = "" reasoning_text = "" + # Reasoning deltas arrive as thousands of tiny append events. Growing the + # transcript with ``reasoning_text += chunk`` copies the full (multi- + # hundred-KB) string on every append inside these closures — quadratic + # (~2.3s of a 4.6s rebuild on a 9.7k-chunk run). Collect chunks and + # materialize lazily; readers (echo probes, strip, final message) join at + # most once per interim segment. + reasoning_parts: list[str] = [] + reasoning_dirty = False + # Incremental folded index over the reasoning transcript: the per-interim + # echo probe consults this instead of re-walking the raw text (see + # ``_CompactEchoIndex``). + reasoning_index = _CompactEchoIndex() messages: list[dict] = [] tool_calls: list[dict] = [] activity_burst_anchors: list[dict] = [] @@ -3498,6 +3549,13 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | last_ts = None reasoning_first_tool_count: int | None = None + def _materialize_reasoning_text() -> str: + nonlocal reasoning_text, reasoning_dirty + if reasoning_dirty: + reasoning_text = "".join(reasoning_parts) + reasoning_dirty = False + return reasoning_text + def mark_boundary() -> int: nonlocal current_activity_burst_id text_end = len(assistant_text) @@ -3562,22 +3620,42 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | tool_calls.append(call) def reasoning_echo_tail_matches(text: str) -> bool: - candidate = _compact_for_echo_compare(text) - if not candidate: - return False - return _compact_for_echo_compare(reasoning_text).endswith(candidate) + # Indexed tail match (no raw-text walk): the incremental index folds + # each reasoning chunk once as it is appended, so this probe costs + # O(len(text)) regardless of how much interior whitespace stretches + # the raw span. The previous raw backward walk re-walked that span + # once per interim event, which turned the replay quadratic on a + # whitespace-heavy transcript (#7569 review, ~55x slower than master + # on a production-shaped journal). + return reasoning_index.matches_tail(text) def strip_reasoning_echo_tail(text: str) -> bool: - nonlocal reasoning_text, reasoning_first_tool_count - next_reasoning, did_remove = _strip_compact_echo_suffix(reasoning_text, text) - if did_remove: - reasoning_text = next_reasoning - if not _compact_for_echo_compare(reasoning_text): - reasoning_first_tool_count = None - return did_remove + nonlocal reasoning_text, reasoning_dirty, reasoning_first_tool_count + cut = reasoning_index.cut_to(text) + if cut is None: + return False + next_reasoning = _materialize_reasoning_text()[:cut].rstrip() + reasoning_text = next_reasoning + reasoning_parts.clear() + reasoning_parts.append(next_reasoning) + reasoning_dirty = False + # Re-index the truncated transcript so later probes match against it + # instead of the pre-strip tail (the index is the match authority). + reasoning_index.reset() + reasoning_index.append(next_reasoning) + if not _compact_for_echo_compare(reasoning_text): + reasoning_first_tool_count = None + return True for event in events: event_name = str(event.get("event") or event.get("type") or "") + if event_name == "metering": + # Metering rows are the bulk of a long live journal (~10k rows, + # ~60% of its bytes) and project nothing onto the snapshot; skip + # their per-row work while preserving the last_ts watermark they + # carry. + last_ts = event.get("created_at", last_ts) + continue payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} last_ts = event.get("created_at", last_ts) if event_name == "token": @@ -3588,9 +3666,12 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | continue if event_name == "reasoning": text = str(payload.get("text") or "") - if text and reasoning_first_tool_count is None: - reasoning_first_tool_count = len(tool_calls) - reasoning_text += text + if text: + if reasoning_first_tool_count is None: + reasoning_first_tool_count = len(tool_calls) + reasoning_parts.append(text) + reasoning_index.append(text) + reasoning_dirty = True continue if event_name == "interim_assistant": visible = str(payload.get("text") or "").strip() @@ -3635,6 +3716,8 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | update_completed_tool(payload) fresh_segment = True + reasoning_text = _materialize_reasoning_text() + if assistant_text or reasoning_text: message = { "role": "assistant", @@ -5323,7 +5406,21 @@ def _handle_session_anchor_scene(handler, body): # this an authenticated request under profile A could persist anchor scenes # onto a session owned by profile B (cross-profile write). Reject as 404 — # same shape the read path uses — and leave anchor_activity_scenes untouched. - if not _session_visible_to_active_profile(getattr(s, "profile", None) or None, handler): + # #7710: cross-profile writes are rejected with 409 + # ``session_profile_mismatch`` so the client can offer to switch + # to the owning profile (mirrors the detail-load endpoint's + # contract at #13043 / #13493). 404 is preserved for the + # None-profile (unknown/legacy) case so the frontend self-heal + # path still fires for actually-missing sids. + _anchor_session_profile = getattr(s, "profile", None) or None + if not _session_visible_to_active_profile(_anchor_session_profile, handler): + if _anchor_session_profile: + return j(handler, { + "error": "Session belongs to a different profile", + "code": "session_profile_mismatch", + "session_id": sid, + "profile": _anchor_session_profile, + }, status=409) return bad(handler, "Session not found", 404) with _get_session_agent_lock(sid): idx, message = _find_anchor_scene_message( @@ -5835,6 +5932,15 @@ def _csrf_rejection_error(handler) -> str: def _check_csrf(handler) -> bool: """Reject cross-origin or tokenless authenticated browser unsafe requests.""" if not _check_same_origin_browser_request(handler): + # CSRF checks run before read_body(), so close rather than reusing an + # HTTP/1.1 connection whose unread body would corrupt the next request -- + # but ONLY when the framing says bytes are really queued. Arming + # unconditionally dropped a healthy pooled connection on a body-less + # write: verified on the wire, `POST /api/session/new` with + # `Origin: http://evil.invalid` and no `Content-Length` (and with + # `Content-Length: 0`) answered 403 + `Connection: close` and the + # pipelined `GET /api/health/agent` was never served. + arm_connection_close_if_body_pending(handler) return False if not _is_browser_unsafe_request(handler): return True # non-browser clients (curl, MCP, agent) have no Origin/Referer @@ -5847,6 +5953,11 @@ def _check_csrf(handler) -> bool: submitted = handler.headers.get(CSRF_HEADER_NAME) or handler.headers.get("X-CSRF-Token") if verify_csrf_token(cookie_val or "", submitted or ""): return True + # Same framing rule as the origin rejection above: a token mismatch on a + # body-less write has nothing unread to protect. Verified on the wire with an + # authenticated same-origin `POST /api/session/new` carrying no CSRF token and + # no `Content-Length`: 403 + `Connection: close`, follow-up dropped. + arm_connection_close_if_body_pending(handler) return _set_csrf_failure_reason(handler, "token_mismatch") @@ -6026,6 +6137,14 @@ def _handle_extension_sidecar_proxy( # DELETE fell through the CSRF compatibility path that intentionally admits # non-browser clients, giving unsafe methods weaker provenance than GET. if not _check_same_origin_browser_request(handler, require_provenance=True): + # Provenance rejection runs before read_body(), so close-and-advertise + # whenever the request DECLARED a body (Content-Length non-zero, or any + # Transfer-Encoding): those bytes are still queued in rfile and a reused + # HTTP/1.1 connection would parse them as the next request line (same + # class as _check_csrf). Gating on read_request_body instead was wrong in + # both directions — it missed a GET that carries a declared body, and it + # closed a healthy connection on a body-less DELETE/PUT/PATCH. + arm_connection_close_if_body_pending(handler) return j(handler, {"error": _csrf_rejection_error(handler)}, status=403) try: request_body = _read_body_bytes(handler) if read_request_body else None @@ -6468,6 +6587,13 @@ def _handle_csp_report(handler) -> bool: "Dropped CSP report from %s: rate limit exceeded", _client_ip_for_rate_limit(handler), ) + # Rate-limit rejection runs before the body is read; close-and-advertise + # so the unread report can't corrupt the next pooled request -- but only + # when a body was really declared. A body-less report answered 204 WITH + # `Connection: close` once the 100-per-60s limiter tripped (reproduced on + # the wire at request 101; the pipelined `GET /api/health/agent` was + # dropped), so a browser that keeps reporting loses its socket each time. + arm_connection_close_if_body_pending(handler) return _send_no_content(handler) payload = _read_csp_report_payload(handler) @@ -9230,6 +9356,7 @@ def _limited_webui_messages_for_display_with_sidecar( state_db_messages, truncation_watermark=getattr(session, "truncation_watermark", None), truncation_boundary=getattr(session, "truncation_boundary", None), + incoming_provenance="state_db", ) if cache_key is not None: _state_key = cache_key[4] @@ -10278,11 +10405,16 @@ def _dedupe_cli_sidebar_sessions_for_api( return _include_project_hidden_background_sidebar_sessions(candidates, visible) -CLI_VISIBLE_SESSION_CAP = 20 +def _cli_visible_session_cap() -> int: + """Shared sidebar window, not a second hard-coded 20.""" + from api.config import CLI_VISIBLE_SESSION_LIMIT + return CLI_VISIBLE_SESSION_LIMIT -def _cap_recent_cli_sessions(sessions: list[dict], cli_cap: int = CLI_VISIBLE_SESSION_CAP) -> list[dict]: +def _cap_recent_cli_sessions(sessions: list[dict], cli_cap: int | None = None) -> list[dict]: """Keep only the most recent CLI-visible sessions after filtering.""" + if cli_cap is None: + cli_cap = _cli_visible_session_cap() if cli_cap <= 0: return sessions kept = [] @@ -10488,6 +10620,11 @@ def _pre_compression_continuation_session_id(session) -> str | None: exists either in memory or on disk. Follow bounded snapshot-to-snapshot hops so repeated compression still lands on the latest visible continuation. """ + from api.compression_continuation import durable_compression_continuation + + sealed, tip = durable_compression_continuation(session) + if sealed: + return tip if not getattr(session, "pre_compression_snapshot", False): return None sid = _safe_first(getattr(session, "session_id", None)) @@ -10698,13 +10835,16 @@ from api.streaming import ( _materialize_pending_user_turn_before_error, generate_session_title_for_session, _compact_for_echo_compare, - _strip_compact_echo_suffix, + _CompactEchoIndex, ) from api.gateway_chat import _run_gateway_chat_streaming, webui_gateway_chat_enabled from api.run_journal import ( _parse_run_journal_event_id as _shared_parse_run_journal_event_id, + _summary_from_events, bound_run_journal_snapshot_args, + find_run_file, find_run_summary, + journal_replay_visible, read_run_events, read_session_run_events, session_journal_fingerprint, @@ -10765,8 +10905,10 @@ from api.route_approvals import ( # noqa: F401 — re-exports for backward comp gateway_pending_mirrors, release_gateway_approval_relay_owner, retire_gateway_pending_mirror, + settle_gateway_pending_run, reconcile_gateway_pending_mirror_locked, resolve_gateway_pending_local, + resolve_gateway_pending_run, resolve_gateway_pending_local_all, resolve_gateway_pending_local_no_run_mirror, set_session_yolo_enabled, @@ -11080,9 +11222,7 @@ button:hover{background:rgba(124,185,255,.25)}

{{BOT_NAME}}

{{LOGIN_SUBTITLE}}

- - - + {{PASSWORD_FORM_HTML}} {{OIDC_LOGIN_HTML}}
@@ -11897,7 +12037,7 @@ def _handle_insights(handler, parsed) -> bool: from api.models import _active_state_db_path db_path = _active_state_db_path() if db_path and db_path.exists(): - with closing(sqlite3.connect(str(db_path))) as conn: + with closing(open_state_db_readonly(db_path)) as conn: conn.row_factory = sqlite3.Row cur = conn.cursor() # cache_read_tokens may not exist on older agent state DBs; @@ -12532,7 +12672,7 @@ def _deep_health_checks(stream_check: dict | None = None) -> tuple[dict, bool]: "ms": round((time.time() - t0) * 1000, 1), } else: - with closing(sqlite3.connect(str(db_path))) as conn: + with closing(open_state_db_readonly(db_path)) as conn: conn.execute("PRAGMA schema_version").fetchone() checks["state_db"] = { "status": "ok", @@ -12853,6 +12993,15 @@ def _handle_shutdown(handler) -> bool: def _handle_health_restart(handler) -> bool: """Restart the Hermes messaging gateway service.""" + # This endpoint never consumes its request body on any outcome, so close when + # one was DECLARED -- and only then. Arming unconditionally closed the socket + # on every call including the successful, body-less one the WebUI actually + # makes: verified on the wire, `POST /api/health/restart` with no + # `Content-Length` answered with `Connection: close` and the pipelined + # `GET /api/health/agent` was never served. The single arming covers every + # outcome below (completed / in_progress / busy / error) because the framing, + # not the result, decides. + arm_connection_close_if_body_pending(handler) outcome = restart_active_profile_gateway() if outcome.get("status") == "completed": @@ -13625,6 +13774,36 @@ def handle_get(handler, parsed) -> bool: ] from urllib.parse import quote from api.updates import WEBUI_VERSION + # #7056: only render the password input / submit / passkey controls + # when password auth is actually enabled. With native OIDC configured + # and ``HERMES_WEBUI_PASSWORD`` unset, the form previously still + # displayed the password prompt and accepted — silently 401-ing at + # the server — every submit. The OIDC SSO entry point stays the + # sole path. ``is_password_auth_enabled`` is the same predicate + # ``/api/auth/status`` reports as ``password_auth_enabled``. + from api.auth import are_passkeys_enabled, is_password_auth_enabled + + # The password INPUT is gated on a configured password, but the passkey + # button must survive a passwordless-passkey deployment: settings expose + # ``passwordless_enabled = passkeys registered AND not password_auth_enabled`` + # (routes.py ~14059) and ``is_auth_enabled()`` counts passkeys as an + # independent auth method, so hiding the button when no password is set + # would remove the ONLY working login affordance for those instances. + _passkey_button_html = ( + '' + ) + if is_password_auth_enabled(): + _password_form_html = ( + f'' + f'' + f'{_passkey_button_html}' + ) + elif are_passkeys_enabled(): + _password_form_html = _passkey_button_html + else: + _password_form_html = "" version_token = quote(WEBUI_VERSION, safe="") _page = ( _LOGIN_PAGE_HTML.replace("{{BOT_NAME}}", _bn) @@ -13633,10 +13812,7 @@ def handle_get(handler, parsed) -> bool: .replace("{{LANG}}", _html.escape(_login_strings["lang"])) .replace("{{LOGIN_TITLE}}", _html.escape(_login_strings["title"])) .replace("{{LOGIN_SUBTITLE}}", _html.escape(_login_strings["subtitle"])) - .replace( - "{{LOGIN_PLACEHOLDER}}", _html.escape(_login_strings["placeholder"]) - ) - .replace("{{LOGIN_BTN}}", _html.escape(_login_strings["btn"])) + .replace("{{PASSWORD_FORM_HTML}}", _password_form_html) .replace("{{LOGIN_INVALID_PW}}", _html.escape(_login_strings["invalid_pw"])) .replace( "{{LOGIN_CONN_FAILED}}", _html.escape(_login_strings["conn_failed"]) @@ -14478,7 +14654,11 @@ def handle_get(handler, parsed) -> bool: if stop_gateway_run(run_id): owner_sid = stream_owner_session_id(stream_id) if owner_sid: - retire_gateway_pending_mirror(owner_sid, run_id=run_id) + settle_gateway_pending_run( + owner_sid, + run_id, + reason="Gateway run was cancelled before approval resolution", + ) else: gateway_stop_blocked = True except Exception: @@ -15144,6 +15324,12 @@ def handle_post(handler, parsed) -> bool: if diag: diag.stage("process_complete_ack_deprecated") try: + # The 410 runs before the stale tab's JSON body is read; + # close-and-advertise so those unread bytes can't corrupt the next + # pooled request -- but a body-less ack has none, and closing then + # just kills keep-alive (on the wire: 410 + `Connection: close` with + # no `Content-Length`, pipelined follow-up dropped). + arm_connection_close_if_body_pending(handler) j( handler, { @@ -18573,6 +18759,8 @@ def _replay_run_journal( max_seq=max_seq, ) for entry in journal.get("events") or []: + if not journal_replay_visible(entry): + continue _sse_with_id( handler, entry.get("event") or entry.get("type") or "message", @@ -19026,14 +19214,14 @@ def _handle_sse_stream(handler, parsed): continue if len(item) >= 3: event, data, queued_event_id = item[0], item[1], item[2] + event_id = queued_event_id else: event, data = item - queued_event_id = STREAM_LAST_EVENT_ID.get(stream_id) + event_id = STREAM_LAST_EVENT_ID.get(stream_id) # Stage-364: emit `id:` from STREAM_LAST_EVENT_ID side-channel so # the frontend's `_lastRunJournalSeq` cursor advances during live # streaming. Without this, mid-stream error→replay would arrive # with after_seq=0 and double-render every journaled event. - event_id = queued_event_id or STREAM_LAST_EVENT_ID.get(stream_id) event_seq = _run_journal_same_run_seq(event_id, stream_id) if replay_cutoff_seq is not None and event_seq is not None and event_seq <= replay_cutoff_seq: continue @@ -19108,6 +19296,8 @@ def _handle_session_run_journal_stream_for_session(handler, parsed, session_id): def emit_replay(events, stream_id, cutoff_seq): for entry in events: + if not journal_replay_visible(entry): + continue event_id = str(entry.get("event_id") or "") event_seq = _run_journal_same_run_seq(event_id, stream_id) if cutoff_seq is not None and event_seq is not None and event_seq > cutoff_seq: @@ -19174,10 +19364,10 @@ def _handle_session_run_journal_stream_for_session(handler, parsed, session_id): continue if len(item) >= 3: event, data, queued_event_id = item[0], item[1], item[2] + event_id = queued_event_id else: event, data = item - queued_event_id = STREAM_LAST_EVENT_ID.get(active_stream_id) - event_id = queued_event_id or STREAM_LAST_EVENT_ID.get(active_stream_id) + event_id = STREAM_LAST_EVENT_ID.get(active_stream_id) event_seq = _run_journal_same_run_seq(event_id, active_stream_id) _is_terminal = event in SSE_RELAY_CLOSE_EVENTS _already_sent = ( @@ -20444,6 +20634,16 @@ def _serve_inline_html_preview(handler, target: Path, cache_control: str, *, csp _MEDIA_TOKEN_RE = re.compile(r"MEDIA:([^\s\)\]]+)") +# #7680 re-gate (9/22): two-pass scan. +# 1. `` `MEDIA:path` `` (backtick-wrapped, inline-code form) → strip +# the wrapping backticks so the bare-token pass below sees a +# plain ``MEDIA:path`` and the closing backtick is not consumed +# as part of the path. +# 2. ``MEDIA:[^\s\)\]]+`` (bare, no backtick in the exclusion +# class) so a filename that legally contains a backtick +# (``report`final.png``) is captured in full instead of being +# truncated at the first backtick. +_BACKTICK_MEDIA_RE = re.compile(r"`MEDIA:([^`\s]+)`") def _message_content_text(content) -> str: @@ -20503,6 +20703,10 @@ def _session_media_token_allows_path(sid: str, target: Path, allowed_mimes: set[ ) if "MEDIA:" not in text: continue + # #7680 re-gate: strip backtick wrappers first so the bare + # class below captures the full path even when the filename + # itself contains a backtick. + text = _BACKTICK_MEDIA_RE.sub(lambda m: f"MEDIA:{m.group(1)}", text) for ref in _MEDIA_TOKEN_RE.findall(text): if "://" in ref: continue @@ -21695,6 +21899,99 @@ def _handle_live_models(handler, parsed): _env = str(_cp.get("key_env") or "").strip() return os.getenv(_env, "").strip() if _env else "" + def _custom_provider_models_discover_is_false(_cp): + """True when ``discover_models`` is an explicit ``false`` opt-out. + + Mirrors ``api.config._provider_discover_allowed`` / Hermes + Agent ``model_switch_providers._discover_flag``: ``discover_models`` + defaults to True, and the string forms ``"false"``/``"no"``/``"0"`` + (case-insensitive) mean False. Used to decide whether a + dict-shaped ``models`` mapping is a hand-pinned allowlist (only + when discovery is off) rather than per-model metadata. + """ + _discover = _cp.get("discover_models", True) if isinstance(_cp, dict) else True + if isinstance(_discover, str): + return _discover.strip().lower() in {"false", "no", "0"} + return not bool(_discover) + + def _custom_provider_allowlist_ids(_cp): + """Plural ``models`` list only — the explicit allowlist signal. + + The singular ``model`` field is sticky/default metadata, NOT an + allowlist: it must not gate live-catalog filtering, otherwise a + provider configured with only ``model: assistant`` (no ``models`` + list) would be collapsed to a single model. Only an explicit + ``models`` allowlist expresses "show exactly these models". + + An auto-discovered catalog is NOT an allowlist: when Hermes + persisted discovery results back into config (``models: {...}`` + plus ``models_discovered: true``), that mapping is a snapshot + of what the gateway exposed at discovery time. Gating on it + would permanently pin the live catalog to the first-discovery + set, silently dropping any model the user pulls in later + (LM Studio / Ollama). ``_provider_models_are_discovered_catalog()`` + is the shared predicate the ``/api/models`` path already uses; + when it says "discovered", return no allowlist and let the + live probe win. An explicit ``discover_models: false`` opt-out + re-pins the catalog (the predicate accounts for it), so a + hand-pinned discovered catalog still filters. + + A dict-shaped ``models`` mapping that is NOT marked discovered is + *per-model metadata* written by the Hermes Agent setup flow + (``hermes_cli/model_switch.py::_save_custom_provider`` and the + setup wizard) — e.g. ``{chat-a: {context_length: 128000}}``. It is + not a catalog narrow: treating its keys as an allowlist would + collapse the live picker to the single saved default (keyless + Ollama) while the CLI live-probe shows the full catalog. Only an + explicit ``discover_models: false`` opts into treating the dict + keys as a pinned allowlist. List/JSON-array-string/Python-literal + shapes remain plain allowlists. + + The plural value is decoded through ``_parse_config_string_list()`` + because ``hermes config set`` / JSON-mode editor saves persist + lists as quoted JSON-array strings (``'["chat-a","chat-b"]'``) or + Python literals (``"['chat-a']"``). Handling only ``dict``/``list`` + made a serialized allowlist fall through to ``[]``, which the + caller reads as "no allowlist configured" and floods the picker + with the full upstream catalog — the same class of bug as the + ``skills.disabled`` regression (#7120 / #7134). + """ + from api.config import _provider_models_are_discovered_catalog + + if _provider_models_are_discovered_catalog(_cp): + return [] + _ids = [] + _models = _cp.get("models") + # Serialized shapes only: ``hermes config set`` / JSON-mode + # editor saves persist lists as quoted JSON-array strings + # (``'["chat-a","chat-b"]'``) or Python literals + # (``"['chat-a']"``). Decode those through the shared helper so + # they are honored; native dict/list values are walked below so + # dict *entries* keep their id|model|name metadata (the decoder + # str()-ifies list members, which would mangle them). + if isinstance(_models, str): + _models = _parse_config_string_list(_models) + if isinstance(_models, dict): + # A dict-shaped ``models`` is per-model metadata written by + # the Hermes Agent setup flow, NOT a catalog narrow. Only a + # ``discover_models: false`` opt-out treats the dict keys as a + # pinned allowlist; otherwise return the empty allowlist so the + # live probe returns the full catalog. + if not _custom_provider_models_discover_is_false(_cp): + return [] + for _mid in _models: + if isinstance(_mid, str) and _mid.strip(): + _ids.append(_mid.strip()) + elif isinstance(_models, (list, tuple)): + for _item in _models: + if isinstance(_item, str) and _item.strip(): + _ids.append(_item.strip()) + elif isinstance(_item, dict): + _mid = _item.get("id") or _item.get("model") or _item.get("name") + if _mid and str(_mid).strip(): + _ids.append(str(_mid).strip()) + return _ids + # For 'custom' and 'custom:*' providers, provider_model_ids() # returns [] because they aren't real hermes_cli endpoints. # Fall back to the custom_providers entries from config.yaml so @@ -21703,11 +22000,13 @@ def _handle_live_models(handler, parsed): # Collect config-specified model IDs separately so they don't # prevent the live fetch below from running (#3718). _config_ids = [] + _allowlist_ids = [] if provider == "custom" or provider.startswith("custom:"): for _cp in _custom_provider_entries_for_request(): if custom_provider_entry is None: custom_provider_entry = _cp _config_ids.extend(_custom_provider_model_ids(_cp)) + _allowlist_ids.extend(_custom_provider_allowlist_ids(_cp)) # Always try live fetch for custom providers — config entries are a # fallback, not a replacement. The live endpoint should return ALL @@ -21778,15 +22077,39 @@ def _handle_live_models(handler, parsed): except Exception as _fetch_err: logger.debug("Live fetch from custom provider failed: %s", _fetch_err) - # If live fetch succeeded, merge with config entries (live takes - # priority). If live fetch failed, fall back to config-only list. + # If live fetch succeeded, filter the live catalog down to the + # config-declared models allowlist first, then append any + # allowlisted models the live endpoint didn't return. Custom + # providers (esp. New-API-style gateways) expose their ENTIRE + # catalog via /v1/models — including image/audio models that + # are not chat models. The picker must not surface models the + # user never declared in config.yaml. Only a NON-EMPTY explicit + # ``models`` allowlist gates the filter — a provider configured + # with just a singular ``model`` (no ``models`` list) keeps the + # unfiltered live catalog. When no allowlist is configured, + # return the live list as-is (preserves the pre-filter + # discovery behaviour). + # + # An empty allowlist (``models: []``, ``models: "[]"``, or a + # value that decodes to no usable ids) is deliberately treated + # as "not configured", NOT as "allow nothing". Gating on + # declared-ness instead would emit an EMPTY picker and make the + # provider unselectable — a harder failure than surfacing a few + # extra models, and unrecoverable from the UI because + # ``custom_providers`` is hand-edited in config.yaml (the WebUI + # has no write path for it). ``[]`` in practice means a + # leftover/placeholder key, not an intentional deny-all, and no + # deny-all use case exists: a provider the user wants hidden is + # removed from ``custom_providers`` outright. if ids: - _live_set = set(ids) - for _cid in _config_ids: - if _cid not in _live_set: - ids.append(_cid) + if _allowlist_ids: + _allowlist_set = set(_allowlist_ids) + ids = [m for m in ids if m in _allowlist_set] or [] + for _aid in _allowlist_ids: + if _aid not in ids: + ids.append(_aid) else: - ids = list(_config_ids) + ids = list(_allowlist_ids or _config_ids) # ── OpenAI-compat live fetch fallback ────────────────────────────────── # When provider_model_ids() is unavailable or returns [] for a provider @@ -22843,6 +23166,54 @@ def _prepare_chat_start_session_for_stream( s.save() +def _cleanup_chat_start_launch_failure(session, stream_id: str) -> None: + """Release state registered before a worker thread successfully starts.""" + clear_session_writeback_owner_if_owned(session.session_id, stream_id) + unregister_stream_owner(stream_id) + with STREAMS_LOCK: + STREAMS.pop(stream_id, None) + STREAM_GOAL_RELATED.pop(stream_id, None) + # The session-field reset needs the same concurrency discipline as the + # registry half: hold the per-session lock and re-resolve the canonical + # session before clearing anything. Mutating the passed-in stale object + # could wipe a concurrent successor turn's pending fields, and saving it + # could resurrect a session deleted while the launch was failing. Same + # pattern as the #1533 race fix (routes.py:3077) and the anchor-scene + # write guard (routes.py:5140). + # + # This runs while the original launch failure is being handled, so it must + # never raise. Lock acquisition and session resolution can fail on their own + # (I/O, deserialization), and an escaping error here would mask the launch + # failure the caller is about to report while leaving the reset half done. + try: + with _get_session_agent_lock(session.session_id): + try: + canonical = get_session(session.session_id) + except KeyError: + return # session deleted while the thread launch was failing + if getattr(canonical, "active_stream_id", None) != stream_id: + return # a successor turn already owns the session + canonical.active_stream_id = None + canonical.pending_user_message = None + canonical.pending_attachments = [] + canonical.pending_started_at = None + canonical.pending_user_source = None + try: + canonical.save() + except Exception: + logger.debug( + "Failed to persist chat-start cleanup after worker launch failure for %s", + stream_id, + exc_info=True, + ) + except Exception: + logger.debug( + "Failed to reset session state after worker launch failure for %s", + stream_id, + exc_info=True, + ) + + def _is_hidden_empty_session(s) -> bool: return ( getattr(s, "title", "Untitled") == "Untitled" @@ -23426,6 +23797,7 @@ def _start_chat_stream_for_session( _clear_gateway_run_starting(stream_id) except Exception: logger.debug("Failed to record gateway run-start failure for stream %s", stream_id, exc_info=True) + _cleanup_chat_start_launch_failure(s, stream_id) raise response = { "stream_id": stream_id, @@ -23957,6 +24329,17 @@ def _handle_session_compression_recovery_start(handler, body): except KeyError: return bad(handler, "Session not found", 404) if not _session_visible_to_active_profile(getattr(source, "profile", None), handler): + # #7710: same contract as the detail-load endpoint — 409 + # ``session_profile_mismatch`` for a known other profile, + # 404 only for the None-profile self-heal path. + _recovery_session_profile = getattr(source, "profile", None) + if _recovery_session_profile: + return j(handler, { + "error": "Session belongs to a different profile", + "code": "session_profile_mismatch", + "session_id": sid, + "profile": _recovery_session_profile, + }, status=409) return bad(handler, "Session not found", 404) recovery = compression_recovery_payload_for_session(source) if not recovery: @@ -24356,8 +24739,29 @@ def _handle_chat_start(handler, body, diag=None): # Empty placeholders can still be retagged when the # requested profile matches the active request profile. s.profile = requested_profile + elif session_profile: + # #7710: known other profile → 409 ``session_profile_mismatch`` + # so the client can offer to switch to it (#5419). + # 404 is preserved only for the None-profile + # (unknown/legacy) self-heal case. + return j(handler, { + "error": "Session belongs to a different profile", + "code": "session_profile_mismatch", + "session_id": body.get("session_id", ""), + "profile": session_profile, + }, status=409) else: return bad(handler, "Session not found", 404) + # Resolve durable rotations before any workspace/model/pending mutation. + # GET navigation adopts the tip; POST never silently replays a user turn. + from api.compression_continuation import durable_compression_continuation + sealed, continuation = durable_compression_continuation(s) + if sealed: + return j(handler, { + "error": "This session was compressed. Open its continuation before sending.", + "code": "session_rotated", + "continuation_session_id": continuation, + }, status=409) regeneration = None if body.get("regenerate") is True: if any(key in body for key in ("message", "attachments", "keep_count", "prompt", "prompt_index")): @@ -26202,7 +26606,6 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str, run_id: st found_target = False gateway_keys = [] local_gateway_approval_id = "" - gateway_head_matches_target = False with _lock: reconcile_gateway_pending_mirror_locked(sid) queue = _pending.get(sid) @@ -26274,9 +26677,6 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str, run_id: st gw_data = getattr(gw_entry, "data", None) or {} gw_approval_id = str(gw_data.get("approval_id") or "").strip() gw_run_id = str(gw_data.get("run_id") or "").strip() - gateway_head_matches_target = bool( - run_id and gw_approval_id == approval_id and gw_run_id == run_id - ) if gw_approval_id == approval_id and (not run_id or gw_run_id == run_id): local_gateway_approval_id = approval_id elif not run_id and found_target and pending: @@ -26337,8 +26737,10 @@ def _resolve_approval_legacy(sid: str, approval_id: str, choice: str, run_id: st local_gateway_resolved, _head, _total = resolve_gateway_pending_local( sid, local_gateway_approval_id, choice ) - elif approval_id and found_target and run_id and gateway_head_matches_target: - gateway_resolved = resolve_gateway_approval(sid, choice, resolve_all=False) or 0 + elif approval_id and found_target and run_id: + gateway_resolved, _head, _total = resolve_gateway_pending_run( + sid, approval_id, run_id, choice + ) elif not approval_id: gateway_resolved = resolve_gateway_approval(sid, choice, resolve_all=False) or 0 # Keep the historical no-id response path truthy for old clients/tests while @@ -28576,6 +28978,16 @@ def _handle_session_import_cli(handler, body): if requested_profile and not _profiles_match(existing_profile, requested_profile): return bad(handler, "Session not found in CLI store", 404) elif not _session_visible_to_active_profile(existing_profile, handler): + # #7710: same contract as the detail-load endpoint — + # 409 ``session_profile_mismatch`` for a known other + # profile, 404 only for the None-profile self-heal path. + if existing_profile: + return j(handler, { + "error": "Session belongs to a different profile", + "code": "session_profile_mismatch", + "session_id": sid, + "profile": existing_profile, + }, status=409) return bad(handler, "Session not found in CLI store", 404) refresh_profile = requested_profile or existing_profile cli_meta = _resolve_cli_import_metadata( diff --git a/api/run_journal.py b/api/run_journal.py index ab8d0a7e..48cd8ef7 100644 --- a/api/run_journal.py +++ b/api/run_journal.py @@ -45,6 +45,14 @@ TERMINAL_SSE_EVENTS = frozenset({"done", "cancel", "apperror", "error", "stream_ SSE_RELAY_CLOSE_EVENTS = frozenset({"stream_end", "cancel", "apperror", "error"}) # Back-compat alias used by older call sites / tests. _TERMINAL_SSE_EVENTS = TERMINAL_SSE_EVENTS +# Events that are live-UI-only telemetry with no recovery value in the run +# journal. They are skipped at WRITE time (never durably journaled, so they +# cannot bloat the journal on marathon runs) and filtered at REPLAY time (so +# legacy journals that already contain a backlog never stream it to a +# reconnecting browser tab). Readers deliberately do NOT filter: cursor math +# (``cursor_event_missing`` bound) and the offline-gap coverage check count +# journal seqs and must keep seeing every row. +REPLAY_SKIPPED_SSE_EVENTS = frozenset({"metering"}) _FSYNC_MODE_ENV = "HERMES_WEBUI_RUN_JOURNAL_FSYNC" _FSYNC_MODE_EAGER = "eager" _FSYNC_MODE_TERMINAL_ONLY = "terminal-only" @@ -443,25 +451,45 @@ class RunJournalWriter: self.session_id = _validate_id(session_id, "session_id") self.run_id = _validate_id(run_id, "run_id") self.session_dir = Path(session_dir) if session_dir is not None else None - self._path = _run_path(self.session_id, self.run_id, session_dir=self.session_dir) - self._lock = _lock_for(self._path) - def append_sse_event(self, event_name: str, payload=None) -> dict: - # Draw from the shared module-level seq cache under the per-path lock so - # this writer and any direct append_run_event() call on the same path - # agree on one monotonic, gapless sequence. - with self._lock: - seq = _reserve_next_seq(self._path) + def append_sse_event(self, event_name: str, payload=None) -> dict | None: + # Live-UI-only telemetry (metering) has no recovery value in the journal: + # nothing reads those rows back for recovery, and journaling them at ~10 Hz + # on marathon runs balloons the durable file (12+ MB of a single 18 MB run + # was metering). Skip the write entirely and return None so callers' + # journal-id plumbing (``(journaled or {}).get("event_id")``) is untouched. + # Not reserving a seq keeps the remaining journaled seqs contiguous, which + # the offline-gap coverage and replay-cursor contiguity checks rely on. + if str(event_name or "").strip() in REPLAY_SKIPPED_SSE_EVENTS: + return None + # Allocate the sequence inside the same per-path transaction that writes + # the row. Reserving here, then releasing the lock before append, lets a + # concurrent writer put a higher sequence on disk first. return append_run_event( self.session_id, self.run_id, event_name, payload or {}, session_dir=self.session_dir, - seq=seq, ) +def journal_replay_visible(event) -> bool: + """Return True when a journal row should be streamed to a reconnecting tab. + + Live-UI-only telemetry rows (see ``REPLAY_SKIPPED_SSE_EVENTS``) carry no + recovery value — replaying a metering backlog only re-paints a stale TPS + number while multiplying the reconnect burst size. Writers no longer journal + them, but legacy journals may already contain them, so the replay emit sites + filter through this predicate. Non-dict rows are passed through (visible) so + an unexpected shape can never silently swallow user-visible output. + """ + if not isinstance(event, dict): + return True + name = str(event.get("event") or event.get("type") or "") + return name not in REPLAY_SKIPPED_SSE_EVENTS + + def read_run_events( session_id: str, run_id: str, @@ -587,6 +615,27 @@ def find_run_summary(run_id: str, *, session_dir: Path | None = None) -> dict | return None +def find_run_file(run_id: str, *, session_dir: Path | None = None) -> tuple[str, Path] | None: + """Locate a run journal file by run id WITHOUT parsing its body. + + Hot callers that immediately read the full journal (the live-snapshot + rebuild) must not pay :func:`find_run_summary`'s full-file parse first: + on a long live run the file holds tens of thousands of rows and parsing + it twice per rebuild dominated the snapshot cost. Returns + ``(session_id, path)`` for the first match, or ``None`` when the run id + is invalid or no journal exists. + """ + try: + rid = _validate_id(run_id, "run_id") + except ValueError: + return None + root = Path(session_dir) if session_dir is not None else _default_session_dir() + journal_root = root / RUN_JOURNAL_DIR_NAME + for path in journal_root.glob(f"*/{rid}.jsonl"): + return path.parent.name, path + return None + + def read_session_run_events( session_id: str, *, diff --git a/api/streaming.py b/api/streaming.py index 71cf8208..dd15620e 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -67,6 +67,7 @@ from api.helpers import ( ) from api.compression_anchor import is_context_compression_marker, visible_messages_for_anchor from api.compression_recovery import stamp_compression_exhausted_recovery +from api.gateway_chat import WEBUI_LOCAL_CHAT_BACKEND from api.metering import meter from api.run_journal import RunJournalWriter from api.todo_state import attach_todo_state, emit_todo_state @@ -196,41 +197,156 @@ def _compact_for_echo_compare(value: str) -> str: return re.sub(r'\s+', '', str(value or '')) -def _strip_compact_echo_suffix(value: str, suffix: str, *, search_window: int = 4096) -> tuple[str, bool]: +class _CompactEchoIndex: + """Incremental whitespace-folded index over a growing text buffer. + + The journal-rebuild echo check runs once per interim event against the + whole accumulated reasoning transcript. Rescanning the raw buffer each + time walks every whitespace character between the tail and the first + non-whitespace character — on a whitespace-heavy transcript that span is + megabytes, and the per-interim walk turns the replay quadratic (the + ``_find_compact_echo_suffix_start`` shape measured ~55x slower than + master on a production-shaped journal in the #7569 review). + + This class keeps the folded view and the raw cut offsets *incrementally*: + + * ``append`` folds each new chunk once and records, per folded character, + its raw index — so the cost of a chunk is proportional to the chunk, + never to the buffer. + * ``matches_tail`` compares only the candidate against the end of the + folded view: O(len(candidate)), with no raw-text walk at all. + * ``cut_to`` converts a folded length back to a raw index by bisecting + the recorded offsets, so one call returns the same cut point the raw + backward walk would have produced. + + ``str.isspace`` is used for folding instead of the ``\\s`` pattern used + by :func:`_compact_for_echo_compare`. The two agree on every Unicode code + point, so the indexed view and the regex-folded view stay consistent. + """ + + __slots__ = ('_compact', '_offsets', '_raw_len') + + def __init__(self) -> None: + self._compact: list[str] = [] + self._offsets: list[int] = [] + self._raw_len = 0 + + def append(self, text: str) -> None: + """Fold ``text`` onto the end of the index (one pass, no rescan).""" + raw = str(text or '') + if not raw: + return + compact = self._compact + offsets = self._offsets + base = self._raw_len + for i, ch in enumerate(raw): + if not ch.isspace(): + compact.append(ch) + offsets.append(base + i) + self._raw_len = base + len(raw) + + def reset(self) -> None: + """Drop all indexed state (used after the raw text is truncated).""" + self._compact.clear() + self._offsets.clear() + self._raw_len = 0 + + @property + def compact_length(self) -> int: + return len(self._compact) + + def matches_tail(self, suffix: str) -> bool: + """True when the folded ``suffix`` equals the folded tail.""" + candidate = _compact_for_echo_compare(suffix) + if not candidate: + return False + n = len(candidate) + if n > len(self._compact): + return False + start = len(self._compact) - n + compact = self._compact + for i in range(n): + if compact[start + i] != candidate[i]: + return False + return True + + def cut_to(self, suffix: str) -> int | None: + """Raw index at which the echo of ``suffix`` starts, else ``None``. + + The returned index is the first raw character of the echo, matching + the leftmost-cut semantics of the raw backward walk. Trailing + whitespace before the echo is left to the caller's ``rstrip``. + """ + candidate = _compact_for_echo_compare(suffix) + if not candidate: + return None + n = len(candidate) + if n > len(self._compact): + return None + start = len(self._compact) - n + compact = self._compact + for i in range(n): + if compact[start + i] != candidate[i]: + return None + return self._offsets[start] + + def compact_view(self) -> str: + """The folded text (materialized on demand — never kept as a string).""" + return ''.join(self._compact) + + +def _find_compact_echo_suffix_start(value: str, suffix: str) -> int | None: + """Return the index where a whitespace-folded ``suffix`` starts at the + end of ``value``, or ``None`` when the tail does not echo it. + + The match walks ``value`` and ``suffix`` from the end, skipping + whitespace in ``value``; only the echo span itself is inspected, so the + cost is linear in the echo length and allocation-free. Unlike a fixed + fold window, the walk cannot miss a compact-equivalent suffix whose raw + span is stretched by interior whitespace. + + ``str.isspace`` is used for the walk instead of the ``\\s`` pattern used + by :func:`_compact_for_echo_compare`. The two agree on every Unicode code + point, so the folded view and the walk stay consistent. + """ + candidate = _compact_for_echo_compare(suffix) + if not candidate: + return None + i = len(value) - 1 + j = len(candidate) - 1 + while j >= 0: + while i >= 0 and value[i].isspace(): + i -= 1 + if i < 0 or value[i] != candidate[j]: + return None + i -= 1 + j -= 1 + return i + 1 + + +def _strip_compact_echo_suffix(value: str, suffix: str) -> tuple[str, bool]: """Remove ``suffix`` from ``value`` when they match after whitespace folding. - The search window is folded once and the cut point is then located by - walking backwards across the echo itself. The previous implementation - probed every candidate cut index and re-folded the whole remaining tail for - each probe, which is quadratic in the window size: a 6000-character final - message cost seconds of CPU, held under the GIL, stalling every other - stream in the process. + The cut point is located by the same backward walk as + :func:`_find_compact_echo_suffix_start`: no fixed search window is + involved, so a compact-equivalent suffix is removed no matter how much + interior whitespace stretches its raw span. The previous windowed + implementation folded a bounded tail on every call; its retired probing + variant re-folded the remaining tail per candidate cut index, which is + quadratic: a 6000-character final message cost seconds of CPU, held under + the GIL, stalling every other stream in the process. - ``str.isspace`` is used for the backwards walk instead of the ``\\s`` - pattern used by :func:`_compact_for_echo_compare`. The two agree on every - Unicode code point, so the folded view and the walk stay consistent. + Whitespace sitting between the kept text and the echo is removed by + ``rstrip``, which lands on the same result as the leftmost cut index the + probing loop used to return. """ raw = str(value or '') - candidate = _compact_for_echo_compare(suffix) - if not raw or not candidate: + if not raw: return raw, False - tail = raw[-max(len(str(suffix or '')) * 3, search_window):] - offset = len(raw) - len(tail) - compact_tail = _compact_for_echo_compare(tail) - if len(candidate) > len(compact_tail) or not compact_tail.endswith(candidate): + start = _find_compact_echo_suffix_start(raw, suffix) + if start is None: return raw, False - # Consume exactly as many non-whitespace characters as the folded suffix - # holds; ``idx`` then sits on the first character of the echo. Whitespace - # sitting between the kept text and the echo is removed by ``rstrip``, - # which is why this lands on the same result as the leftmost cut index the - # probing loop used to return. - remaining = len(candidate) - idx = len(tail) - while remaining and idx: - idx -= 1 - if not tail[idx].isspace(): - remaining -= 1 - return raw[: offset + idx].rstrip(), True + return raw[:start].rstrip(), True def _redacted_session_payload_with_full_messages(session, *, tool_calls=None) -> dict | None: @@ -779,6 +895,32 @@ def _is_fallback_lifecycle_message(kind: str, message: str) -> bool: ) +# Session turn-lease notices emitted by the Agent (agent/turn_facade_lease.py) +# while another Hermes process (gateway, CLI, cron) holds this session's turn +# lease. Emitted via ``_emit_status`` (kind ``lifecycle``) while waiting and on +# admission, and via ``_emit_warning`` (kind ``warn``) when the wait times out +# and the message was not processed. +_SESSION_LEASE_WAIT_MARKERS = ( + 'another hermes process is using this session', + 'still waiting for the other hermes process', + 'another hermes process kept this session busy', + 'session is free; loading the latest transcript', +) + + +def _is_session_lease_wait_message(kind: str, message: str) -> bool: + """Return True for Agent session turn-lease wait notices. + + Classification keys on the Agent status kind (``lifecycle`` / ``warn``) so + user-authored text can never be promoted to a warning. + """ + k = str(kind or '').strip().lower() + if k not in ('lifecycle', 'warn'): + return False + m = str(message or '').strip().lower() + return any(marker in m for marker in _SESSION_LEASE_WAIT_MARKERS) + + def _is_agent_compression_start_status(kind: str, message: str) -> bool: """Return True only for real Hermes context-compression start notices. @@ -4378,9 +4520,10 @@ def _title_language_mismatch(user_text: str, title: str) -> bool: short and frequently embed a borrowed Latin technical term (e.g. a CJK title containing the word "Python"), the title side uses a proportion threshold (>=35% of the title's alphabetic characters in a non-start - script, min 2 chars) rather than a strict majority -- so a CJK title with - one English word still trips, while an English title with a single - foreign place-name does not. + script, min 2 chars) rather than a strict majority. CJK titles with + borrowed Latin terms are allowed when the title also contains CJK + characters (#7693), but pure-Latin titles for CJK conversations are + still rejected. 2. The legacy German-start → English-title heuristic, preserved verbatim so the original behavior keeps working for same-script (latin) drift that the script check can't see. @@ -4390,13 +4533,26 @@ def _title_language_mismatch(user_text: str, title: str) -> bool: return False # (1) Cross-script mismatch — language-agnostic. + # CJK text routinely borrows Latin product/technical terms (e.g. "WeChat + # Pay", "Python", "ProRes RAW"), so when the user writes in CJK, Latin + # characters in the title are acceptable as long as the title also + # contains CJK — i.e. the title is genuinely mixed, not pure drift. + # A pure-Latin title for a CJK conversation is still rejected. + # Unrelated scripts (Cyrillic, Arabic, Greek …) are always flagged. user_script = _dominant_script(user_text) if user_script: title_counts = _script_counts(candidate) title_total = sum(title_counts.values()) if title_total >= 2: for script, n in title_counts.items(): - if script != user_script and n >= 2 and (n / title_total) >= 0.35: + if script == user_script: + continue + # When user writes in CJK, Latin in the title is a borrowed + # term as long as the title also contains CJK characters. + if user_script == 'cjk' and script == 'latin': + if title_counts.get('cjk', 0) >= 2: + continue + if n >= 2 and (n / title_total) >= 0.35: return True # (2) Legacy same-script German→English heuristic. @@ -7949,7 +8105,7 @@ def _extract_tool_calls_from_messages(messages, live_tool_calls=None): pending_names[tid] = part.get('name', '') pending_args[tid] = part.get('input', {}) pending_asst_idx[tid] = msg_idx - for tc in m.get('tool_calls', []): + for tc in m.get('tool_calls') or []: if not isinstance(tc, dict): continue tid = tc.get('id', '') or tc.get('call_id', '') @@ -9169,7 +9325,32 @@ def _run_agent_streaming( """ _turn_route_model = model _turn_route_provider = model_provider + cancel_event = threading.Event() q = peek_stream(stream_id) + if q is not None: + # Snapshot lookup is not admission: Stop can detach the stream before + # we publish the initial run. Register ownership and its retained cancel + # signal on the same STREAMS_LOCK edge used by cancellation. + with STREAMS_LOCK: + cancel_event = CANCEL_FLAGS.get(stream_id, cancel_event) + if stream_id not in STREAMS or cancel_event.is_set(): + q = None + else: + CANCEL_FLAGS[stream_id] = cancel_event + STREAM_PARTIAL_TEXT[stream_id] = '' + STREAM_REASONING_TEXT[stream_id] = '' + STREAM_LIVE_TOOL_CALLS[stream_id] = [] + register_active_run( + stream_id, + session_id=session_id, + started_at=time.time(), + phase="starting", + workspace=str(workspace), + model=model, + provider=model_provider, + ephemeral=bool(ephemeral), + backend=WEBUI_LOCAL_CHAT_BACKEND, + ) if q is None: # The stream was cancelled before the worker started; the route layer # already registered the stream owner, so release it here to avoid @@ -9183,16 +9364,6 @@ def _run_agent_streaming( exc_info=True, ) return - register_active_run( - stream_id, - session_id=session_id, - started_at=time.time(), - phase="starting", - workspace=str(workspace), - model=model, - provider=model_provider, - ephemeral=bool(ephemeral), - ) try: run_journal = RunJournalWriter(session_id, stream_id) except Exception: @@ -9224,14 +9395,6 @@ def _run_agent_streaming( # profile's mcp_servers because os.environ['HERMES_HOME'] hadn't been # rewritten yet. See https://github.com/nesquena/hermes-webui/issues/1968. - # Sprint 10: create a cancel event for this stream - cancel_event = threading.Event() - with STREAMS_LOCK: - CANCEL_FLAGS[stream_id] = cancel_event - STREAM_PARTIAL_TEXT[stream_id] = '' # start accumulating partial text (#893) - STREAM_REASONING_TEXT[stream_id] = '' # start accumulating reasoning trace (#1361 §A) - STREAM_LIVE_TOOL_CALLS[stream_id] = [] # start accumulating tool calls (#1361 §B) - agent = None _live_prompt_estimate_tokens = [0] _live_prompt_exact_tokens = [0] @@ -9533,8 +9696,42 @@ def _run_agent_streaming( _metering_thread = threading.Thread(target=_metering_ticker, daemon=True) _success_writeback_committed = False + _steer_settled = False + _returned_pending_steer = [] + + def _remember_pending_steer_result(value): + text = value.get('pending_steer') if isinstance(value, dict) else None + if isinstance(text, str) and text: + _returned_pending_steer.append(text) + + def _settle_pending_steer(): + # Single terminal boundary for success, returned errors, exceptions and + # successful self-heal. Admission closes before the last slot consumer; + # registry locks never cover SSE writes or Agent drain callbacks. + nonlocal _steer_settled + with STREAMS_LOCK: + if _steer_settled: + return + _steer_settled = True + target = AGENT_INSTANCES.get(stream_id) or agent + if stream_id in STREAMS and not cancel_event.is_set(): + update_active_run(stream_id, phase="finalizing") + leftovers = list(_returned_pending_steer) + try: + drain = getattr(target, '_drain_pending_steer', None) + text = drain() if callable(drain) else None + if text: + leftovers.append(str(text)) + except Exception: + logger.debug("Failed to drain pending steer for session %s", session_id) + if leftovers: + put('pending_steer_leftover', { + 'session_id': session_id, 'text': '\n'.join(leftovers), + }) def put(event, data): + if event in ('done', 'apperror', 'stream_end'): + _settle_pending_steer() # If cancelled, drop all further events except the cancel event itself if cancel_event.is_set() and not _success_writeback_committed and event not in ('cancel', 'apperror'): return @@ -9558,7 +9755,7 @@ def _run_agent_streaming( except Exception: logger.debug("Failed to note event_id %s for stream %s", event_id, stream_id, exc_info=True) try: - queue_item = (event, data, event_id) if event_id and hasattr(q, "subscribe_with_snapshot") else (event, data) + queue_item = (event, data, event_id) if hasattr(q, "subscribe_with_snapshot") else (event, data) q.put_nowait(queue_item) except Exception: logger.debug("Failed to put event to queue") @@ -9605,6 +9802,11 @@ def _run_agent_streaming( return # Pass through rate-limit and fallback messages so the frontend can # show them as warnings via the existing messages.js 'warning' listener. + # Session turn-lease waits (another Hermes process owns this session) + # use the same channel so a delayed turn explains itself. + if _is_session_lease_wait_message(_kind, _message): + put('warning', {'type': 'session_lease_wait', 'message': _message}) + return _is_fallback_notice = _is_fallback_lifecycle_message(_kind, _message) if _is_fallback_notice: put('warning', {'type': 'fallback', 'message': _message}) @@ -9620,6 +9822,84 @@ def _run_agent_streaming( _streaming_skill_home_snapshot = None _restore_streaming_skill_home_modules = False _acquired_streaming_skill_home_patch_lock = False + def _register_agent_if_current(candidate, cache_signature=None): + nonlocal agent + # Every constructor, including both credential self-heal branches, must + # share the Stop publication edge. CANCEL_FLAGS may already be detached; + # the worker-retained event and stream membership remain authoritative. + with STREAMS_LOCK: + cancelled = cancel_event.is_set() or stream_id not in STREAMS + if not cancelled: + AGENT_INSTANCES[stream_id] = candidate + # Check and publication share one admission, not a check followed + # by an unlocked cache/lifecycle write that could undo Stop. + if cache_signature is not None: + from api.config import SESSION_AGENT_CACHE, SESSION_AGENT_CACHE_LOCK + with SESSION_AGENT_CACHE_LOCK: + SESSION_AGENT_CACHE[session_id] = (candidate, cache_signature) + SESSION_AGENT_CACHE.move_to_end(session_id) + if not ephemeral: + try: + # Pure in-memory bookkeeping; no Agent/provider call. + from api.session_lifecycle import register_agent + register_agent(session_id, candidate) + except Exception: + logger.debug("Lifecycle register_agent failed for session %s", session_id, exc_info=True) + if not cancelled: + return True + # A cache hit is a borrowed reusable object, not this worker's private + # candidate. Stop may already have let a successor reuse it. Rejected + # cache-hit admission must not interrupt that other turn. + if cache_signature is None and not ephemeral: + if agent is candidate: + agent = None + return False + # Newly constructed candidates were never published on this failed + # admission; they cannot have been borrowed by a successor from us. + # No Agent call, session persistence or SSE write under registry locks. + try: + candidate.interrupt("Cancelled before start") + except Exception: + logger.debug("Failed to interrupt cancelled candidate agent") + return False + + def _agent_can_invoke(candidate): + nonlocal agent + # Prompt preparation and LRU cleanup can yield after registration. Admit + # each invocation again at its point of use; never hold registry locks + # across run_conversation or interrupt. A later Stop interrupts the + # already-admitted invocation through the registered Agent as usual. + from api.config import ( + SESSION_AGENT_CACHE, SESSION_AGENT_CACHE_LOCK, + SESSION_WRITEBACK_OWNERS, SESSION_WRITEBACK_OWNERS_LOCK, + ) + with STREAMS_LOCK: + current = (not cancel_event.is_set() and stream_id in STREAMS + and AGENT_INSTANCES.get(stream_id) is candidate) + if not current: + if AGENT_INSTANCES.get(stream_id) is candidate: + AGENT_INSTANCES.pop(stream_id, None) + # Agent identity is not turn identity: a successor can reuse the + # same cached object. Use the existing cancellation-surviving + # ownership record, holding its lock through cache retirement. + # Missing ownership also fails closed (successor may have ended). + with SESSION_WRITEBACK_OWNERS_LOCK: + if SESSION_WRITEBACK_OWNERS.get(session_id) == stream_id: + with SESSION_AGENT_CACHE_LOCK: + entry = SESSION_AGENT_CACHE.get(session_id) + if entry and entry[0] is candidate: + SESSION_AGENT_CACHE.pop(session_id, None) + # Dirty memory segments keep their original owner. + from api.session_lifecycle import unregister_agent + unregister_agent(session_id) + # This invocation never started. Stop owns any required interrupt of the + # running Agent; a late duplicate interrupt could hit its new borrower. + # Drop our local borrowed handle too: final Steer drain must not reach + # the successor through the old worker's fallback `agent` reference. + if not current and agent is candidate: + agent = None + return current + # Initialised here (before any code that may raise) so the outer `finally` # block can safely check `if _checkpoint_stop is not None` even when an # exception fires before the checkpoint thread is created (Issue #765). @@ -9964,6 +10244,14 @@ def _run_agent_streaming( # (#3587) replaces the flat _reasoning_text string so each intermediate # assistant turn (before tool calls) keeps its own reasoning segment. _reasoning_segments: dict = {} + # Incremental folded indexes mirroring each reasoning buffer. The + # live echo strip consults them instead of re-walking the raw + # buffers: an unbounded raw scan of the growing transcript is + # quadratic on whitespace-heavy output (#7569 review). Each index + # is fed on every append and re-indexed after a strip. + _reasoning_segment_indexes: dict = {} + _stream_reasoning_index = _CompactEchoIndex() + _reasoning_buffer_index = _CompactEchoIndex() _current_reasoning_idx = 0 _tool_boundary_advanced = False _live_tool_calls = [] # tool progress fallback when final messages omit tool IDs @@ -9986,6 +10274,10 @@ def _run_agent_streaming( if _reasoning_buffer[0]: put('reasoning', {'text': _reasoning_buffer[0]}) _reasoning_buffer[0] = '' + # The folded index mirrors this buffer 1:1 — dropping the + # text without dropping the index would leave it describing + # text that is no longer there. + _reasoning_buffer_index.reset() def _emit_metering(): @@ -10025,30 +10317,37 @@ def _run_agent_streaming( nonlocal _reasoning_segments removed = False if stream_id in STREAM_REASONING_TEXT: - next_text, did_remove = _strip_compact_echo_suffix( - STREAM_REASONING_TEXT.get(stream_id, ''), - text, - ) - if did_remove: + cut = _stream_reasoning_index.cut_to(text) + if cut is not None: + next_text = STREAM_REASONING_TEXT.get(stream_id, '')[:cut].rstrip() STREAM_REASONING_TEXT[stream_id] = next_text + _stream_reasoning_index.reset() + _stream_reasoning_index.append(next_text) removed = True - next_buffer, did_remove_buffer = _strip_compact_echo_suffix(_reasoning_buffer[0], text) - if did_remove_buffer: + cut = _reasoning_buffer_index.cut_to(text) + if cut is not None: + next_buffer = _reasoning_buffer[0][:cut].rstrip() _reasoning_buffer[0] = next_buffer + _reasoning_buffer_index.reset() + _reasoning_buffer_index.append(next_buffer) removed = True for idx in (_current_reasoning_idx, _current_reasoning_idx - 1): if idx not in _reasoning_segments: continue - next_segment, did_remove_segment = _strip_compact_echo_suffix( - _reasoning_segments.get(idx, ''), - text, - ) - if not did_remove_segment: + segment_index = _reasoning_segment_indexes.get(idx) + if segment_index is None: continue + cut = segment_index.cut_to(text) + if cut is None: + continue + next_segment = _reasoning_segments.get(idx, '')[:cut].rstrip() if next_segment: _reasoning_segments[idx] = next_segment + segment_index.reset() + segment_index.append(next_segment) else: _reasoning_segments.pop(idx, None) + _reasoning_segment_indexes.pop(idx, None) removed = True break return removed @@ -10112,15 +10411,21 @@ def _run_agent_streaming( _reasoning_segments[_current_reasoning_idx] = ( _reasoning_segments.get(_current_reasoning_idx, '') + reasoning_delta ) + # Keep the folded index in step with the segment text. + _reasoning_segment_indexes.setdefault( + _current_reasoning_idx, _CompactEchoIndex() + ).append(reasoning_delta) # Mirror full concatenation to shared dict so cancel_stream() can persist # it (#1361 §A). Cancel only creates one partial message, so the flat # concatenation is correct there. # Lock-free GIL-atomic mirror — see the STREAMS_LOCK contract in on_token. if stream_id in STREAM_REASONING_TEXT: STREAM_REASONING_TEXT[stream_id] += reasoning_delta + _stream_reasoning_index.append(reasoning_delta) # Accumulate into a coalescing buffer so every delta reaches the # browser — reasoning deltas are incremental, not idempotent. _reasoning_buffer[0] += reasoning_delta + _reasoning_buffer_index.append(reasoning_delta) # Throttle reasoning SSE events to ~10 Hz to avoid overwhelming the # frontend renderer. Each event triggers _parseStreamState() which # scans the full accumulated text — 10k+ reasoning tokens/second @@ -10131,6 +10436,10 @@ def _run_agent_streaming( _reasoning_last_put[0] = now put('reasoning', {'text': _reasoning_buffer[0]}) _reasoning_buffer[0] = '' + # The folded index mirrors this buffer 1:1 — dropping the + # text without dropping the index would leave it describing + # text that is no longer there. + _reasoning_buffer_index.reset() # Track reasoning deltas in the meter so live TPS reflects all AI output. _metering_reasoning_deltas[0] += 1 meter().record_reasoning(stream_id, _metering_reasoning_deltas[0]) @@ -10239,10 +10548,14 @@ def _run_agent_streaming( _reasoning_segments[_current_reasoning_idx] = ( _reasoning_segments.get(_current_reasoning_idx, '') + reason_delta ) + _reasoning_segment_indexes.setdefault( + _current_reasoning_idx, _CompactEchoIndex() + ).append(reason_delta) # Mirror full concatenation to shared dict (#1361 §A) # Lock-free GIL-atomic mirror — see STREAMS_LOCK contract in on_token. if stream_id in STREAM_REASONING_TEXT: STREAM_REASONING_TEXT[stream_id] += reason_delta + _stream_reasoning_index.append(reason_delta) put('reasoning', {'text': reason_delta}) _metering_reasoning_deltas[0] += 1 meter().record_reasoning(stream_id, _metering_reasoning_deltas[0]) @@ -10809,6 +11122,7 @@ def _run_agent_streaming( # ── Agent cache: reuse across messages in the same session ── # Mirrors gateway _agent_cache. Keeps _user_turn_count alive so # injectionFrequency: "first-turn" actually suppresses after turn 1. + _cache_new_agent = False if ephemeral: agent = _AIAgent(**_agent_kwargs) logger.debug('[webui] Created ephemeral agent for session %s', session_id) @@ -10851,14 +11165,6 @@ def _run_agent_streaming( session_id, _cached_agent_session_identity(_cached_agent), ) - if agent is not None: - # Reopened/cache-hit sessions must register the agent - # so later lifecycle commits can find it. - try: - from api.session_lifecycle import register_agent - register_agent(session_id, agent) - except Exception: - logger.debug("Lifecycle register_agent failed for cached session %s", session_id, exc_info=True) if _identity_mismatch_entry is not None: try: @@ -10928,77 +11234,64 @@ def _run_agent_streaming( agent._interrupt_message = None else: agent = _AIAgent(**_agent_kwargs) - # Register the new agent with the memory lifecycle so - # its commit_memory_session() can be found later. - try: - from api.session_lifecycle import register_agent - register_agent(session_id, agent) - except Exception: - logger.debug("Lifecycle register_agent failed for new session %s", session_id, exc_info=True) - _evicted_items = [] - # Snapshot the set of session_ids with a LIVE agent worker - # BEFORE taking SESSION_AGENT_CACHE_LOCK, so LRU eviction never - # closes an agent mid-run AND we never nest ACTIVE_RUNS_LOCK - # inside SESSION_AGENT_CACHE_LOCK (avoids any lock-ordering - # deadlock). A cancel/reconnect can drop STREAMS while the - # worker is still unwinding or blocked in a provider call, so - # ACTIVE_RUNS (worker lifecycle) is the authoritative liveness - # signal, not STREAMS. (#3536 review round 2) - _active_sids = set() - try: - from api.config import ACTIVE_RUNS, ACTIVE_RUNS_LOCK - with ACTIVE_RUNS_LOCK: - for _entry in (ACTIVE_RUNS or {}).values(): - _sid = (_entry or {}).get("session_id") - if _sid: - _active_sids.add(_sid) - except Exception: - _active_sids = set() - with SESSION_AGENT_CACHE_LOCK: - SESSION_AGENT_CACHE[session_id] = (agent, _agent_sig) - SESSION_AGENT_CACHE.move_to_end(session_id) # LRU: mark as recently used - from api.config import SESSION_AGENT_CACHE_MAX - # Evict the oldest INACTIVE entries first. Walk LRU order - # (front = oldest); skip any session with a live run. If - # every over-cap entry is active, leave the cache - # temporarily above cap rather than close a live worker's - # agent — a later insertion/finalization trims it once the - # run ends. - while len(SESSION_AGENT_CACHE) > SESSION_AGENT_CACHE_MAX: - _evictable_sid = None - for _sid in list(SESSION_AGENT_CACHE.keys()): - if _sid not in _active_sids: - _evictable_sid = _sid - break - if _evictable_sid is None: - break # all over-cap entries are active; defer - evicted_entry = SESSION_AGENT_CACHE.pop(_evictable_sid) - _evicted_items.append((_evictable_sid, evicted_entry)) - # Commit and close evicted agents outside the cache lock so - # concurrent cache users are not blocked by provider I/O. - for _evicted_sid, _evicted_entry in _evicted_items: - try: - _evicted_agent = _evicted_entry[0] if isinstance(_evicted_entry, tuple) else None - _close_evicted_agent_at_session_boundary(_evicted_sid, _evicted_agent) - except Exception: - logger.debug("Failed to close evicted agent for session %s", _evicted_sid, exc_info=True) - logger.debug('[webui] Evicted LRU agent from cache: %s', _evicted_sid) - logger.debug('[webui] Created new agent for session %s', session_id) + _cache_new_agent = True - # Store agent instance for cancel/interrupt propagation - with STREAMS_LOCK: - AGENT_INSTANCES[stream_id] = agent - # Check if cancel was requested during agent initialization - if stream_id in CANCEL_FLAGS and CANCEL_FLAGS[stream_id].is_set(): - # Cancel arrived during agent creation - interrupt immediately + if not _register_agent_if_current(agent, _agent_sig if _cache_new_agent else None): + with _agent_lock: + _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) + put('cancel', _cancel_event_payload('Cancelled by user')) + return + + # Cache publication is already fenced with Stop above. Keep LRU + # eviction and any memory-provider close outside the stream lock. + if _cache_new_agent: + _evicted_items = [] + # Snapshot the set of session_ids with a LIVE agent worker + # BEFORE taking SESSION_AGENT_CACHE_LOCK, so LRU eviction never + # closes an agent mid-run AND we never nest ACTIVE_RUNS_LOCK + # inside SESSION_AGENT_CACHE_LOCK (avoids any lock-ordering + # deadlock). A cancel/reconnect can drop STREAMS while the + # worker is still unwinding or blocked in a provider call, so + # ACTIVE_RUNS (worker lifecycle) is the authoritative liveness + # signal, not STREAMS. (#3536 review round 2) + _active_sids = set() + try: + from api.config import ACTIVE_RUNS, ACTIVE_RUNS_LOCK + with ACTIVE_RUNS_LOCK: + for _entry in (ACTIVE_RUNS or {}).values(): + _sid = (_entry or {}).get("session_id") + if _sid: + _active_sids.add(_sid) + except Exception: + _active_sids = set() + with SESSION_AGENT_CACHE_LOCK: + from api.config import SESSION_AGENT_CACHE_MAX + # Evict the oldest INACTIVE entries first. Walk LRU order + # (front = oldest); skip any session with a live run. If + # every over-cap entry is active, leave the cache + # temporarily above cap rather than close a live worker's + # agent — a later insertion/finalization trims it once the + # run ends. + while len(SESSION_AGENT_CACHE) > SESSION_AGENT_CACHE_MAX: + _evictable_sid = None + for _sid in list(SESSION_AGENT_CACHE.keys()): + if _sid not in _active_sids: + _evictable_sid = _sid + break + if _evictable_sid is None: + break # all over-cap entries are active; defer + evicted_entry = SESSION_AGENT_CACHE.pop(_evictable_sid) + _evicted_items.append((_evictable_sid, evicted_entry)) + # Commit and close evicted agents outside the cache lock so + # concurrent cache users are not blocked by provider I/O. + for _evicted_sid, _evicted_entry in _evicted_items: try: - agent.interrupt("Cancelled before start") + _evicted_agent = _evicted_entry[0] if isinstance(_evicted_entry, tuple) else None + _close_evicted_agent_at_session_boundary(_evicted_sid, _evicted_agent) except Exception: - logger.debug("Failed to interrupt agent before start") - with _agent_lock: - _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) - put('cancel', _cancel_event_payload('Cancelled by user')) - return + logger.debug("Failed to close evicted agent for session %s", _evicted_sid, exc_info=True) + logger.debug('[webui] Evicted LRU agent from cache: %s', _evicted_sid) + logger.debug('[webui] Created new agent for session %s', session_id) # Prepend workspace context so the agent always knows which directory # to use for file operations, regardless of session age or AGENTS.md defaults. @@ -11259,7 +11552,13 @@ def _run_agent_streaming( ) _run_conversation_kwargs["user_message"] = user_message _result_partial_pre_call_context = list(_previous_context_messages) + if not _agent_can_invoke(agent): + with _agent_lock: + _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) + put('cancel', _cancel_event_payload('Cancelled by user')) + return result = agent.run_conversation(**_run_conversation_kwargs) + _remember_pending_steer_result(result) _active_turn_identity = _resolve_active_turn_authority( _active_turn_identity, result=result, @@ -11740,8 +12039,6 @@ def _run_agent_streaming( if 'credential_pool' in _agent_params: _agent_kwargs['credential_pool'] = _runtime_bundle['credential_pool'] agent = _AIAgent(**_agent_kwargs) - with STREAMS_LOCK: - AGENT_INSTANCES[stream_id] = agent _agent_sig = _compute_agent_cache_signature( resolved_model, resolved_api_key, @@ -11758,10 +12055,11 @@ def _run_agent_streaming( profile_home=_profile_home, safe_profile_runtime_env=_safe_profile_runtime_env, ) - from api.config import SESSION_AGENT_CACHE as _SAC, SESSION_AGENT_CACHE_LOCK as _SAC_L - with _SAC_L: - _SAC[session_id] = (agent, _agent_sig) - _SAC.move_to_end(session_id) + if not _register_agent_if_current(agent, _agent_sig): + # Returned-error settlement already owns _agent_lock. + _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) + put('cancel', _cancel_event_payload('Cancelled by user')) + return # Retry the conversation once with fresh credentials _self_healed = True _token_sent = False @@ -11794,7 +12092,13 @@ def _run_agent_streaming( _result_partial_pre_call_context = list( _heal_context_messages ) + if not _agent_can_invoke(agent): + # Returned-error settlement already owns the lock. + _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) + put('cancel', _cancel_event_payload('Cancelled by user')) + return _heal_result = agent.run_conversation(**_heal_kwargs) + _remember_pending_steer_result(_heal_result) _active_turn_identity = _resolve_active_turn_authority( _active_turn_identity, result=_heal_result, @@ -12741,23 +13045,7 @@ def _run_agent_streaming( else None ) # (reasoning trace already attached + saved above, before s.save()) - # Leftover-steer delivery: if a /steer was queued (via - # api/chat/steer) but the agent finished its turn before - # reaching a tool-result boundary that would consume it, - # the text is still stashed in agent._pending_steer. Drain - # it now and emit a pending_steer_leftover SSE event so the - # frontend can queue it for the next turn — same fallback - # path as the CLI in cli.py:8788-8794. - try: - _drain_pending_steer = getattr(agent, '_drain_pending_steer', None) - _leftover = _drain_pending_steer() if _drain_pending_steer else None - if _leftover: - put('pending_steer_leftover', { - 'session_id': session_id, - 'text': str(_leftover), - }) - except Exception: - logger.debug("Failed to drain pending steer for session %s", session_id) + _settle_pending_steer() # /goal parity: after a successful assistant turn, run the Hermes # GoalManager judge before terminal done/stream_end events. The # frontend surfaces the status line and queues continuation_prompt as @@ -13107,8 +13395,6 @@ def _run_agent_streaming( if 'credential_pool' in _agent_params: _heal_kwargs['credential_pool'] = _runtime_bundle['credential_pool'] _heal_agent = _AIAgent(**_heal_kwargs) - with STREAMS_LOCK: - AGENT_INSTANCES[stream_id] = _heal_agent _agent_sig = _compute_agent_cache_signature( resolved_model, resolved_api_key, @@ -13125,10 +13411,11 @@ def _run_agent_streaming( profile_home=_profile_home, safe_profile_runtime_env=_safe_profile_runtime_env, ) - from api.config import SESSION_AGENT_CACHE as _SAC2, SESSION_AGENT_CACHE_LOCK as _SAC2_L - with _SAC2_L: - _SAC2[session_id] = (_heal_agent, _agent_sig) - _SAC2.move_to_end(session_id) + if not _register_agent_if_current(_heal_agent, _agent_sig): + with _agent_lock: + _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) + put('cancel', _cancel_event_payload('Cancelled by user')) + return # Retry the conversation _token_sent = False try: @@ -13160,7 +13447,13 @@ def _run_agent_streaming( _result_partial_pre_call_context = list( _heal_context_messages ) + if not _agent_can_invoke(_heal_agent): + with _agent_lock: + _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) + put('cancel', _cancel_event_payload('Cancelled by user')) + return _heal_result = _heal_agent.run_conversation(**_heal_kwargs2) + _remember_pending_steer_result(_heal_result) _active_turn_identity = _resolve_active_turn_authority( _active_turn_identity, result=_heal_result, @@ -13414,6 +13707,7 @@ def _run_agent_streaming( _error_payload['old_session_id'] = session_id put('apperror', _error_payload) finally: + _settle_pending_steer() # #4633/#2476: symmetric metering teardown. begin_session() (top of the # outer try) had no paired end_session(), so zero-token turns leaked a # _sessions[stream_id] entry that get_stats() pruning never reclaims (its @@ -13441,7 +13735,6 @@ def _run_agent_streaming( if (s is not None and getattr(s, 'active_stream_id', None) == stream_id and getattr(s, 'pending_user_message', None)): - update_active_run(stream_id, phase="finalizing") _last_resort_sync_from_core(s, stream_id, _agent_lock) _clear_thread_env() # TD1: always clear thread-local context if _streaming_cron_profile_home_token is not None: @@ -13536,13 +13829,68 @@ def _run_agent_streaming( # ============================================================ +# Local lifecycle phases with a pending-steer consumer still ahead of them. +# Gateway uses its own phase vocabulary and never falls through to local delivery. +_LOCAL_STEERABLE_PHASES = frozenset({"starting", "running"}) + + +def _steer_bound_stream(sid: str, stream_id: str, text: str) -> dict | None: + """Deliver to a verified live worker; None retains cache-only compatibility. + + Never perform HTTP writes or cache/database teardown under stream locks. + """ + from api import config as cfg + + with cfg.STREAMS_LOCK: + agent = cfg.AGENT_INSTANCES.get(stream_id) + owner = cfg.stream_owner_session_id(stream_id) + with cfg.ACTIVE_RUNS_LOCK: + run = dict(cfg.ACTIVE_RUNS.get(stream_id) or {}) + # Gateway owns transport even when no in-process worker is registered. + # A reusable local cache entry must never override that authority. + if run.get("backend") == "gateway": + if (stream_id not in cfg.STREAMS + or owner != sid or run.get("session_id") != sid + or run.get("phase") == "cancelling"): + return {"accepted": False, "fallback": "stream_dead", "stream_id": None} + return {"accepted": False, "fallback": "gateway_steer_queued", "stream_id": stream_id} + if (stream_id in cfg.STREAMS and owner == sid + and run.get("session_id") == sid + and run.get("backend") == WEBUI_LOCAL_CHAT_BACKEND + and run.get("phase") == "finalizing"): + return {"accepted": False, "fallback": "not_running", "stream_id": stream_id} + if agent is None: + if ((owner and owner != sid) + or (run.get("session_id") and run["session_id"] != sid) + or (run and run.get("phase") not in _LOCAL_STEERABLE_PHASES)): + return {"accepted": False, "fallback": "stream_dead", "stream_id": None} + return None + # A cache hit cannot override missing or conflicting worker ownership. + if (stream_id not in cfg.STREAMS + or owner != sid or run.get("session_id") != sid + or run.get("phase") not in _LOCAL_STEERABLE_PHASES + or run.get("backend") == "gateway"): + return {"accepted": False, "fallback": "stream_dead", "stream_id": None} + if not callable(getattr(agent, "steer", None)): + return {"accepted": False, "fallback": "agent_lacks_steer", "stream_id": None} + # steer() only stashes input; serializing it with stream teardown keeps + # cancel from detaching the selected worker before delivery. + try: + accepted = bool(agent.steer(text)) + except Exception: + logger.debug("Stream-bound steer failed for session %s", sid, exc_info=True) + return {"accepted": False, "fallback": "steer_error", "stream_id": stream_id} + return {"accepted": accepted, "fallback": None, "stream_id": stream_id} + + def _handle_chat_steer(handler, body: dict) -> bool: """Inject a /steer payload into the active agent for a session. Mirrors the CLI's `/steer ` command (cli.py:6140-6155): - - Look up the cached AIAgent for the session (PR #1051's - SESSION_AGENT_CACHE). + - Prefer the active stream's registered AIAgent, with explicit stream + and worker ownership; use the cache only for legacy cache-only runs. - Verify a stream is currently active for this session. + - Never evict or close an agent while delivering steering input. - Call agent.steer(text) — thread-safe, stashes text in _pending_steer for application at the next tool-result boundary. @@ -13571,24 +13919,23 @@ def _handle_chat_steer(handler, body: dict) -> bool: if not text: return bad(handler, "text required") - evicted_cached_entry = None + # A compression rotates agent.session_id without replacing the running + # worker. Resolve the stream-bound instance BEFORE consulting the reusable + # cache. Steer never owns cache eviction or agent/database teardown. + try: + session = get_session(sid) + except KeyError: + session = None + stream_id = getattr(session, "active_stream_id", None) or None + if stream_id: + result = _steer_bound_stream(sid, stream_id, text) + if result is not None: + return j(handler, result) + with _cfg.SESSION_AGENT_CACHE_LOCK: cached = _cfg.SESSION_AGENT_CACHE.get(sid) - if cached: - agent = cached[0] - if not _cached_agent_matches_session(agent, sid): - evicted_cached_entry = _cfg.SESSION_AGENT_CACHE.pop(sid, None) - logger.warning( - '[webui] Evicted cached agent before steer due to mismatched session identity: cache_key=%s agent_session_id=%s', - sid, - _cached_agent_session_identity(agent), - ) - cached = None - if evicted_cached_entry is not None: - try: - _close_cached_agent_entry_at_session_boundary(sid, evicted_cached_entry) - except Exception: - logger.debug("Failed to close steer identity-mismatched cached agent for session %s", sid, exc_info=True) + if cached and not _cached_agent_matches_session(cached[0], sid): + cached = None if not cached: try: s = get_session(sid) @@ -13634,22 +13981,54 @@ def _handle_chat_steer(handler, body: dict) -> bool: if not active_stream_id: return j(handler, {"accepted": False, "fallback": "not_running", "stream_id": None}) + + # Cache-only compatibility: no registered worker for this stream. The + # liveness/ownership revalidation and the enqueue must share the same + # stream-ownership edge as Stop (STREAMS_LOCK -> ACTIVE_RUNS_LOCK). A Stop + # that claims cancellation between an unlocked check and agent.steer() + # would strand guidance this response still reports as accepted. steer() + # only stashes input; interrupt, persistence, and HTTP writes stay outside + # the lock, and the cached agent is never evicted or closed here. + # + # Ownership must be proven positively, not merely unrefuted: BOTH the + # stream owner AND the active-run session must equal the requesting + # session. Missing metadata is ambiguous and fails closed (stream_dead); + # an unfenced cache object is never steered on absent ownership. + # + # The active-run backend is revalidated the same way. With no registered + # worker, the backend tag is the only proof that an in-process runtime + # owns this run: Gateway resolves to its own outcome, the explicit local + # tag may enqueue, and a missing, empty, or foreign backend fails closed. + result = {"accepted": False, "fallback": "stream_dead", "stream_id": None} with _cfg.STREAMS_LOCK: - stream_alive = active_stream_id in _cfg.STREAMS - if not stream_alive: - # Active stream id is stale — stream has ended; caller falls back - return j(handler, {"accepted": False, "fallback": "stream_dead", - "stream_id": None}) + if active_stream_id in _cfg.STREAMS: + owner = _cfg.stream_owner_session_id(active_stream_id) + with _cfg.ACTIVE_RUNS_LOCK: + run = dict((_cfg.ACTIVE_RUNS or {}).get(str(active_stream_id)) or {}) + owned_stream = bool(owner) and owner == sid + owned_run = bool(run.get("session_id")) and run["session_id"] == sid + if owned_stream and owned_run and run.get("phase") != "cancelling": + backend = run.get("backend") + if backend == "gateway": + # Gateway owns transport; a local cache object is never steered. + result = {"accepted": False, "fallback": "gateway_steer_queued", + "stream_id": active_stream_id} + elif backend == WEBUI_LOCAL_CHAT_BACKEND and run.get("phase") == "finalizing": + result = {"accepted": False, "fallback": "not_running", + "stream_id": active_stream_id} + elif (backend == WEBUI_LOCAL_CHAT_BACKEND + and run.get("phase") in _LOCAL_STEERABLE_PHASES): + try: + accepted = bool(agent.steer(text)) + except Exception as exc: + logger.debug("agent.steer() raised for session=%s: %s", sid, exc) + result = {"accepted": False, "fallback": "steer_error", + "stream_id": active_stream_id} + else: + result = {"accepted": accepted, "fallback": None, + "stream_id": active_stream_id} - try: - accepted = bool(agent.steer(text)) - except Exception as exc: - logger.debug("agent.steer() raised for session=%s: %s", sid, exc) - return j(handler, {"accepted": False, "fallback": "steer_error", - "stream_id": active_stream_id}) - - return j(handler, {"accepted": accepted, "fallback": None, - "stream_id": active_stream_id}) + return j(handler, result) def cancel_stream(stream_id: str) -> bool: @@ -13746,21 +14125,29 @@ def cancel_stream(stream_id: str) -> bool: return False active_run_session_id = str(active_run_entry.get("session_id") or "").strip() or None - if active_run_entry is None: - try: - with _live_config.ACTIVE_RUNS_LOCK: - active_run_entry = dict((_live_config.ACTIVE_RUNS or {}).get(stream_id) or {}) - except Exception: - active_run_entry = None - if active_run_entry and not active_run_session_id: - active_run_session_id = str(active_run_entry.get("session_id") or "").strip() or None + if active_run_entry is None: + try: + with _live_config.ACTIVE_RUNS_LOCK: + active_run_entry = dict((_live_config.ACTIVE_RUNS or {}).get(stream_id) or {}) + except Exception: + active_run_entry = None + if active_run_entry and not active_run_session_id: + active_run_session_id = str(active_run_entry.get("session_id") or "").strip() or None - # Mark the worker lifecycle registry immediately. The SSE maps may be popped - # below while the worker is still unwinding; ACTIVE_RUNS is what recovery / - # health polling sees during that detached window. Stamp cancelled_at so - # _clear_stale_stream_state() can eventually reclaim the session if the - # worker is stuck in C-level I/O and never reaches its finally (#6623). - update_active_run(stream_id, phase="cancelling", cancelled_at=time.time()) + # Mark the worker lifecycle registry immediately. The SSE maps may be popped + # below while the worker is still unwinding; ACTIVE_RUNS is what recovery / + # health polling sees during that detached window. Stamp cancelled_at so + # _clear_stale_stream_state() can eventually reclaim the session if the + # worker is stuck in C-level I/O and never reaches its finally (#6623). + update_active_run(stream_id, phase="cancelling", cancelled_at=time.time()) + + # Stop and Steer share STREAMS_LOCK -> ACTIVE_RUNS_LOCK ordering. + # Publish cancellation and detach ownership before releasing the edge; + # later Steer cannot enqueue into a turn already claimed by Stop. + if stream_present: + streams.pop(stream_id, None) + cancel_flags.pop(stream_id, None) + agent_instances.pop(stream_id, None) # Set WebUI layer cancel flag. Prefer the snapshot captured under the lock; # fall back to a fresh lookup for the ACTIVE_RUNS-only path (stream absent). @@ -13813,16 +14200,7 @@ def cancel_stream(stream_id: str) -> bool: # worker save and show cancel in the client while persistence says done. _emit_cancel_event = True - # ── Eager session lock release (fixes #653) ────────────────────────── - # Pop stream state now so the 409 guard in routes.py sees the session - # as idle and allows new /api/chat/start immediately after cancel, - # even if the agent thread is still blocked in a C-level syscall. - # The worker thread's finally block uses .pop(key, None) too, so a - # double-pop here is safe (no-op). - if stream_present: - streams.pop(stream_id, None) - cancel_flags.pop(stream_id, None) - agent_instances.pop(stream_id, None) + # Stream ownership was detached under streams_lock before interrupting. # STREAM_PARTIAL_TEXT is intentionally NOT popped here — the agent thread may # still be appending tokens, and the streaming finally block handles cleanup # when the thread exits. We already snapshotted the buffers under streams_lock diff --git a/api/upload.py b/api/upload.py index 6269fca5..3701e669 100644 --- a/api/upload.py +++ b/api/upload.py @@ -8,7 +8,12 @@ import tempfile from pathlib import Path from api.config import MAX_UPLOAD_BYTES, STATE_DIR -from api.helpers import j +from api.helpers import ( + arm_connection_close_if_body_pending, + j, + unreadable_content_length, + unsupported_transfer_encoding, +) from api.models import get_session from api.profiles import _profiles_match, get_active_profile_name as _get_active_profile_name from api.workspace import ( @@ -45,6 +50,89 @@ def _max_extracted_bytes() -> int: _MAX_EXTRACTED_BYTES = 10 * MAX_UPLOAD_BYTES +def _reject_before_read(handler, message, status): + """Reject without reading the body; close so unread bytes can't poison HTTP/1.1 reuse. + + Framing decides, exactly as at the other reject-before-read sites: every + rejection raised by the preflight above (any ``Transfer-Encoding``, an + unreadable or conflicting ``Content-Length``, an over-cap length) already + declares a body and still closes. Only the boundary/`parse_multipart` rejection + can fire on a request that declares none, and closing there dropped a healthy + pooled connection -- on the wire, ``POST /api/upload`` with + ``Content-Type: application/json`` and no ``Content-Length`` answered 400 + "No boundary in Content-Type" WITH ``Connection: close`` and the pipelined + ``GET /api/health/agent`` was never served. + """ + arm_connection_close_if_body_pending(handler) + return j(handler, {'error': message}, status=status) + + +class _RejectBeforeRead(ValueError): + """A rejection raised before any byte of the request body was consumed. + + Carries the status the caller must answer with. Subclasses ``ValueError`` so + that if one ever escapes an unusual call path, each handler's existing + ``except ValueError`` still answers 400 with the same message instead of + turning it into a 500. + """ + + def __init__(self, message: str, status: int): + super().__init__(message) + self.message = message + self.status = status + + +def _read_multipart_or_reject(handler): + """Framing preflight + body read, shared by all four multipart handlers. + + Returns ``(fields, files)``. Raises ``_RejectBeforeRead`` when the request + must be refused before its body is read -- the caller answers through + ``_reject_before_read()``, which arms ``Connection: close`` first, because the + declared body is still queued in ``rfile`` and a reused HTTP/1.1 connection + would parse those bytes as the next request line. + + Framing is judged from the headers, not from a present ``Content-Length``: + + * ANY ``Transfer-Encoding`` is refused, ``identity`` included: ``http.server`` + decodes none of them (``rfile`` is the raw socket) and RFC 9112 section 6.3 + says a request whose final coding is not ``chunked`` has a length no + recipient can determine. A coded request carries no usable Content-Length, + so the old code read it as length 0, found no parts, and answered "No file + field in request" with keep-alive intact -- leaving every payload byte on + the socket to poison the next pooled request. A Content-Length sent BESIDE a + coding does not rescue it: that pairing is the smuggling shape 6.3 rejects, + and the coding still hides how the body is framed. + * a non-numeric or oversized Content-Length cannot be read either, and + ``parse_multipart()`` validates the boundary and the length before its + first ``rfile.read()`` — so every raise below is still pre-read. + * two DISAGREEING Content-Length headers read back as whichever came first, + so ``Content-Length: 0`` ahead of the real length made the body invisible + and left every byte of it on the socket. There is no honest length to + drain, so the request is refused before ``rfile`` is touched. + """ + content_type = handler.headers.get('Content-Type', '') + coding = unsupported_transfer_encoding(handler) + if coding is not None: + raise _RejectBeforeRead( + f'Unsupported Transfer-Encoding: {coding} ' + '(uploads must send an undecoded body with a Content-Length)', + 411, + ) + if unreadable_content_length(handler): + raise _RejectBeforeRead('Invalid Content-Length', 400) + try: + content_length = int(handler.headers.get('Content-Length', 0) or 0) + except (TypeError, ValueError): + raise _RejectBeforeRead('Invalid Content-Length', 400) from None + if content_length > MAX_UPLOAD_BYTES: + raise _RejectBeforeRead( + f'File too large (max {MAX_UPLOAD_BYTES//1024//1024}MB)', 413) + try: + return parse_multipart(handler.rfile, content_type, content_length) + except ValueError as exc: + raise _RejectBeforeRead(str(exc), 400) from None + + def parse_multipart(rfile, content_type, content_length) -> tuple: import re as _re, email.parser as _ep # Imported locally (not just module-level) so the function stays @@ -206,11 +294,10 @@ def _write_office_upload_sidecar(workspace: Path, dest: Path, file_bytes: bytes) def handle_upload(handler): import traceback as _tb try: - content_type = handler.headers.get('Content-Type', '') - content_length = int(handler.headers.get('Content-Length', 0) or 0) - if content_length > MAX_UPLOAD_BYTES: - return j(handler, {'error': f'File too large (max {MAX_UPLOAD_BYTES//1024//1024}MB)'}, status=413) - fields, files = parse_multipart(handler.rfile, content_type, content_length) + try: + fields, files = _read_multipart_or_reject(handler) + except _RejectBeforeRead as rejection: + return _reject_before_read(handler, rejection.message, rejection.status) session_id = fields.get('session_id', '') if 'file' not in files: return j(handler, {'error': 'No file field in request'}, status=400) @@ -396,11 +483,10 @@ def handle_upload_extract(handler): """Handle archive upload and extraction.""" import traceback as _tb try: - content_type = handler.headers.get('Content-Type', '') - content_length = int(handler.headers.get('Content-Length', 0) or 0) - if content_length > MAX_UPLOAD_BYTES: - return j(handler, {'error': f'File too large (max {MAX_UPLOAD_BYTES//1024//1024}MB)'}, status=413) - fields, files = parse_multipart(handler.rfile, content_type, content_length) + try: + fields, files = _read_multipart_or_reject(handler) + except _RejectBeforeRead as rejection: + return _reject_before_read(handler, rejection.message, rejection.status) session_id = fields.get('session_id', '') if 'file' not in files: return j(handler, {'error': 'No file field in request'}, status=400) @@ -428,11 +514,10 @@ def handle_transcribe(handler): import traceback as _tb temp_path = None try: - content_type = handler.headers.get('Content-Type', '') - content_length = int(handler.headers.get('Content-Length', 0) or 0) - if content_length > MAX_UPLOAD_BYTES: - return j(handler, {'error': f'File too large (max {MAX_UPLOAD_BYTES//1024//1024}MB)'}, status=413) - fields, files = parse_multipart(handler.rfile, content_type, content_length) + try: + fields, files = _read_multipart_or_reject(handler) + except _RejectBeforeRead as rejection: + return _reject_before_read(handler, rejection.message, rejection.status) if 'file' not in files: return j(handler, {'error': 'No file field in request'}, status=400) filename, file_bytes = files['file'] @@ -612,12 +697,10 @@ def handle_workspace_upload(handler): """ import traceback as _tb try: - content_type = handler.headers.get('Content-Type', '') - content_length = int(handler.headers.get('Content-Length', 0) or 0) - if content_length > MAX_UPLOAD_BYTES: - return j(handler, {'error': f'File too large (max {MAX_UPLOAD_BYTES//1024//1024}MB)'}, status=413) - - fields, files = parse_multipart(handler.rfile, content_type, content_length) + try: + fields, files = _read_multipart_or_reject(handler) + except _RejectBeforeRead as rejection: + return _reject_before_read(handler, rejection.message, rejection.status) session_id = fields.get('session_id', '') subpath = fields.get('path', '') diff --git a/docs/CONTRACTS.md b/docs/CONTRACTS.md index 2989b587..0b81b9bb 100644 --- a/docs/CONTRACTS.md +++ b/docs/CONTRACTS.md @@ -89,6 +89,15 @@ contributor guidance; it does not change runtime behavior or CI gates. `models_cache.json` refresh timestamps), the stat-identity fallbacks, and the schema/version stamps. Start here before changing model-catalog caching, the `/api/models` cache keys, or the Codex catalog dependency (#2443, #7540). +- [`docs/architecture/live-models-allowlist.md`](architecture/live-models-allowlist.md): + current contract for how `/api/models/live` filters a custom provider's + upstream catalog: the four signals in evaluation order (discovered catalog + defers to the live probe, explicit plural `models:` allowlist filters, + singular `model:` never gates, no allowlist shows the full catalog), the + serialized-list shapes `hermes config set` persists, the deliberate + empty-allowlist-is-not-configured rule, and the probe-failure fallback. + Start here before changing custom-provider model filtering or + discovery-vs-allowlist semantics (#7165, #7404). - [`docs/architecture/profile-home-resolve-cache.md`](architecture/profile-home-resolve-cache.md): current contract for the call-scoped memoization around `_resolve_profile_home_param()` in `api/workspace.py`: what it caches, the diff --git a/docs/architecture/live-models-allowlist.md b/docs/architecture/live-models-allowlist.md new file mode 100644 index 00000000..83c7363a --- /dev/null +++ b/docs/architecture/live-models-allowlist.md @@ -0,0 +1,123 @@ +# Live model picker allowlist contract (`/api/models/live`) + +This document records the runtime contract for how `/api/models/live` decides +which models a `custom` / `custom:*` provider contributes to the chat model +picker. It describes shipped behavior and changes no runtime behavior. It was +added because the #7165 review flagged the contract as undocumented: the +filtering rules live in `_handle_live_models` (`api/routes.py`) and had only +code comments backing them. + +Start here before changing how the live catalog of a custom provider is +filtered, how discovery snapshots are distinguished from allowlists, or what +the endpoint returns when the upstream probe fails. + +## Why the endpoint filters at all + +OpenAI-compatible gateways (New-API-style aggregators, LM Studio, Ollama +proxies) expose their **entire** catalog from `/v1/models`, including +image-generation, audio, TTS, and embedding models that are not chat models. +The background enrichment path must not surface models the user never opted +into — but it also must not hide models the gateway gained since config was +last edited. The contract below is how those two goals are reconciled. + +## The four signals, in evaluation order + +For a custom provider the handler consults these signals: + +1. **Discovered catalog → no allowlist (live probe wins).** + When the provider entry carries `models_discovered: true` and discovery is + allowed (`discover_models` is not an explicit opt-out), the `models:` + mapping is a *snapshot Hermes persisted from a past discovery*, not a + hand-curated list. It must not gate the live probe: a model the user pulls + into LM Studio / Ollama after discovery has to appear in the picker. + Detected by the shared predicate + `api.config._provider_models_are_discovered_catalog()` — the same + predicate the `/api/models` catalog path uses, so both endpoints agree. + With this signal, no allowlist is derived and the full live catalog is + returned. + +2. **Explicit plural `models:` allowlist → filter.** + A non-empty plural `models:` value on an entry that is *not* a discovered + catalog expresses "show exactly these models". The live catalog is + intersected with it, then any allowlisted model missing from the live + response is appended (so an allowlisted-but-offline model stays + selectable). This is the #7165 filter. Only **list** and **string** + (serialized) shapes are plain allowlists — see "Dict-shaped `models:`" + below. + +3. **Singular `model:` → never gates.** + The singular field is sticky/default metadata (the model to preselect), + not an allowlist. A provider configured only with `model: assistant` keeps + the full unfiltered live catalog; gating on it would collapse discovery to + a single model. + +4. **No allowlist → full live catalog.** + Preserves the pre-#7165 discovery behavior for setups that rely on it. + +### Serialized shapes + +`hermes config set` and the JSON-mode editor persist lists as quoted +JSON-array strings (`'["chat-a","chat-b"]'`) or Python literals +(`"['chat-a']"`). The plural value is decoded through the shared +`_parse_config_string_list()` before id extraction — but only when the value +is a `str`; native `dict` / `list` values are walked directly so dict *entries* +keep their `id|model|name` metadata (the decoder stringifies members). A +dict-only parser drops serialized allowlists to "no allowlist" and floods the +picker — the same bug class as `skills.disabled` (#7120 / #7134). + +### Dict-shaped `models:` + +A dict-shaped `models:` mapping (e.g. `{chat-a: {context_length: 128000}}`) +is *per-model metadata* written by the Hermes Agent setup flow — +`hermes_cli/model_switch.py::_save_custom_provider` and the setup wizard — +**not** a catalog narrow. Treating its keys as an allowlist would collapse the +live picker to the single saved default (keyless Ollama) while the CLI +live-probe shows the full catalog. So a dict is **never** a plain allowlist: + +- without an explicit `discover_models: false`, a dict-shaped `models:` + contributes **no allowlist** and the full live catalog is returned; +- with `discover_models: false` (bool `False` or the string forms + `"false"`/`"no"`/`"0"`, case-insensitive — mirroring + `model_switch_providers.py:557`), the dict **keys** are honored as a pinned + allowlist. + +This complements the discovered-catalog rule: `models_discovered: true` +without the opt-out is already ignored (signal 1). The dict guard covers the +Agent-setup shape that is *not* flagged discovered. + +If someone later "fixes" the dict shape back into an always-allowlist, the +regression test `test_dict_models_without_discover_false_does_not_gate` fails +and forces the trade-off to be re-argued. + +### Empty allowlist semantics + +`models: []`, `models: "[]"`, an empty mapping, or a blank scalar are +deliberately treated as **not configured**, never as "allow nothing". The +filter runs only on a non-empty allowlist because gating on declared-ness +would emit an empty picker and make the provider unselectable — a harder +failure than showing a few extra models, and unrecoverable from the UI +(`custom_providers` has no WebUI write path; it is hand-edited in +config.yaml). There is no deny-all use case: a provider the user wants hidden +is removed from `custom_providers` outright. + +### Probe failure / empty probe + +When the upstream probe fails or returns no models, the endpoint falls back +to `allowlist or config ids` — the non-empty plural allowlist first, so a +singular `model:` outside the allowlist cannot leak in precisely when the +probe is unavailable; with no allowlist, the configured ids (singular + +plural) are served. A discovered catalog therefore still serves its saved +models on a transient probe failure instead of emptying the picker. + +## Tests + +- `tests/test_issue3718_live_models_custom_probe.py` — real-handler + (`_handle_live_models`) coverage of every signal above: plural list / + JSON-array-string / Python-literal / list-of-dicts filtering, dict-shaped + `models:` (not gated) and dict + `discover_models: false` (pinned), + singular passthrough, malformed safety, discovered-catalog passthrough, + `discover_models: false` pinning, and both fallback branches. +- `tests/test_issue7404_models_discovered_not_allowlist.py` — the + `/api/models` side of the discovered-catalog contract. +- `tests/test_byok_model_dropdown.py` — picker rendering, profile-scoped + caching, and current-session model preservation around this endpoint. diff --git a/docs/pr-media/6469-sidebar-handoff/README.md b/docs/pr-media/6469-sidebar-handoff/README.md new file mode 100644 index 00000000..d458093d --- /dev/null +++ b/docs/pr-media/6469-sidebar-handoff/README.md @@ -0,0 +1,91 @@ +# Sidebar idle / terminal handoff evidence + +## Failure and scope + +The independent `/api/sessions` response can report idle before the chat SSE's +terminal frame reaches the page. Three sidebar paths were retiring chat state: +idle reconciliation, INFLIGHT purging, and optimistic-row retirement. The idle +path clears the Worklog and active stream ID; the subsequent `done` is rejected +as stale, so no settled Anchor scene is persisted. + +A real-HTTP barrier reproduces this on both published PR head +`75c8ff8d2a991f22389e3e4263ee03fb8d8f67a0` and frozen upstream +`b09fded7b8c98b549a49aaaece3721015337d307`. Applying only the `sessions.js` +product fix to the frozen upstream also passes the identical oracle: this is a +shared lifecycle defect, not evidence that artifact replay introduced it. The +historical hosted run lacks this instrumentation, so its exact interleaving +cannot be proved retrospectively; its empty-surface/no-persistence signature +matches the controlled reproduction. + +## Reproduction + +```sh +LIFECYCLE_SCENARIO=normal LIFECYCLE_TEST_BITE=settle-worklog-frame-proof \ + .venv/bin/python tests/browser_conversation_lifecycle.py +./scripts/test.sh tests/test_sidebar_terminal_handoff.py -q +``` + +The existing frame proof now uses a test-only server wrapper to hold the actual +`done` HTTP write. The real worker finishes; the real browser receives an idle +sidebar response through `renderSessionList()` while its native EventSource +remains open. The test verifies pane, INFLIGHT and registry ownership, then +releases `done` and retains all existing painted-frame, final-answer, semantic +activity and hard-reload assertions. The fixture always releases/cleans up and +has a bounded wait. Normal production server startup is unchanged. + +No renderer, terminal payload or persistence response is fabricated. Runtime +events come from the existing deterministic localhost Gateway fixture, not a +live model/provider. All state and files are disposable. These are browser +viewport checks, not physical-device acceptance. + +## Before / after + +Each screenshot is taken after the idle sidebar response, before the terminal +HTTP frame is released. The exact published source fails at both widths. The +patched source preserves three live activity rows and then passes completion +and hard reload. `results.json` records the predicate results without machine +paths or credentials. + +| Viewport | Before | After | +| --- | --- | --- | +| 1440 x 900 | [Empty surface](before-1440.png) | [Owned handoff retained](after-1440.png) | +| 390 x 844 | [Empty surface](before-390.png) | [Owned handoff retained](after-390.png) | + +The fix only defers these sidebar cleanups for the current pane's exact OPEN +chat transport. Missing, CONNECTING, CLOSED and mismatched sources still take +the existing stale recovery path. + +## Missing terminal frame: bounded recovery + +The OPEN-only guard at `85f1c2d34b62b0ade85f51ed2cf8c1cf56f5ab9e` could wait +forever when the terminal frame never arrived. The follow-up keeps the live +handoff for 1.5 seconds, then probes the exact stream through +`/api/chat/stream/status` with an eight-second timeout and no retries. +Repeated sidebar hints cannot restart the deadline or create concurrent +requests. `active:true` keeps the OPEN owner intact even when the persisted +session fields already look idle; only explicit runtime inactivity permits the +canonical session restore. A failed or malformed runtime probe uses the +existing interrupted-stream path rather than claiming successful settlement. +Normal terminal delivery, transport replacement/close, navigation, and a newer +optimistic turn invalidate the old request. Already-running stream-end recovery +remains the sole recovery owner. + +```sh +LIFECYCLE_SCENARIO=normal LIFECYCLE_TEST_BITE=sidebar-idle-missing-terminal \ + .venv/bin/python tests/browser_conversation_lifecycle.py +./scripts/test.sh tests/test_sidebar_idle_stream_recovery.py -q +``` + +The new browser row keeps the real terminal HTTP barrier closed until cleanup; +the client must recover without receiving `done` or `stream_end`. Published +`85f1c2d34b` fails the 11-second completion bound with `S.busy` still true. +The patched implementation restores the final answer and passes the unchanged +painted-frame, semantic activity, persistence and hard-reload checks before the +barrier is released. The cloud workflow runs both delayed- and missing-terminal +rows. No live provider or production state is used. + +A production Gateway regression also blocks post-turn goal evaluation after +the success transcript has already cleared `active_stream_id` and pending +fields. During that interval the worker and exact entry in `STREAMS` remain +live and no `done` event has been emitted, proving why persisted session flags +cannot authorize sidebar settlement on their own. diff --git a/docs/pr-media/6469-sidebar-handoff/after-1440.png b/docs/pr-media/6469-sidebar-handoff/after-1440.png new file mode 100644 index 00000000..e28d30be Binary files /dev/null and b/docs/pr-media/6469-sidebar-handoff/after-1440.png differ diff --git a/docs/pr-media/6469-sidebar-handoff/after-390.png b/docs/pr-media/6469-sidebar-handoff/after-390.png new file mode 100644 index 00000000..f876ed24 Binary files /dev/null and b/docs/pr-media/6469-sidebar-handoff/after-390.png differ diff --git a/docs/pr-media/6469-sidebar-handoff/before-1440.png b/docs/pr-media/6469-sidebar-handoff/before-1440.png new file mode 100644 index 00000000..87953ff1 Binary files /dev/null and b/docs/pr-media/6469-sidebar-handoff/before-1440.png differ diff --git a/docs/pr-media/6469-sidebar-handoff/before-390.png b/docs/pr-media/6469-sidebar-handoff/before-390.png new file mode 100644 index 00000000..20c745cb Binary files /dev/null and b/docs/pr-media/6469-sidebar-handoff/before-390.png differ diff --git a/docs/pr-media/6469-sidebar-handoff/results.json b/docs/pr-media/6469-sidebar-handoff/results.json new file mode 100644 index 00000000..c1564591 --- /dev/null +++ b/docs/pr-media/6469-sidebar-handoff/results.json @@ -0,0 +1,66 @@ +[ + { + "mode": "before", + "viewport": [ + 1440, + 900 + ], + "exit": 1, + "observation": { + "serverIdle": true, + "paneRetained": false, + "inflightRetained": false, + "sourceRetained": true, + "registryRetained": true, + "liveRows": 0 + } + }, + { + "mode": "before", + "viewport": [ + 390, + 844 + ], + "exit": 1, + "observation": { + "serverIdle": true, + "paneRetained": false, + "inflightRetained": false, + "sourceRetained": true, + "registryRetained": true, + "liveRows": 0 + } + }, + { + "mode": "after", + "viewport": [ + 1440, + 900 + ], + "exit": 0, + "observation": { + "serverIdle": true, + "paneRetained": true, + "inflightRetained": true, + "sourceRetained": true, + "registryRetained": true, + "liveRows": 3 + } + }, + { + "mode": "after", + "viewport": [ + 390, + 844 + ], + "exit": 0, + "observation": { + "serverIdle": true, + "paneRetained": true, + "inflightRetained": true, + "sourceRetained": true, + "registryRetained": true, + "liveRows": 3 + } + } +] diff --git a/docs/rfcs/live-to-final-assistant-replies.md b/docs/rfcs/live-to-final-assistant-replies.md index 6564390e..60e762c6 100644 --- a/docs/rfcs/live-to-final-assistant-replies.md +++ b/docs/rfcs/live-to-final-assistant-replies.md @@ -470,6 +470,19 @@ Expected behavior: - A session row's running indicator should reflect a real active run or a clearly restorable state, not stale persisted metadata alone. +- An idle sidebar response does not prove that the browser has consumed the + terminal chat frame. While the current pane still owns its exact OPEN chat + transport, list reconciliation and cache purging must preserve that handoff. + Missing, connecting, closed, or mismatched transports keep the normal stale + state recovery path; a cached stream ID or busy flag is insufficient. + OPEN is not proof that a terminal frame will arrive: an idle hint must also + schedule bounded recovery without erasing the live scene. Before reading a + settled session snapshot, that recovery must check the exact stream's runtime + status. A runtime-active stream retains ownership even when persisted + `active_stream_id` and pending fields are already clear; only explicit + runtime inactivity permits canonical snapshot settlement. Probe failure is a + degraded/interrupted state, not evidence of successful completion. A newer + turn, replacement source, or received terminal event invalidates the request. - Background completion, cancellation, or failure should be represented without stealing the visible pane from the user. - Session switching should not erase pending live context, in-flight snapshots, diff --git a/docs/rfcs/session-sse-contract-v1.md b/docs/rfcs/session-sse-contract-v1.md index 65b13366..e6248e03 100644 --- a/docs/rfcs/session-sse-contract-v1.md +++ b/docs/rfcs/session-sse-contract-v1.md @@ -183,7 +183,7 @@ when integrating with the live chat SSE relay. | `cancel` | Run cancelled | | `done` | Turn finalized (session payload); title/`stream_end` may follow | | `stream_end` | SSE fence — close the client EventSource | -| `metering` | Token/cost metering snapshot | +| `metering` | Best-effort live token/cost metering snapshot; not journal-replayed | | `context_status` | Context window / usage status | | `goal` | Goal / plan card update | | `goal_continue` | Goal continuation signal | @@ -228,6 +228,31 @@ A future implementation must replay from the run journal via the existing `_replay_run_journal()` path (in `api/routes.py`) and fall back to the snapshot mechanism when journal entries are unavailable for a given cursor. +### Live-only metering (implemented replay behavior) + +`metering` is best-effort live telemetry, not durable recovery content. +`RunJournalWriter.append_sse_event()` returns `None` for metering without +writing a row or allocating a sequence number. Other journaled events retain +contiguous sequence numbers. Local-agent and gateway producers carry an +explicit per-frame event ID, including `None`, through `StreamChannel`; +an unjournaled metering frame emits no new SSE `id:` and must not borrow the +previous frame's journal ID. The last-ID fallback is limited to legacy +two-element queue entries. + +Existing journals are not rewritten. Direct readers (`read_run_events()` and +`read_session_run_events()`) retain legacy metering rows for cursor validation +and gap/coverage checks. Both run-level and per-session journal replay emitters +apply `journal_replay_visible()` to omit those rows from reconnect delivery. +Consequently, replayed legacy event IDs can have gaps where metering was +filtered; clients must not infer missing durable content from those gaps. +Metering missed while disconnected is not recovered from the journal; fresh +live updates remain available. This exception does not remove token, reasoning, +tool, or terminal events from durable replay. + +Coverage: `tests/test_run_journal.py`, `tests/test_run_journal_routes.py`, +`tests/test_issue4812_session_sse_stream.py`, and +`tests/test_stage364_opus_live_sse_event_id.py`. + ## Snapshot fallback When the `Last-Event-ID` cursor is evicted, expired, unknown, or refers to a diff --git a/docs/rfcs/stable-assistant-turn-anchors.md b/docs/rfcs/stable-assistant-turn-anchors.md index 9c522519..39be9aa4 100644 --- a/docs/rfcs/stable-assistant-turn-anchors.md +++ b/docs/rfcs/stable-assistant-turn-anchors.md @@ -574,6 +574,38 @@ early renderers can avoid chasing `S.messages` during every paint. If settlement later rewrites the transcript message, the anchor must be refreshed from that message instead of allowing the two copies to drift silently. +### Settled ownership of a restored live turn + +A preserved or snapshotted live-turn node is not evidence of a running turn. The +settled assistant message is persisted a few milliseconds before the stream's +terminal event clears the active stream id, so a live node whose stream has +already ended can survive a transcript rebuild or a session switch and be +appended under the settled answer, putting the same answer on screen twice +(#6948 follow-up; upstream symptom report #2051). + +A live-turn node is therefore discarded only where the settled transcript is +proved to own it: the transcript ends with a settled assistant message, no +message still carries a live-projection marker, that settled message's +persisted stream identity equals the id of the stream that built the live node +— or, only where no scene identity was persisted, the markdown the stream +produced equals the persisted message source — and the node carries nothing the +settled rebuild could not have produced, meaning no unpersisted tool card, +reasoning row or transparent-stream row and no second live segment. + +Ownership is a state proof, never a comparison of two rendered DOM trees: the +live body comes from the streaming parser and the settled body from the +markdown renderer, and an answer that reads the same as the previous one must +not delete a genuinely live turn. An unrecognised node shape keeps the turn. +Only the two branches that ADD a turn are gated; a branch that replaces a node +cannot duplicate one. + +A settled activity scene projects one answer as well. A `process_prose` row +whose text duplicates the scene's final answer — the same text, or a prefix of +it within ten percent of its length — is dropped where both settled renderers +take their rows, so the answer is not rebuilt as a second assistant segment +above itself. Live rendering is unchanged: while the turn streams, the inline +live segment is hidden and the prose row is the visible answer. + ## Replay, Reload, And Reconstruction Replay/reload should reconstruct the same Assistant Turn Anchor from durable diff --git a/docs/rfcs/webui-run-state-consistency-contract.md b/docs/rfcs/webui-run-state-consistency-contract.md index ca185de6..7df7f1a0 100644 --- a/docs/rfcs/webui-run-state-consistency-contract.md +++ b/docs/rfcs/webui-run-state-consistency-contract.md @@ -30,6 +30,69 @@ This RFC defines a consistency contract for those layers. It complements the larger run adapter direction in #1925 by documenting what must remain coherent while WebUI still has multiple overlapping state stores. +## Agent registration after cancellation + +Initial and credential self-heal Agent construction use the same registration +boundary. Under `STREAMS_LOCK`, both the worker-retained cancellation event and +live stream membership must permit registration. A removed `CANCEL_FLAGS` entry +is not permission to restart. If Stop won during initial or self-heal +construction, the candidate must not enter the reusable cache or call +`run_conversation`. Stream registration, reusable-cache publication and the +in-memory lifecycle handle share one atomic Stop admission, using lock order +`STREAMS_LOCK` then `SESSION_AGENT_CACHE_LOCK`. Merely moving a cache write +after a cancellation check does not protect that check-to-publication gap. + +After prompt preparation and immediately before each initial/self-heal invocation, +revalidate the retained cancel event, stream membership and exact registered +Agent. If Stop already won, retire a matching reusable entry only while the +existing `SESSION_WRITEBACK_OWNERS` record still equals this exact stream. +Hold that ownership lock through cache/lifecycle retirement. Object identity is +insufficient because successors can reuse the same Agent; absent ownership is +not permission either, since a completed successor clears its record. Never +clear a successor's cache or lifecycle handle, and do not issue another Agent +interrupt for an invocation that never started. Rejected cache-hit registration +likewise must not interrupt the borrowed Agent; only a never-published newly +constructed candidate may receive construction-cancellation cleanup. Drop the +old worker's local borrowed handle as well, so final pending-Steer drain cannot +reach a successor through an object-reference fallback. +Stop after invocation admission uses the existing Agent interrupt mechanism; +registry locks must not span provider or tool execution. LRU eviction/close stays +outside the stream lock and retains the existing active-worker policy. + +Interrupt and cancellation finalization occur outside the stream registry lock. +Session finalization still owns the session lock: the returned-error path already +holds it, while initial registration and the exception path acquire it. Do not +reacquire this non-reentrant lock from a branch that already owns it. + +## Run-journal sequence publication + +Within one WebUI process, auto-numbered appends to the same journal allocate +sequence numbers and write their rows under the same per-path lock. +`RunJournalWriter` delegates both operations to `append_run_event`; it must not +reserve a sequence and release the lock before the physical append. Otherwise +individually valid rows can reach disk out of order and the session replay +reader must reject them as noncontiguous. This does not change caller-supplied +sequence semantics, cross-process ownership, or failed-write recovery. + +## Inactive compression continuation recovery + +The Agent profile's SQLite compression lineage owns the canonical continuation, +including when Desktop/CLI compressed a session without updating WebUI's +`pre_compression_snapshot` sidecar flag. `GET /api/session` may expose the +existing `continuation_session_id` hint from that read-only lineage. Automatic +`idle_timeout` closure does not hide the continuation; explicit/unknown terminal +reasons, foreign-profile rows and delegated/tool children do not authorize it. +This read does not reopen sessions or copy ancestor display history into context. + +A stale `POST /api/chat/start` returns HTTP 409 with `code=session_rotated` +and the continuation hint before workspace, model, pending-turn or worker +mutation. The browser loads the continuation through normal session access +checks and restores the rejected text and attachments as a draft. The user +sends again explicitly; there is no automatic POST replay or migration of the +parent's workspace binding. Clients without this handling must reload the +session before retrying. Server wakeups, regeneration semantics and Gateway +routing are not silently retargeted by this recovery path. + ## Goals - Define the state layers involved in active and recovered WebUI turns. @@ -112,6 +175,23 @@ and 5; it does not mark every run-state boundary implemented. Visible interim assistant progress must remain visible timeline content; a compact Activity disclosure may summarize adjacent tool/debug detail, but it must not be the only place where the user can see emitted progress text. + Interim assistant text that duplicates the tail of the accumulated reasoning + transcript is stripped from the reasoning copy so the restored snapshot + shows the content once. That echo match is whitespace-insensitive and + carries no fixed search window: a compact-equivalent suffix is recognized + however much interior whitespace stretches its raw span. Both consumers + (the live-stream echo path and the journal replay in `api/routes.py`) + match through an incremental folded index (`_CompactEchoIndex` in + `api/streaming.py`): the folded view and its raw cut offsets are built as + each chunk is appended, so a probe costs O(len(candidate)) and never + rescans the transcript's whitespace. The retired windowed variants could + drop the strip when the span exceeded the window, duplicating the interim + text; the retired raw backward walk was correct but re-walked the span per + interim event, quadratic on whitespace-heavy transcripts. Regressions: + `tests/test_live_snapshot_echo_dedup.py` pins the single-occurrence + result, `tests/test_live_snapshot_echo_scan_scaling.py` pins the scaling + property (a fixed-size fixture cannot catch a per-interim rescan), and + `tests/test_compact_echo_index.py` pins index/oracle equivalence. 6. **Compression is not current intent.** Automatic compression summaries and reference cards are recovery/handoff material. They must not be treated as a new user request, active-turn content, or the default visible explanation for diff --git a/docs/sse-streams.md b/docs/sse-streams.md index 07fca5b6..12eb3898 100644 --- a/docs/sse-streams.md +++ b/docs/sse-streams.md @@ -56,6 +56,21 @@ infer it from the status code. Clients that only need session updates should use `/api/session/stream` directly; the gateway probe is only relevant for clients that display CLI/TUI/messaging sessions. +## Gateway watcher projection lifecycle + +The optional gateway stream's watcher owns a cheap database fingerprint and a +timestamp for its last full session projection. While clients are subscribed, +an unchanged fingerprint can skip the expensive read until the bounded parity +interval. With no subscribers the poll loop parks instead of reading the DB. +When the final subscriber leaves (including eviction of a full or failed +consumer queue), the watcher invalidates both cache fields. A reconnect must +therefore trigger a fresh projection even when the database fingerprint has not +changed and the parity interval has not elapsed. Final removal also advances a +subscriber epoch: an older in-flight DB read cannot restore the invalidated +cache or publish its result to a later subscriber cohort. The subscriber lock +protects removal, epoch checks and cache commits, but is not held during DB +reads, so disconnect is not blocked by a slow projection. + ## Heartbeats and proxy behavior - All long-lived streams emit SSE keepalive comment lines on the diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 35cba162..bb59d431 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -231,6 +231,36 @@ For a foreground `python3 bootstrap.py`, stop it with Ctrl-C and start it again. --- +## Agent sessions list slowly (or the `state.db` read index is missing) + +**Symptom.** The sidebar's imported/CLI session list takes seconds per refresh on a large Hermes profile, or a log line says a `state.db` read failed. Sessions still appear; nothing is lost. + +**Why.** Every WebUI reader of the agent's `state.db` (session listing, transcript reads, lineage, gateway watcher, cron sidebar, insights, health) opens it strictly read-only (`file:...?mode=ro`). A reader never upgrades to a write-capable handle and never creates an index: on a multi-GiB `messages` table `CREATE INDEX` holds the SQLite writer lock for minutes and stalls the agent streaming into the same WAL database. When the agent's standard `idx_messages_session` index is missing (older agent, hand-rebuilt or re-imported DB), listings degrade to a bounded one-pass pre-aggregation — slower than the indexed seek, but read-only. A read-only open failure propagates to the caller's existing error boundary (the listing returns empty for that profile) instead of silently reopening the file writable. + +**Diagnostic.** + +```bash +sqlite3 "file:$HOME/.hermes/state.db?mode=ro" "PRAGMA index_list(messages)" +``` + +`idx_messages_session` should be listed. If it is not, the agent has not created it and WebUI will not create it for you. + +**Fix.** Create the covering read indexes in an explicit drained maintenance window: stop the agent (and any gateway/cron runner) writing to that `state.db`, then run: + +```bash +python3 scripts/ensure_state_db_read_indexes.py --db ~/.hermes/state.db --confirm-drained +``` + +- `--confirm-drained` is mandatory: it is your assertion that no agent turn is running against the database. The tool does not verify it. +- `--lock-file PATH` (optional) additionally holds an exclusive non-blocking lock on `PATH` (`flock` on POSIX, `msvcrt.locking` on Windows) for deployments that serialise agent turns on a lock file; a held lock makes the tool exit without touching the database. Without `--lock-file` no lock primitive is required, so the script runs on native Windows as well. On Windows the lock covers byte 0 of `PATH`; a new or empty lock file is initialised with one byte first (an existing lock file is never rewritten). +- The tool opens the database `mode=rw` (never `rwc`): a mistyped path raises instead of creating an empty database. Indexes are created inside one `BEGIN IMMEDIATE` transaction and rolled back on any error. +- It is idempotent and prints a JSON status per index (`created` / `existing` / `skipped`). `skipped` means this database's schema lacks a column that index keys on (an older agent); the indexes the schema does support are still created. An existing index with a different table, key shape or collation is reported as `Incompatible index` and never replaced; an index that is not covering (`EXPLAIN QUERY PLAN`) is reported as `Index is not covering`. +- Windows UNC profiles (`HERMES_HOME=\\server\share\...`) are supported: readers and this tool build the empty-authority URI `file:////server/share/state.db` that the bundled SQLite accepts. + +**When to file a bug.** File a WebUI bug if the listing stays slow after the tool reports `existing` for `idx_messages_session`, if the tool reports `Incompatible index` on an untouched agent-created database, or if a read-only open fails on a local path. Include the tool's JSON output, the `PRAGMA index_list(messages)` result, and the sanitized error text. + +--- + ## 404 after login when password auth is enabled **Symptom.** After enabling password authentication (`HERMES_WEBUI_PASSWORD`), logging in redirects to `/sessions` and the browser shows a `404 not found` error instead of the chat interface. @@ -241,6 +271,44 @@ For a foreground `python3 bootstrap.py`, stop it with Ctrl-C and start it again. --- +## "OpenCode Go model picker shows a model that errors when you send" (or is missing newly released models) + +**Symptom.** One of two directions: + +1. A model selected from the OpenCode Go group fails on the first message (`model not found`, `Model is unavailable`, or a region error), even though the picker offered it. +2. A newly released OpenCode Go model does not appear in the picker at all, and must be typed into the Custom Model ID box. + +**Why.** The Go picker follows the **live** Go-tier catalog (`https://opencode.ai/zen/go/v1/models`) whenever the installed Hermes Agent is v0.20.5 or newer. That endpoint advertises a superset of what a given tier, key, or region can actually serve — an id can be listed and still fail on send. Conversely, an id the endpoint serves but the WebUI's static fallback list predates is missing when the live path is unavailable (Agent older than v0.20.5, probe failure, or offline); the static list mirrors Hermes core's curated Go catalog and can lag new releases by design. + +**Diagnostic.** Check which path is feeding your picker: + +```bash +hermes --version # live path requires >= 0.20.5 +curl -sS https://opencode.ai/zen/go/v1/models \ + -H "Authorization: Bearer $OPENCODE_GO_API_KEY" | head -50 +``` + +Interpret the two together: + +- **Failing model absent from the `curl` output** → it was delisted upstream (for example `ox-alpha-free`, removed from the relay on 2026-09-09). A picker can still offer it from a **stale catalog merge**: Hermes core merges its own curated Go list into the live result, and every Agent release *through v0.21.1 (tag `v2026.9.7`)* still carries the delisted id in that list — verified at runtime against v0.21.0, where the live path serves 37 ids including `ox-alpha-free`. The removal is committed on core `main` (2026-09-09 sync) but **no released Agent version includes it yet** — and `main` reports the same `0.21.1` version string as the stale tag, so a version number alone cannot tell you whether the fix is in. Until a release notes the 2026-09-09 catalog sync, the **verified workaround is the config allowlist** (below); selecting the dead entry is harmless to other models (it errors on send, nothing else). +- **Failing model present in the `curl` output** → the relay lists it but your tier/region cannot serve it; the picker is behaving correctly. Pin the models you actually use with an explicit allowlist in `config.yaml`, which takes precedence over both the live catalog and the fallback: + + ```yaml + providers: + opencode-go: + models: + - kimi-k3 + - glm-5.3 + ``` + + A lighter per-provider exclude capability is tracked in #7507. +- **Missing new model, Agent ≥ v0.20.5** → the live catalog is the source; refresh or check the endpoint with the `curl` above (cold rebuilds are also bounded by a 4-second foreground budget — the first picker open after a restart can serve the last-known list while the live rebuild finishes in the background, so re-open the picker once before concluding it's stale). +- **Missing new model, Agent older than v0.20.5** → the static fallback is serving by design; upgrade the Agent to ≥ v0.20.5 so the picker reads the live catalog. + +**When to file a bug.** File a WebUI bug if a model fails on send *and* appears in the `curl` output for your key (a routing problem), or if a model is missing with Agent ≥ v0.20.5 and the live catalog reachable (fallback used when it should not be). + +--- + ## Other troubleshooting This document grows over time. If a recurring failure mode isn't covered here yet, add it via PR. The format for each entry: **Symptom → Why → Diagnostic commands → Fix → When to file a bug**. diff --git a/scripts/ensure_state_db_read_indexes.py b/scripts/ensure_state_db_read_indexes.py new file mode 100644 index 00000000..0d3cbfe0 --- /dev/null +++ b/scripts/ensure_state_db_read_indexes.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Create the covering read indexes on the agent ``state.db`` in an explicit, +drained maintenance window. + +The WebUI read paths (session listing, lineage reads, gateway watcher, cron +sidebar, insights, health) never create indexes themselves: ``CREATE INDEX`` +on a multi-GiB ``messages`` table holds the SQLite writer lock for minutes, +which stalls the agent streaming into the same WAL database. Index +maintenance is therefore an operator action run while the agent is idle. + +Usage:: + + python scripts/ensure_state_db_read_indexes.py --db ~/.hermes/state.db \\ + --confirm-drained [--lock-file /path/to/agent-activity.lock] + +``--confirm-drained`` is mandatory. When ``--lock-file`` is given, the tool +takes an exclusive non-blocking lock on that path (``flock`` on POSIX, +``msvcrt.locking`` on Windows) so that a deployment which serialises agent +turns on a lock file cannot start a turn mid-rebuild. Without ``--lock-file`` +no lock primitive is needed, so the script runs on every supported platform. + +Existing indexes are verified for table, key shape and collation and must be +covering (``EXPLAIN QUERY PLAN``); an incompatible index is reported, never +silently replaced. +""" +import argparse +from contextlib import closing, nullcontext +import json +import os +from pathlib import Path +import sqlite3 +import sys + +try: # POSIX + import fcntl +except ImportError: # native Windows + fcntl = None +try: # Windows + import msvcrt +except ImportError: # POSIX + msvcrt = None + +_REPO_ROOT = Path(__file__).resolve().parents[1] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) +from api.agent_sessions import state_db_file_uri # same UNC-aware URI as the readers + +INDEXES = { + "idx_messages_session": ("messages", (("session_id", "BINARY"), ("timestamp", "BINARY"))), + "idx_messages_session_role": ("messages", (("session_id", "BINARY"), ("role", "NOCASE"))), + "idx_sessions_webui_fingerprint": ("sessions", tuple((c, "BINARY") for c in + ("source", "id", "message_count", "last_activity_at"))), +} + +# The only schema gap an older Agent is known to have: sessions.last_activity_at +# arrived later than the other keyed columns. An index whose missing columns are +# all listed here is reported "skipped"; any other missing column means this is +# not an Agent state.db (or a damaged one), and the tool fails closed. +_LEGACY_OPTIONAL_COLUMNS = { + "idx_sessions_webui_fingerprint": frozenset({"last_activity_at"}), +} + + +def _require_agent_schema(db) -> None: + """Fail closed unless this is an Agent ``state.db``. + + Matching column names alone would accept a look-alike chat database. Every + Agent ``state.db`` since the first schema has carried a ``schema_version`` + row and ``sessions.id`` as the primary key, so require both before any + ``CREATE INDEX``. + """ + has_marker = db.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_version'" + ).fetchone() + version = None + if has_marker: + cols = {r[1] for r in db.execute("PRAGMA table_info(schema_version)")} + if "version" in cols: + row = db.execute("SELECT version FROM schema_version LIMIT 1").fetchone() + version = row[0] if row else None + if not isinstance(version, int) or version < 1: + raise RuntimeError( + "state.db has no Agent schema_version marker; not an Agent state.db, refusing to continue" + ) + pk = [r[1] for r in db.execute("PRAGMA table_info(sessions)") if r[5]] + if pk != ["id"]: + raise RuntimeError( + "state.db sessions table is not keyed on id; not an Agent state.db, refusing to continue" + ) + + +def _exclusive_lock(lock_file): + if lock_file is None: + return nullcontext() + if fcntl is None and msvcrt is None: + raise RuntimeError("--lock-file needs fcntl (POSIX) or msvcrt (Windows); " + "neither is available on this interpreter") + handle = open(lock_file, "a+") + try: + if fcntl is not None: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + else: + # Lock one byte at offset 0 without blocking; raises OSError when held. + # ``a+`` may have just created an empty file, and Windows cannot lock + # a range beyond EOF, so make sure byte 0 exists first. An existing + # (deployment-owned) lock file is left untouched. Write the byte on + # the descriptor: a text-mode ``handle.write("\n")`` becomes two + # bytes (``\r\n``) on native Windows. + if os.fstat(handle.fileno()).st_size == 0: + os.write(handle.fileno(), b"\0") + handle.flush() + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + except BaseException: + handle.close() + raise + return closing(handle) + + +def ensure_read_indexes(db_path, *, confirmed_drained=False, lock_file=None): + if not confirmed_drained: + raise RuntimeError("requires --confirm-drained") + db_path = Path(db_path).resolve(strict=True) + statuses = {} + with _exclusive_lock(lock_file): + # mode=rw (not rwc): never create a ghost database at a mistyped path. + with closing(sqlite3.connect(state_db_file_uri(db_path) + "?mode=rw", uri=True)) as db: + db.execute("BEGIN IMMEDIATE") + try: + # Validate the whole Agent schema before creating anything, so a + # wrong database fails closed without a partial index set. + _require_agent_schema(db) + missing_by_index = {} + for name, (table, keys) in INDEXES.items(): + present = {r[1] for r in db.execute(f"PRAGMA table_info({table})")} + if not present: + raise RuntimeError(f"state.db has no {table!r} table; refusing to continue") + missing = {col for col, _ in keys if col not in present} + if missing - _LEGACY_OPTIONAL_COLUMNS.get(name, frozenset()): + raise RuntimeError( + f"state.db {table!r} table lacks Agent column(s) " + f"{sorted(missing)}; not an Agent state.db, refusing to continue" + ) + missing_by_index[name] = missing + for name, (table, keys) in INDEXES.items(): + existing = db.execute("SELECT tbl_name FROM sqlite_master WHERE name=?", (name,)).fetchone() + if missing_by_index[name]: + # Known legacy gap. A same-named index here cannot be the + # shape we expect (it keys on the absent column), so it is + # incompatible, not something to skip past. + if existing: + raise RuntimeError(f"Incompatible index: {name}") + statuses[name] = "skipped" + continue + if existing: + actual = tuple((r[2], r[4]) for r in db.execute(f"PRAGMA index_xinfo({name})") if r[5]) + flags = next((r for r in db.execute(f"PRAGMA index_list({table})") if r[1] == name), None) + if existing[0] != table or actual != keys or not flags or flags[2] or flags[4]: + raise RuntimeError(f"Incompatible index: {name}") + statuses[name] = "existing" + else: + columns = ", ".join(f"{col} COLLATE {collation}" for col, collation in keys) + db.execute(f"CREATE INDEX {name} ON {table}({columns})") + statuses[name] = "created" + columns = ", ".join(col for col, _ in keys) + plan = db.execute(f"EXPLAIN QUERY PLAN SELECT {columns} FROM {table} INDEXED BY {name}").fetchall() + if not any(f"COVERING INDEX {name}" in row[3] for row in plan): + raise RuntimeError(f"Index is not covering: {name}") + db.commit() + except BaseException: + db.rollback() + raise + return statuses + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--db", type=Path, required=True, help="path to the agent state.db") + parser.add_argument("--lock-file", type=Path, default=None, + help="optional lock file to hold exclusively (flock) while indexes are built") + parser.add_argument("--confirm-drained", action="store_true", + help="assert that no agent turn is running against this database") + args = parser.parse_args() + result = ensure_read_indexes(args.db, confirmed_drained=args.confirm_drained, + lock_file=args.lock_file) + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/server.py b/server.py index 040d80c1..06fc9a21 100644 --- a/server.py +++ b/server.py @@ -100,10 +100,11 @@ from urllib.parse import urlparse logger = logging.getLogger(__name__) from api.request_logging import emit_request_log -from api.auth import check_auth, reset_trusted_auth_request_state +from api.auth import check_auth_or_close, reset_trusted_auth_request_state from api.config import HOST, PORT, STATE_DIR, SESSION_DIR, DEFAULT_WORKSPACE from api.helpers import ( j, + advertise_connection_close, get_profile_cookie, _build_csp_report_only_policy, _CLIENT_DISCONNECT_ERRORS, @@ -332,6 +333,7 @@ class Handler(BaseHTTPRequestHandler): extra_frame_src = getattr(self, "_csp_extra_frame_src", None) self.send_header("Content-Security-Policy-Report-Only", self.csp_report_only_policy(extra_connect_src, extra_frame_src)) self.send_header("Report-To", self._CSP_REPORT_TO) + advertise_connection_close(self) # tell the client when the socket dies super().end_headers() def log_message(self, fmt, *args): pass # suppress default Apache-style log @@ -376,7 +378,8 @@ class Handler(BaseHTTPRequestHandler): set_request_profile(cookie_profile) try: parsed = urlparse(self.path) - if not check_auth(self, parsed): return + # Body-pending-aware: a body-bearing GET failing auth would poison reuse (#7550). + if not check_auth_or_close(self, parsed): return result = handle_get(self, parsed) if result is False: return j(self, {'error': 'not found'}, status=404) @@ -401,10 +404,8 @@ class Handler(BaseHTTPRequestHandler): set_request_profile(cookie_profile) try: parsed = urlparse(self.path) - _is_csp_report_post = ( - parsed.path == "/api/csp-report" and self.command == "POST" - ) - if not _is_csp_report_post and not check_auth(self, parsed): return + _is_csp_report_post = parsed.path == "/api/csp-report" and self.command == "POST" + if not _is_csp_report_post and not check_auth_or_close(self, parsed): return result = route_func(self, parsed) if result is False: return j(self, {'error': 'not found'}, status=404) diff --git a/static/boot.js b/static/boot.js index d6476b60..06c3888d 100644 --- a/static/boot.js +++ b/static/boot.js @@ -3431,8 +3431,17 @@ window._mirrorSpeechSettingsFromServer=_mirrorSpeechSettingsFromServer; localStorage.setItem('hermes-font-size',fontSize); _applyFontSize(fontSize); if(typeof setLocale==='function'){ + // #7622 (round 3): the settings payload's `s.language` is + // absent (None) for a fresh install, so an explicit non-empty + // value is the user's genuine saved choice. The browser + // navigator hint is now read via the guarded + // `_detectBrowserLanguageHint()` helper (round-3 finding 2) so + // a throwing `navigator` accessor in some embedded webviews + // can no longer abort this branch and reset loaded preferences. + // The fallback ternary preserves the pre-#7622 boot behaviour + // when neither helper is in scope (defence in depth). const _lang=typeof resolvePreferredLocale==='function' - ? resolvePreferredLocale(s.language, localStorage.getItem('hermes-lang')) + ? resolvePreferredLocale(s.language, localStorage.getItem('hermes-lang'), _detectBrowserLanguageHint()) : (s.language || localStorage.getItem('hermes-lang') || 'en'); setLocale(_lang); if(typeof applyLocaleToDOM==='function')applyLocaleToDOM(); diff --git a/static/commands.js b/static/commands.js index 685cbf5c..da741566 100644 --- a/static/commands.js +++ b/static/commands.js @@ -1,3 +1,7 @@ +const _WEBUI_DISPATCHABLE_AGENT_COMMANDS = new Set([ + 'reload-mcp','reload-skills','codex-runtime','credits', + 'moa','sessions','resume','pet' +]); // ── Slash commands ────────────────────────────────────────────────────────── // Built-in commands intercepted before send(). Each command runs locally // (no round-trip to the agent) and shows feedback via toast or local message. @@ -182,6 +186,31 @@ function executeCommand(text){ return {noEcho:!!cmd.noEcho}; } +// Agent-registry slash commands that the WebUI actually dispatches when +// submitted. The autocomplete must not advertise anything outside this set: +// a non-dispatchable registry command (e.g. /agents) would otherwise fall +// through send() to the ordinary chat path and trigger an unintended +// model/API request. +// +// This set holds canonical registry names only. getMatchingCommands() matches +// cmd.name (the canonical metadata name), so alias/underscore forms such as +// /reload_mcp are covered implicitly: typing one resolves through +// getAgentCommandMetadata() at dispatch time and the canonical name is what +// gets tested against the dispatcher allowlist (#6951). +// +// Keep this in sync with _AGENT_COMMANDS_RUN_ON_WEBUI in messages.js and +// _ALLOWED_AGENT_COMMANDS in api/commands.py -- the announced list is a +// subset of the dispatched list (fail-closed), never a superset. +// Plugin-category commands are always dispatchable via the plugin exec +// transport. #6951. + +function _isWebuiDispatchableAgentCommand(cmd){ + const name=String(cmd&&cmd.name||'').trim().toLowerCase(); + if(_WEBUI_DISPATCHABLE_AGENT_COMMANDS.has(name))return true; + // Plugin-registered commands execute via the /api/commands/exec plugin transport. + return String(cmd&&cmd.category||'').trim()==='Plugin'; +} + function getMatchingCommands(prefix){ const q=prefix.toLowerCase(); const matches=COMMANDS.filter(c=>c.name.startsWith(q)).map(c=>({...c,source:'builtin'})); @@ -214,9 +243,14 @@ function getMatchingCommands(prefix){ const name=String(cmd&&cmd.name||'').toLowerCase(); if(!name.startsWith(q)||seen.has(name))continue; if(cmd.cli_only&&name!=='pet')continue; + // #6951: only announce commands send() actually dispatches -- anything + // else would silently fall through to the normal chat path as plain text. + if(!_isWebuiDispatchableAgentCommand(cmd))continue; matches.push({ name, desc:String(cmd&&cmd.description||'').trim()||'Agent command', + // Surface the registry's argument hint on the autocomplete row (#6951). + arg:String(cmd&&cmd.args_hint||'').trim()||undefined, source:cmd.category==='Plugin'?'plugin':'agent', }); seen.add(name); @@ -957,6 +991,12 @@ async function resumeManualCompressionForSession(sid){ // No active compression job or transient server error — not a real failure. // 404: route missed or session gone; 5xx: backend exception during status check. if(e&&(!e.status||e.status===404||e.status>=500)) return; + // #7710: a cross-profile refusal now arrives as 409 + // (``session_profile_mismatch``) where it used to be a 404 that hit the + // benign early-return above. It is not a compression failure, so do not + // render the error state or locally settle the compression UI. + if(e&&e.status===409&&typeof _sessionProfileMismatchFromError==='function' + &&_sessionProfileMismatchFromError(e)) return; if(S.session&&S.session.session_id===sid&&typeof setCompressionUi==='function'){ const visibleMessages=_manualCompressionVisibleMessages(); setCompressionUi({ diff --git a/static/i18n.js b/static/i18n.js index dcca45af..269ffe43 100644 --- a/static/i18n.js +++ b/static/i18n.js @@ -1770,7 +1770,7 @@ const LOCALES = { yolo_disabled: 'YOLO mode OFF', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'YOLO mode active — click to disable', - approval_skip_all: '⚡ Skip all this session', + approval_skip_all: 'Skip all this session', approval_skip_all_title: 'Skip all approval prompts for this session', // composer action tooltips composer_send: 'Send message', @@ -3572,7 +3572,7 @@ const LOCALES = { yolo_disabled: 'Modalità YOLO OFF', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'Modalità YOLO attiva — clicca per disabilitare', - approval_skip_all: '⚡ Salta tutto questa sessione', + approval_skip_all: 'Salta tutto questa sessione', approval_skip_all_title: 'Salta tutte le richieste di approvazione per questa sessione', // composer action tooltips composer_send: 'Invia messaggio', @@ -5382,7 +5382,7 @@ const LOCALES = { yolo_disabled: 'YOLO モード OFF', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'YOLO モード有効 — クリックで無効化', - approval_skip_all: '⚡ このセッションはすべてスキップ', + approval_skip_all: 'このセッションはすべてスキップ', approval_skip_all_title: 'このセッションのすべての承認プロンプトをスキップ', // composer action tooltips composer_send: 'メッセージを送信', @@ -7063,7 +7063,7 @@ const LOCALES = { yolo_disabled: 'YOLO режим ВЫКЛ', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'YOLO режим активен — нажмите для отключения', - approval_skip_all: '⚡ Пропустить все в этом сеансе', + approval_skip_all: 'Пропустить все в этом сеансе', approval_skip_all_title: 'Пропустить все запросы подтверждения в этом сеансе', // composer action tooltips composer_send: 'Отправить сообщение', @@ -8830,7 +8830,7 @@ const LOCALES = { yolo_disabled: 'Modo YOLO desactivado', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'Modo YOLO activo — haz clic para desactivar', - approval_skip_all: '⚡ Saltar todo en esta sesión', + approval_skip_all: 'Saltar todo en esta sesión', approval_skip_all_title: 'Saltar todas las solicitudes de aprobación en esta sesión', // composer action tooltips composer_send: 'Enviar mensaje', @@ -10236,7 +10236,7 @@ const LOCALES = { yolo_disabled: 'YOLO-Modus AUS', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'YOLO-Modus aktiv — Klicken zum Deaktivieren', - approval_skip_all: '⚡ Alle in dieser Sitzung überspringen', + approval_skip_all: 'Alle in dieser Sitzung überspringen', approval_skip_all_title: 'Alle Genehmigungsanfragen in dieser Sitzung überspringen', // composer action tooltips composer_send: 'Nachricht senden', @@ -12312,7 +12312,7 @@ const LOCALES = { yolo_disabled: 'YOLO 模式已关闭', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'YOLO 模式激活 — 点击关闭', - approval_skip_all: '⚡ 本次会话全部跳过', + approval_skip_all: '本次会话全部跳过', approval_skip_all_title: '跳过本次会话的所有审批提示', // composer action tooltips composer_send: '发送消息', @@ -14208,7 +14208,7 @@ const LOCALES = { yolo_disabled: 'YOLO 模式已關閉', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'YOLO 模式已啟用，點選即可關閉', - approval_skip_all: '⚡ 本次工作階段全部跳過', + approval_skip_all: '本次工作階段全部跳過', approval_skip_all_title: '略過本次工作階段的所有核准提示', // composer action tooltips composer_send: '傳送訊息', @@ -17540,7 +17540,7 @@ const LOCALES = { yolo_disabled: 'YOLO 모드 꺼짐', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'YOLO 모드 활성 — 클릭하여 비활성화', - approval_skip_all: '⚡ 이 세션에서 모두 건너뛰기', + approval_skip_all: '이 세션에서 모두 건너뛰기', approval_skip_all_title: '이 세션의 모든 승인 요청 건너뛰기', // composer action tooltips composer_send: '메시지 보내기', @@ -19412,7 +19412,7 @@ const LOCALES = { yolo_disabled: 'Mode YOLO désactivé', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'Mode YOLO actif — cliquez pour désactiver', - approval_skip_all: '⚡ Ignorer tout cette session', + approval_skip_all: 'Ignorer tout cette session', approval_skip_all_title: 'Ignorer toutes les demandes d\'approbation pour cette session', composer_send: 'Envoyer le message', composer_queue: 'Mettre en file d\'attente', @@ -19540,7 +19540,7 @@ const LOCALES = { add_failed: 'Přidání selhalo: ', agent_soul: 'Duch agenta', all_uploads_failed: (n) => `Všech ${n} nahrávání selhalo`, - approval_skip_all: '⚡ Přeskočit vše pro tuto relaci', + approval_skip_all: 'Přeskočit vše pro tuto relaci', approval_skip_all_title: 'Přeskočit všechny výzvy ke schválení pro tuto relaci', archive_extracted: (n, c) => `Rozbaleno ${n} souborů z ${c} archivů`, auth_ack_save_failed: 'Nepodařilo se aktualizovat potvrzení: ', @@ -22894,7 +22894,7 @@ const LOCALES = { yolo_disabled: 'YOLO modu KAPALI', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'YOLO modu etkin — devre dışı bırakmak için tıklayın', - approval_skip_all: '⚡ Tüm bu oturumu atla', + approval_skip_all: 'Tüm bu oturumu atla', approval_skip_all_title: 'Bu oturuma ilişkin tüm onay istemlerini atla', // composer action tooltips composer_send: 'Mesaj gönder', @@ -24778,7 +24778,7 @@ const LOCALES = { yolo_disabled: 'Tryb YOLO WYŁĄCZONY', yolo_pill_label: 'YOLO', yolo_pill_title_active: 'Tryb YOLO aktywny — kliknij, aby wyłączyć', - approval_skip_all: '⚡ Pomiń wszystkie w tej sesji', + approval_skip_all: 'Pomiń wszystkie w tej sesji', approval_skip_all_title: 'Pomiń wszystkie monity o aprobatę w tej sesji', // composer action tooltips composer_send: 'Wyślij wiadomość', @@ -26411,7 +26411,7 @@ const LOCALES = { approval_gateway_unsupported: 'Phê duyệt cần gateway mới hơn. Hãy nâng cấp Hermes gateway đang kết nối để bật tính năng này.', approval_gateway_unsupported_label: 'Không hỗ trợ phê duyệt', approval_pending_count: (n) => `1 trong ${n} đang chờ`, - approval_skip_all: '⚡ Bỏ qua tất cả trong phiên này', + approval_skip_all: 'Bỏ qua tất cả trong phiên này', approval_skip_all_title: 'Bỏ qua mọi yêu cầu phê duyệt cho phiên này', checkpoint_title: 'Checkpoint', checkpoint_empty: 'Không tìm thấy checkpoint nào cho workspace này.', @@ -26983,15 +26983,68 @@ function resolveLocale(lang) { /** * Resolve locale with precedence: - * 1) primary (typically server setting) - * 2) fallback (typically localStorage) - * 3) English - * @param {string} primary - * @param {string} fallback + * 1) primary (typically server setting) + * 2) fallback (typically localStorage) + * 3) fallback2 (typically browser navigator hint — first-visit only) + * 4) 'en' (final fallback) + * + * Resolve a language preference to a known LOCALES key, applying a strict + * `primary -> fallback -> fallback2 -> 'en'` precedence chain. + * + * #7622 (round 3): the server no longer injects `"language": "en"` as a + * schema default — `load_settings()` reports `None` (the key is absent + * from the response) until the user explicitly picks a locale in the + * Settings modal. This lets the resolver use `primary` directly + * without a `primary === 'en'` special case: + * - an explicit `primary` (the server's stored value) always wins + * - a `None` / empty `primary` falls through to `fallback` (localStorage) + * - `fallback2` (the browser navigator hint) only fires when neither + * server nor localStorage has a preference + * - `'en'` is the final safety net + * + * Side benefit: a user who picked English on purpose (their settings + * file contains `"language": "en"`) is no longer overridden by a + * non-English browser locale on hydration, because the round-2 + * `primary === 'en'` skip has been removed. + * + * @param {string|null|undefined} primary server-stored language + * @param {string|null|undefined} fallback localStorage `hermes-lang` + * @param {string|null|undefined} fallback2 browser navigator hint * @returns {string} */ -function resolvePreferredLocale(primary, fallback) { - return resolveLocale(primary) || resolveLocale(fallback) || 'en'; +function resolvePreferredLocale(primary, fallback, fallback2) { + return resolveLocale(primary) + || resolveLocale(fallback) + || resolveLocale(fallback2) + || 'en'; +} + +/** + * Detect the browser's preferred language for first-visit locale + * selection. Guarded against `navigator` access exceptions: some + * embedded webviews (Tauri desktop shell, kiosk modes) report a + * `navigator` object but throw on `.languages` / `.language` reads. + * Callers (loadLocale, boot.js, panels.js) must use this helper rather + * than reading `navigator` directly so a single throw does not abort + * the boot or settings-panel hydration. + * + * @returns {string|null} the first entry of `navigator.languages`, or + * `navigator.language`, or `null` when neither is reachable. + */ +function _detectBrowserLanguageHint() { + try { + const nav = (typeof navigator !== 'undefined') ? navigator : null; + if (!nav) return null; + if (Array.isArray(nav.languages) && nav.languages.length) { + return nav.languages[0] || null; + } + if (typeof nav.language === 'string' && nav.language) { + return nav.language; + } + return null; + } catch (_) { + return null; + } } /** @@ -27029,11 +27082,26 @@ function setLocale(lang) { /** * Load locale from localStorage (called once at boot, before DOMContentLoaded). * Server-persisted preference is applied later in loadSettingsPanel(). + * + * #7622: on the FIRST visit (no stored preference), fall back to the + * browser's preferred language (`navigator.languages[0]` / + * `navigator.language`) so non-English speakers don't have to dig into + * Settings after every fresh install. This is strictly a first-visit + * hint — the moment the user picks a language in Settings (or the + * server-side preference is applied via `loadSettingsPanel`), the + * stored value takes over and the browser hint is never consulted + * again for that browser profile. */ function loadLocale() { let stored = null; try { stored = localStorage.getItem('hermes-lang'); } catch (_) {} - setLocale(resolvePreferredLocale(null, stored)); + // Only consult the browser hint when no stored preference exists. + // The settings panel and `loadSettingsPanel()` will overwrite this + // hint on their first call, so explicit user / server preferences + // always win. #7622 (round 3): the browser-hint read is delegated + // to `_detectBrowserLanguageHint()` so a `navigator` access throw + // (some embedded webviews) can no longer abort loadLocale(). + setLocale(resolvePreferredLocale(stored, _detectBrowserLanguageHint())); } /** diff --git a/static/login.js b/static/login.js index 3bba3bb1..d702ca67 100644 --- a/static/login.js +++ b/static/login.js @@ -7,7 +7,11 @@ document.addEventListener('DOMContentLoaded', function () { var input = document.getElementById('pw'); var passkeyBtn = document.getElementById('passkey-login'); - if (!form || !input) return; + // #7056: the password input is absent on a passwordless deployment (OIDC-only, + // or passkey-only). The form itself still renders and carries the i18n data + // attributes, and the passkey button below must still be wired up, so only the + // form is required here — every password-specific path is guarded individually. + if (!form) return; var invalidPw = form.getAttribute('data-invalid-pw') || 'Invalid password'; var connFailed = form.getAttribute('data-conn-failed') || 'Connection failed'; @@ -59,6 +63,8 @@ document.addEventListener('DOMContentLoaded', function () { async function doLogin(e) { e.preventDefault(); + // No password input on a passwordless deployment: nothing to submit. + if (!input) return; var pw = input.value; hideErr(); try { @@ -147,12 +153,14 @@ document.addEventListener('DOMContentLoaded', function () { passkeyBtn.addEventListener('click', doPasskeyLogin); } - input.addEventListener('keydown', function (e) { - if (e.key === 'Enter') { - e.preventDefault(); - doLogin(e); - } - }); + if (input) { + input.addEventListener('keydown', function (e) { + if (e.key === 'Enter') { + e.preventDefault(); + doLogin(e); + } + }); + } // On page load, probe the server so we can distinguish "can't reach server" // (Tailscale off, wrong network) from "session expired / need to log in". diff --git a/static/messages.js b/static/messages.js index 92a10712..81e021ec 100644 --- a/static/messages.js +++ b/static/messages.js @@ -1,3 +1,13 @@ +const _AGENT_COMMAND_ALIASES = { + 'reload_mcp': 'reload-mcp', + 'reload_skills': 'reload-skills', + 'codex_runtime': 'codex-runtime', + 'credits': 'credits' +}; +const _AGENT_COMMANDS_RUN_ON_WEBUI = new Set([ + 'reload-mcp','reload-skills','codex-runtime','credits', + 'reload_mcp','reload_skills','codex_runtime','credits' +]); function _markSessionViewed(sid, messageCount) { if(typeof _setSessionViewedCount!=='function' || !sid) return; const next = Number.isFinite(messageCount) ? Number(messageCount) : 0; @@ -1184,7 +1194,6 @@ const _sessionTitleProvisionalBySid = new Map(); // their canonical command is registered on the backend (for example // /reload-mcp). Keep this intentionally narrow and include underscore variants // observed by users so typing either form still routes through executeAgentCommand. -const _AGENT_COMMANDS_RUN_ON_WEBUI = new Set(['reload-mcp', 'reload_mcp', 'reload-skills', 'reload_skills', 'codex-runtime', 'codex_runtime', 'credits']); function _clearStaleBusyStateBeforeSend({compressionRunning=false}={}){ if(!S||!S.busy||compressionRunning) return false; @@ -1287,6 +1296,31 @@ function applySessionTitleUpdate(sid, titleText, options={}){ // BEFORE slash rewrites (/moa, bundles) mutate the payload and BEFORE // uploadPendingFiles() drains S.pendingFiles — so we restore what the user // actually typed, not the transformed send payload. +async function _recoverCompressedSend(error,sid,draftText,filesSnapshot,clearPromise){ + let payload; + try{ payload=JSON.parse(error&&error.body||'{}'); }catch(_){ return false; } + const target=payload&&payload.continuation_session_id; + if(!error||error.status!==409||!payload||payload.code!=='session_rotated'||typeof target!=='string'||!target||target===sid) return false; + // A failed POST has not admitted a turn. Never resend automatically: the + // continuation may already be busy, and attachments must remain a draft. + if(!S.session||S.session.session_id!==sid) return false; + delete INFLIGHT[sid]; + if(typeof clearInflightState==='function') clearInflightState(sid); + if(typeof clearOptimisticSessionStreaming==='function') clearOptimisticSessionStreaming(sid); + stopApprovalPolling();stopClarifyPolling();removeThinking();setBusy(false); + try{ + await loadSession(target); + if(!S.session||S.session.session_id===sid) return false; + // loadSession can lose its navigation race to another tab selection. Never + // place the rejected message into that unrelated session's composer. + _restoreComposerDraftAfterFailedSend(draftText,filesSnapshot,target,clearPromise); + if(S.session.session_id!==target) return true; + setComposerStatus('Session resumed. Your message is preserved; send it when ready.'); + showToast('Session resumed after compression. Your draft is preserved.',4000); + return true; + }catch(_){ return false; } +} + function _restoreComposerDraftAfterFailedSend(draftText, filesSnapshot, sid, clearPromise){ const restore=String(draftText||''); const files=Array.isArray(filesSnapshot)?filesSnapshot.filter(Boolean):[]; @@ -1334,7 +1368,7 @@ function _restoreComposerDraftAfterFailedSend(draftText, filesSnapshot, sid, cle } else if(!restoredVisible){ // Background failure (sid was never the visible session): no live // composer to read, so persist the captured snapshot — it's the only copy. - _saveComposerDraftNow(sid, restore, []); + _saveComposerDraftNow(sid, restore, files); } // else: restored the visible composer, then the user switched away — the // session-switch save path already saved sid's composer; skip stale write. @@ -1409,14 +1443,16 @@ async function send(){ if(!S.session){await newSession();await renderSessionList();} // Busy-control slash commands must be intercepted HERE, before the // defaultMessageMode routing block, so the user can always type /steer, /interrupt, - // /queue, /terminal, /goal, or /yolo while the agent is running and have + // /queue, /terminal, /goal, /yolo, or /stop while the agent is running and have // them execute immediately. // Without this intercept they fall through to the queue and execute after // the current turn ends — by which point there is no active stream and // cmdSteer / cmdInterrupt say "No active task to stop." + // /stop must cancel the active run immediately instead of being steered + // or queued as the literal text "/stop" (#6951). if(text.startsWith('/')&&!literalSlash){ const _pc=typeof parseCommand==='function'&&parseCommand(text); - if(_pc&&['steer','interrupt','queue','terminal','goal','yolo'].includes(_pc.name)){ + if(_pc&&['steer','interrupt','queue','terminal','goal','yolo','stop'].includes(_pc.name)){ const _bc=COMMANDS.find(c=>c.name===_pc.name); if(_bc){ $('msg').value='';autoResize(); @@ -1839,6 +1875,7 @@ async function send(){ if(typeof renderSessionList==='function') void renderSessionList(); return; } + if(await _recoverCompressedSend(e,activeSid,_failedSendDraftText,_failedSendFilesSnapshot,_composerDraftClearPromise)) return; const conflictActiveStream=/session already has an active stream/i.test(errMsg); if(conflictActiveStream){ delete INFLIGHT[activeSid]; @@ -2052,6 +2089,7 @@ function closeLiveStream(sessionId, streamId, source){ if(!live) return; if(streamId&&live.streamId!==streamId) return; if(source&&live.source!==source) return; + if(typeof live.cancelIdleRecovery==='function') live.cancelIdleRecovery(); // Snapshot the current live-turn DOM BEFORE tearing the stream down. The // per-event snapshot (snapshotLiveTurn) only fires on content/tool_complete // SSE events, so switching away during a quiet window (mid tool-exec, silent @@ -4808,7 +4846,12 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ function _smdMediaTailFlushEntry(entry){ const chunk=_smdMediaTailEntryChunk(entry); if(!chunk) return; - const m=/^MEDIA:([^\s\)\]]+)$/.exec(String(chunk)); + // #7680 re-gate (9/22): strip backtick wrappers so the bare-token + // match below sees a plain ``MEDIA:path`` and the bare class + // (no backtick in the exclusion set) captures the full filename + // even when the path itself contains a backtick. + const normalized = String(chunk).replace(/`MEDIA:([^`\s]+)`/g, 'MEDIA:$1'); + const m=/^MEDIA:([^\s\)\]]+)$/.exec(normalized); const emitted=!!(m && entry && entry.parent && _smdAppendMediaNode(entry.parent, m[1])); if(!emitted && entry) _smdMediaWriteText(entry.parent, entry.data, entry.baseAddText, entry.writeText, chunk); } @@ -4856,23 +4899,29 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ // Prose runs go through the owning text writer. MEDIA tokens go through // the single-token DOMParser helper only after a delimiter or // reliable filename suffix proves the ref is complete. + // #7680 re-gate (9/22): strip backtick wrappers first so the bare + // class (no backtick in the exclusion set) captures the full + // filename even when the path itself contains a backtick. + // The pre-pass replaces `` `MEDIA:path` `` with ``MEDIA:path`` + // so the wrapped form is consumed before the bare scan. + const normalized = combined.replace(/`MEDIA:([^`\s]+)`/g, 'MEDIA:$1'); const re=/MEDIA:([^\s\)\]]+)/g; let last=0, m; let unmatchedTail=null; - while((m=re.exec(combined))){ + while((m=re.exec(normalized))){ const matchEnd = m.index + m[0].length; if(m.index>last){ - const slice = combined.slice(last, m.index); + const slice = normalized.slice(last, m.index); writeCurrent(slice); } - if(matchEnd===combined.length && !_smdMediaRefHasReliableBoundary(m[1])){ - const candidate = combined.slice(m.index); + if(matchEnd===normalized.length && !_smdMediaRefHasReliableBoundary(m[1])){ + const candidate = normalized.slice(m.index); if(candidate.length < _MEDIA_TAIL_MAX){ unmatchedTail = candidate; } else { writeCurrent(candidate); } - last = combined.length; + last = normalized.length; break; } if(!_smdAppendMediaNode(parent, m[1])) writeCurrent(m[0]); @@ -4880,7 +4929,7 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ } // Tail buffer — hold trailing bytes that look like an unterminated // MEDIA prefix; flush any prose before the partial MEDIA suffix. - const rest = combined.slice(last); + const rest = normalized.slice(last); if(rest){ const tailMatch = /MEDIA:[^\s\)\]]*$/.exec(rest); const prefixTail = tailMatch ? '' : _smdMediaPrefixTail(rest); @@ -5708,12 +5757,82 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ return true; } + function _bindSidebarIdleRecovery(live){ + let pending=null; + live.cancelIdleRecovery=()=>{ + if(pending&&pending.timer) clearTimeout(pending.timer); + pending=null; + }; + live.recoverFromSidebarIdle=()=>{ + if(pending||_streamFinalized||_terminalStateReached||_pendingStreamEndRecovery) return; + const request={inflight:INFLIGHT[activeSid],timer:null}; + const isCurrent=()=>pending===request&&!_streamFinalized&&!_terminalStateReached&& + LIVE_STREAMS[activeSid]===live&&live.source.readyState===1&& + S.session&&S.session.session_id===activeSid&&S.activeStreamId===streamId&& + INFLIGHT[activeSid]===request.inflight&& + !(typeof _sendInProgress!=='undefined'&&_sendInProgress&&activeSid===_sendInProgressSid); + pending=request; + // Give the independently delivered terminal frame a short handoff window, + // then use canonical session recovery even if the transport stays OPEN. + // One ticket spans both timer and request; list refreshes cannot extend it. + request.timer=setTimeout(async()=>{ + request.timer=null; + if(!isCurrent()){ + if(pending===request) live.cancelIdleRecovery(); + return; + } + try{ + let runtimeStatus=null; + try{ + runtimeStatus=await api( + `/api/chat/stream/status?stream_id=${encodeURIComponent(streamId)}`, + {timeoutMs:8000,retries:0,timeoutToast:false} + ); + }catch(_){ + // Runtime ownership is authoritative here. If the probe itself + // fails, do not infer completion from already-cleared session + // fields; surface the existing interrupted-stream path instead. + if(isCurrent()) _handleStreamError(live.source); + return; + } + if(!isCurrent()) return; + if(!runtimeStatus||typeof runtimeStatus.active!=='boolean'){ + _handleStreamError(live.source); + return; + } + // Gateway success writeback clears persisted active/pending fields + // before post-turn goal evaluation and terminal event emission. + // STREAMS-backed status therefore owns this decision: an exact active + // runtime must keep its OPEN browser handoff even if the sidebar row + // already looks idle. + if(runtimeStatus.active) return; + const status=await _restoreSettledSession(live.source,{ + status:true, + isCurrent, + requestOptions:{timeoutMs:8000,retries:0,timeoutToast:false}, + preserveVisibleOnShorterTerminalSnapshot:true, + }); + // A stale sidebar response is not permission to terminate a worker + // which the authoritative session snapshot still reports as active. + if(isCurrent()&&status!=='restored'&&status!=='active') _handleStreamError(live.source); + }finally{ + if(pending===request) live.cancelIdleRecovery(); + } + },1500); + }; + for(const event of ['done','cancel','apperror','stream_end','error']){ + live.source.addEventListener(event,live.cancelIdleRecovery); + } + } + function _wireSSE(source){ const existingLive=LIVE_STREAMS[activeSid]; if(existingLive&&existingLive.source&&existingLive.source!==source){ + if(typeof existingLive.cancelIdleRecovery==='function') existingLive.cancelIdleRecovery(); try{if(existingLive.source.readyState!==2)existingLive.source.close();}catch(_){ } } LIVE_STREAMS[activeSid]={streamId,source}; + _bindSidebarIdleRecovery(LIVE_STREAMS[activeSid]); // Note on #631 Bug B: the original PR description stated the server // "replays buffered token events" on reconnect, and proposed resetting @@ -6987,13 +7106,16 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ async function _restoreSettledSession(source, options=null){ const returnStatus=!!(options&&options.status); + const isCurrent=options&&typeof options.isCurrent==='function'?options.isCurrent:null; + if(isCurrent&&!isCurrent()) return returnStatus?'stale':false; const preserveVisibleOnShorterTerminalSnapshot=!!(options&&options.preserveVisibleOnShorterTerminalSnapshot); if(_isActiveSession() && S.activeStreamId!==streamId){ _closeSource(source); return returnStatus?'stale':false; } try{ - const data=await api(`/api/session?session_id=${encodeURIComponent(activeSid)}`); + const data=await api(`/api/session?session_id=${encodeURIComponent(activeSid)}`,options&&options.requestOptions||{}); + if(isCurrent&&!isCurrent()) return returnStatus?'stale':false; // Opus #2852 race-fix: if a late `done` event ran the finalize path while // we were awaiting the network roundtrip, bail out — done already settled. if(_streamFinalized) return returnStatus?'restored':true; @@ -8091,7 +8213,22 @@ function _startHiddenActiveStreamPoll(sid) { if (S.activeStreamId) return; // already rendering; wait it out try { fetch(_apiUrl('api/session/status?session_id=' + encodeURIComponent(sid)), {credentials: 'same-origin'}) - .then(r => r.ok ? r.json() : null) + .then(r => { + // #7299: 404 Not Found / 410 Gone are TERMINAL for this + // session-owned poll. The session has been deleted or no + // longer exists in the active state directory, so further + // polls are guaranteed to fail. Stop the poll immediately + // to avoid the infinite 404 loop on stale background tabs + // (one tab could fire ~10 such requests per minute; multiple + // tabs multiply the noise). Transient failures (5xx, rate + // limit, network error) keep polling — only the missing + // session itself is terminal. + if ((r.status === 404 || r.status === 410) && _sessionStreamHiddenPollSid === sid) { + _stopHiddenActiveStreamPoll(); + return null; + } + return r.ok ? r.json() : null; + }) .then(d => { if (!d || _sessionStreamHiddenPollSid !== sid) return; const streamId = d.active_stream_id; @@ -8951,6 +9088,19 @@ async function respondClarify(response) { // not tear B down on A's late 409. The SSE/poll path will re-render the // next prompt's card from scratch via ``showClarifyCard`` either way. if (e && e.status === 409) { + // #7710: a cross-profile refusal now also arrives as 409 + // (``session_profile_mismatch``). The prompt is NOT expired — the write + // was refused because the session belongs to another profile. Treating + // it as expired would hide a live clarification card and mislabel the + // cause, so leave the card standing and report the real reason. + if (typeof _sessionProfileMismatchFromError === 'function' + && _sessionProfileMismatchFromError(e)) { + _clarifySetControlsDisabled(false, false); + if (typeof setStatus === "function") { + setStatus("Clarify: session belongs to a different profile"); + } + return; + } if (_clarifyId === clarifyId) { // Same card still showing — dismiss it and rescue the typed draft. // Order matters: ``_stashClarifyDraft`` (called from diff --git a/static/panels.js b/static/panels.js index dd5090c3..9215cc77 100644 --- a/static/panels.js +++ b/static/panels.js @@ -9300,8 +9300,17 @@ async function loadSettingsPanel(){ _setHiddenTabs(hiddenTabs); _applyTabVisibility(hiddenTabs); _renderTabVisibilityChips(); + // #7622 (round 3): the settings payload's `settings.language` is + // absent (None) for a fresh install, so an explicit non-empty + // value is the user's genuine saved choice. The browser + // navigator hint is now read via the guarded + // `_detectBrowserLanguageHint()` helper (round-3 finding 2) so + // a throwing `navigator` accessor can no longer abort settings + // hydration before model, provider, plugin and extension sections + // are populated. The fallback ternary preserves the pre-#7622 + // settings-modal behaviour when neither helper is in scope. const resolvedLanguage=(typeof resolvePreferredLocale==='function') - ? resolvePreferredLocale(settings.language, localStorage.getItem('hermes-lang')) + ? resolvePreferredLocale(settings.language, localStorage.getItem('hermes-lang'), _detectBrowserLanguageHint()) : (settings.language || localStorage.getItem('hermes-lang') || 'en'); // Keep settings modal and current page strings in sync with the resolved locale. if(typeof setLocale==='function'){ diff --git a/static/sessions.js b/static/sessions.js index c6a302a4..f748f45a 100644 --- a/static/sessions.js +++ b/static/sessions.js @@ -852,6 +852,14 @@ function _reconcileActiveSessionIdleStateFromList(serverRows) { const serverRow=serverRows.find(s=>s&&s.session_id===sid); if (!serverRow) return false; if (!_isServerIdleSessionRow(serverRow)) return false; + // Sidebar idle metadata can beat the terminal frame on the independent chat + // SSE. Let its exact OPEN transport finish the Anchor handoff; orphaned or + // disconnected streams still use the existing idle recovery below. + if (_hasOwnedOpenLiveStream(sid)) { + const live=LIVE_STREAMS[sid]; + if(typeof live.recoverFromSidebarIdle==='function') live.recoverFromSidebarIdle(); + return false; + } let changed=false; if (S.busy) { S.busy=false; changed=true; } if (S.activeStreamId) { S.activeStreamId=null; changed=true; } @@ -940,6 +948,8 @@ function _purgeStaleInflightEntries() { if (typeof _sendInProgress !== 'undefined' && _sendInProgress && sid === _sendInProgressSid) { continue; } + // The sidebar render must not purge what the idle reconciler preserved. + if (_hasOwnedOpenLiveStream(sid)) continue; if (!sessionsById.has(sid)) { const knownSource = sourceById ? sourceById.get(sid) : null; if (currentSidebarSource && (!knownSource || knownSource !== currentSidebarSource)) { @@ -961,6 +971,14 @@ function _purgeStaleInflightEntries() { } } +function _hasOwnedOpenLiveStream(sid) { + if (typeof S === 'undefined' || !S || !S.session || S.session.session_id !== sid || !S.activeStreamId) return false; + const live = typeof LIVE_STREAMS === 'object' && LIVE_STREAMS ? LIVE_STREAMS[sid] : null; + // readyState 1 is EventSource.OPEN. A cached stream ID or busy flag alone + // is not ownership and must never disable recovery for a stuck indicator. + return Boolean(live && live.streamId === S.activeStreamId && live.source && live.source.readyState === 1); +} + function _rememberSessionListSource(s, sid = null, allowScopeFallback = true) { const resolvedSid = sid || (s && s.session_id); if (!resolvedSid) return; @@ -2169,6 +2187,12 @@ async function loadSession(sid){ S.activeStreamId=activeStreamId; const liveToolReplayId=(tc)=>String(tc&&(tc.tid||tc.id||tc.tool_call_id||tc.tool_use_id||tc.call_id||'')||'').trim(); const replayPersistedLiveToolCards=(opts)=>{ + // The journal-backed Anchor scene is authoritative through its resume + // cursor; newer rows arrive through the reattached SSE stream. Replaying + // the older INFLIGHT tool cache after a successful scene restore would + // redraw all N rows N times. Keep the #3707 replay only for legacy HTML + // restoration or a failed/unavailable scene render. + if(restoredAnchorScene) return; const liveToolCalls=Array.isArray(S.toolCalls) ? S.toolCalls : (Array.isArray(INFLIGHT[sid]&&INFLIGHT[sid].toolCalls)?INFLIGHT[sid].toolCalls:[]); @@ -5272,7 +5296,8 @@ function _shouldKeepLocalOnlyOptimisticSessionRow(local){ function _dropStaleOptimisticSessionRow(sid){ if(!sid) return; if(typeof _rememberSessionListSource==='function') _rememberSessionListSource(null, sid, false); - if(INFLIGHT&&INFLIGHT[sid]){ + // Retiring sidebar optimism must not retire the independent chat owner. + if(INFLIGHT&&INFLIGHT[sid]&&!_hasOwnedOpenLiveStream(sid)){ delete INFLIGHT[sid]; if(typeof clearInflightState==='function') clearInflightState(sid); } diff --git a/static/ui.js b/static/ui.js index 37742c5a..201eb7fd 100644 --- a/static/ui.js +++ b/static/ui.js @@ -508,6 +508,16 @@ async function startCompressionRecovery(btn){ const composer=$('msg'); if(composer&&typeof composer.focus==='function') composer.focus(); }catch(e){ + // #7710: a cross-profile refusal now also arrives as 409 + // (``session_profile_mismatch``). That is NOT a stale recovery action — + // the card is still valid, the request was simply refused because the + // session belongs to another profile. Retiring it would hide a live card + // and show a false "conversation already moved on" note. + if(e&&e.status===409&&typeof _sessionProfileMismatchFromError==='function' + &&_sessionProfileMismatchFromError(e)){ + if(typeof setStatus==='function') setStatus('Session belongs to a different profile'); + return; + } // A 409 means this session no longer has an active recovery action (the // session already moved on — e.g. a substantive prompt cleared it). The // persisted card in the transcript is stale, so retire it and show a neutral @@ -3939,6 +3949,27 @@ function _isEquivalentConfiguredModelEntry(modelId,badge,entries){ // providers (@custom:name:model) without collapsing matching model IDs from // different providers. const rawId=String(modelId||''); + // `/` is another routing spelling of ``, so its badge + // key must not become a second picker row. configured_model_badges holds every + // spelling of a configured model and renderModelDropdown() synthesises a row + // for each key this predicate does not recognise. The provider-qualified + // spelling was missed because _normalizeConfiguredModelKey() strips only one + // leading slash segment (#3360 keeps `vendor_a/x` and `vendor_b/y/x` distinct), + // so `acme/example-model` and `custom/acme/example-model` normalise to + // different keys and the picker lists one model twice. + // Match it the way the `@provider:` rule below does: the badge declares a + // provider, the key starts with that provider's `/` prefix, and an + // existing row from the same provider normalises equal to the remainder. Two + // different models never satisfy the last clause, so this can only drop a + // duplicate of a row the catalog already produced. + const slashPrefix=provider?`${provider}/`:''; + if(slashPrefix&&rawId.toLowerCase().startsWith(slashPrefix)){ + const slashRoutedId=rawId.slice(slashPrefix.length); + if(slashRoutedId&&(entries||[]).some(entry=> + String(entry.providerId||'').toLowerCase()===provider + &&_normalizeConfiguredModelKey(entry.value)===_normalizeConfiguredModelKey(slashRoutedId) + )) return true; + } const prefix=provider?`@${provider}:`:''; if(!prefix||!rawId.toLowerCase().startsWith(prefix)) return false; const routedId=rawId.slice(prefix.length); @@ -7692,6 +7723,16 @@ function renderMd(raw){ // generated images) and replace them with inline or download links. // Stashed so the path/URL is never processed as markdown. const media_stash=[]; + // #7680 re-gate (9/22): two-pass scan. + // 1. `` `MEDIA:path` `` (backtick-wrapped, inline-code form) → strip + // the wrapping backticks so the bare-token pass below sees a + // plain ``MEDIA:path`` and the closing backtick is not consumed + // as part of the path. + // 2. ``MEDIA:[^\s\)\]]+`` (bare, no backtick in the exclusion + // class) so a filename that legally contains a backtick + // (``report`final.png``) is captured in full instead of being + // truncated at the first backtick. + s=s.replace(/`MEDIA:([^`\s]+)`/g,'MEDIA:$1'); s=s.replace(/MEDIA:([^\s\)\]]+)/g,(_,raw_ref)=>{ media_stash.push(raw_ref); return '\x00D'+(media_stash.length-1)+'\x00'; @@ -9139,6 +9180,12 @@ function _stripForTTS(text){ // Strip links, keep text text=text.replace(/\[([^\]]+)\]\([^)]+\)/g,'$1'); // Replace MEDIA: paths with a simple label + // #7680 re-gate (9/22): TTS does not need the backtick-aware + // terminator. Inline code was already stripped at the top of this + // function (`:9137`), so a bare ``MEDIA:[^\s]+`` is correct and + // round-trips the original filename including a backtick in the + // path — the only thing the TTS ever does with the captured + // substring is throw it away. text=text.replace(/MEDIA:[^\s]+/g,'a file'); // Strip emoji and emoticons text=text.replace(/[\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F1E0}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}\u{FE00}-\u{FE0F}\u{1F900}-\u{1F9FF}\u{1FA00}-\u{1FA6F}\u{1FA70}-\u{1FAFF}\u{200D}]/gu,''); @@ -9918,6 +9965,99 @@ function _liveAssistantSegmentTextLength(seg){ return String(body.textContent||'').trim().length; } +// ── #6948 follow-up: settled-transcript ownership of a live-turn node ────── +// (duplicate assistant answer; upstream symptom report #2051) +// +// Both live-turn insertion paths — the renderMessages() re-attach and +// restoreLiveTurnHtmlForSession() — have one branch that ADDS a turn the +// settled rebuild did not produce. The assistant row is persisted a few ms +// BEFORE the stream's terminal event clears S.activeStreamId, so a dead live +// node left behind by an INFLIGHT entry that outlived its stream is re-attached +// on top of the settled transcript: the same answer twice, self-sustaining +// across later renders (the node keeps id=liveAssistantTurn and is re-preserved +// by the #6948 guard), healed only by a reload. +// +// Dropping a live node must be an OWNERSHIP proof, never "the rendered text +// looks the same": +// * the live body is built by the streaming smd parser and the settled body +// by renderMd + post-processing, so their textContent differs for anything +// past single-paragraph plain text (lists, fenced code with its language +// label, tables, `_underscore_` emphasis, the injected copy button) — a +// text comparison would miss most real answers; and +// * an identical answer earlier in the history (a repeated "Done.", a +// greeting, a same-prompt resend) would delete a GENUINELY LIVE turn: while +// the current turn is unpersisted, the last settled assistant message IS the +// previous turn's answer. +// So ownership is proved from state: the transcript must END with a settled +// assistant message that THIS stream produced, and the node must carry nothing +// the settled rebuild could not have produced. + +// Stream identity persisted on a settled assistant message. The server stamps +// _anchor_stream_id from the anchor-scene sidecar record (api/routes.py); the +// client stamps _anchor_stream_id and scene.identity.stream_id at stream end +// (_attachProjectedAnchorSceneToLastAssistant, static/messages.js). +function _settledAssistantStreamId(message){ + if(!message) return ''; + const scene=message._anchor_activity_scene||null; + const identity=(scene&&scene.identity)||null; + return String(message._anchor_stream_id||(scene&&scene.stream_id)||(identity&&identity.stream_id)||''); +} +// The live-projection markers the #6948 preserve guard already treats as proof +// of a live owner. A message the client still considers live is never evidence +// that the turn has settled. +function _messageHasLiveAssistantProjection(m){ + return !!(m&&m.role==='assistant'&&(m._live||m._activityBurstId!==undefined||m._liveSegmentSeq!==undefined)); +} +// True when the live turn holds something the settled rebuild cannot have +// produced from S.messages — an unpersisted tool card, reasoning/thinking row, +// transparent-stream row, or a second live segment. Such a node is never +// dropped: #3714's whole premise is that the live DOM can be AHEAD of +// S.messages, and a tail-segment match must not discard the tool card or the +// earlier segment above it. An unknown node shape fails closed (keep the turn). +function _liveTurnCarriesUnsettledContent(turn){ + if(!turn||typeof turn.querySelectorAll!=='function') return true; + if(turn.querySelectorAll( + '.tool-card-row,.wl-reason,.agent-activity-thinking,.thinking-card-row,.transparent-event-row' + ).length) return true; + return turn.querySelectorAll('[data-live-assistant="1"]').length>1; +} +function _settledTranscriptOwnsLiveTurn(sid, turn){ + if(!turn) return false; + const msgs=(typeof S!=='undefined'&&S&&Array.isArray(S.messages))?S.messages:null; + if(!msgs||!msgs.length) return false; + // The transcript must END with a settled assistant answer. A trailing user + // turn means the live turn IS the current turn — nothing of it is persisted + // yet, so it can never be a leftover (this is the case a text comparison gets + // wrong when the previous answer happens to read the same), and a live + // projection anywhere means the rebuild still owns a live turn of its own. + const last=msgs[msgs.length-1]; + if(!last||last.role!=='assistant') return false; + if(msgs.some(_messageHasLiveAssistantProjection)) return false; + const inflight=(typeof INFLIGHT!=='undefined'&&INFLIGHT)?INFLIGHT[sid]:null; + // Owner of the live DOM: INFLIGHT carries the id of the stream that built it + // (attachLiveStream), and S.activeStreamId is that same id until the terminal + // event clears it. + const liveStreamId=String((inflight&&inflight.streamId)||(typeof S!=='undefined'&&S&&S.activeStreamId)||''); + const settledStreamId=_settledAssistantStreamId(last); + if(settledStreamId){ + // Identity recorded on both sides decides — renderer- and text-independent, + // so code blocks, lists and tables are handled like plain prose. + if(!liveStreamId||settledStreamId!==liveStreamId) return false; + }else{ + // No persisted identity on the tail (a turn whose scene was not worklog + // worthy). Compare SOURCE with SOURCE — the markdown this stream produced + // (INFLIGHT.lastAssistantText) against the persisted message content — + // never two independently rendered DOM trees. Regeneration cannot reach + // this branch: startRegeneration() truncates S.messages at the user turn, + // so the tail is a user message while a regenerated answer streams. + const streamed=String((inflight&&inflight.lastAssistantText)||'').replace(/\s+/g,' ').trim(); + if(!streamed) return false; + const settled=String((typeof msgContent==='function'?msgContent(last):last.content)||'').replace(/\s+/g,' ').trim(); + if(!settled||streamed!==settled) return false; + } + return !_liveTurnCarriesUnsettledContent(turn); +} + function _mergeRestoredLiveAssistantSegment(restored, existing){ if(!restored||!existing) return; const existingLive=existing.querySelector('[data-live-assistant="1"]'); @@ -9954,6 +10094,18 @@ function restoreLiveTurnHtmlForSession(sid){ const existing=$('liveAssistantTurn'); _mergeRestoredLiveAssistantSegment(restored, existing); if(existing) existing.replaceWith(restored); + // #6948 follow-up (duplicate assistant answer; #2051): only this branch ADDS a + // turn — replacing an existing live turn cannot duplicate. When the settled + // transcript already owns the stream this snapshot belongs to, appending it + // pins a SECOND copy of the same answer under the settled one. Release the + // stale snapshot (no other consumer can use it — it would be re-refused on + // every later restore) and report "nothing restored", so loadSession takes the + // same path it already takes for an INFLIGHT entry that never carried one. + else if(typeof _settledTranscriptOwnsLiveTurn==='function' + &&_settledTranscriptOwnsLiveTurn(sid, restored)){ + inflight.liveTurnHtml=null; + return false; + } else inner.appendChild(restored); // Transparent Stream: liveTurnHtml is restored via template.innerHTML, which // drops the property-bound onclick/onkeydown handlers wired by @@ -11459,6 +11611,15 @@ function _assistantMessageBelongsInWorklog(m, rawIdx, toolCallAssistantIdxs, vis const isTurnFinalAssistant=!!(opts&&opts.isTurnFinalAssistant); const visibleText=String(visibleContent!==undefined?visibleContent:msgContent(m)||'').trim(); const hasVisibleText=!!visibleText&&!_isAssistantEmptyPlaceholderContent(m, visibleText); + // The caller only consults this predicate once the turn has settled (it gates + // on `!S.busy`), so an `_live` marker seen here is a leftover of the + // live-snapshot projection, not an ongoing stream. Folding on it hid the turn's + // final answer inline and echoed it into the Worklog, leaving the turn with no + // visible content until the #3875 blank-turn fail-safe revealed both copies. A + // turn-final assistant message with visible text is the answer, so keep it out + // of the fold; every other `_live` message folds as before. Kept as a separate + // guard so the live rule below keeps its position. + if(m._live&&hasVisibleText&&isTurnFinalAssistant) return false; if(m._live) return true; if(hasVisibleText&&m._anchor_activity_scene) return false; if(hasVisibleText&&isTurnFinalAssistant) return false; @@ -13141,6 +13302,17 @@ function _anchorSceneRowsForRendering(scene, opts){ const rows=Array.isArray(scene&&scene.activity_rows)?scene.activity_rows:[]; const settled=!!(opts&&opts.settled); const live=!settled; + // #6948 follow-up (duplicate assistant answer; #2051): on a SETTLED turn the + // assistant segment owns the final answer, so a process_prose row carrying + // that same text must not also be rebuilt into the activity scene. The + // transparent-stream renderer suppresses it per row + // (_anchorSceneTransparentNodeForRow), but the compact-worklog row builder + // (_anchorSceneNodeForRow) has no such guard and renders the row as a second + // .assistant-segment above the settled one. Drop it here — the single place + // BOTH settled renderers (and the #5839 deferred-rows path) get their rows. + // LIVE rendering is untouched: while streaming, the inline live segment is + // hidden and the prose row IS the visible answer. + const settledFinalAnswer=settled?String((scene&&scene.final_answer)||'').trim():''; const out=[]; const byKey=new Map(); const liveProseTextKeys=new Map(); @@ -13163,6 +13335,9 @@ function _anchorSceneRowsForRendering(scene, opts){ if(_anchorSceneIsSettledSuccessfulCompression(row,settled)) continue; const text=String(row.text||'').trim(); if((row.role==='prose'||row.role==='thinking')&&!text) continue; + if(settledFinalAnswer&&row.role==='prose' + &&typeof _anchorSceneProseDuplicatesFinalAnswer==='function' + &&_anchorSceneProseDuplicatesFinalAnswer(text,settledFinalAnswer)) continue; // #6948 follow-up const key=keyFor(row); if(byKey.has(key)){ const index=byKey.get(key); @@ -13437,6 +13612,26 @@ function _anchorSceneProseMatchesFinalAnswer(proseText, finalAnswer){ const shorter=Math.min(a.length,b.length), longer=Math.max(a.length,b.length); return shorter>=80 && (shorter/longer)>=0.9; } +// #6948 follow-up (duplicate assistant answer; #2051): the same near-equality +// test as _anchorSceneProseMatchesFinalAnswer, minus its absolute `shorter>=80` +// floor. That floor makes the matcher a no-op for every SHORT final answer +// (greetings, "Done.", one-liners), so a prose row that is the answer minus its +// last streamed token — e.g. "Hey! What can I help you with today" vs +// "...today?" — was rebuilt as a SECOND assistant-segment beside the settled +// one. The >=0.9 length ratio is what actually keeps a distinct short +// intermediate sentence from being swallowed by a long final answer (Codex +// #4568), so it alone is kept. Deliberately a separate helper: the per-row +// render-time matcher above, and the tests pinning its exact predicate, stay +// untouched. +function _anchorSceneProseDuplicatesFinalAnswer(proseText, finalAnswer){ + if(_anchorSceneProseMatchesFinalAnswer(proseText,finalAnswer)) return true; + const norm=(s)=>String(s||'').replace(/\s+/g,' ').trim(); + const a=norm(proseText), b=norm(finalAnswer); + if(!a||!b) return false; + if(!(a.startsWith(b)||b.startsWith(a))) return false; + const shorter=Math.min(a.length,b.length), longer=Math.max(a.length,b.length); + return (shorter/longer)>=0.9; +} function _anchorSceneWorklogGroup(blocks, opts){ if(!blocks) return null; const live=!!(opts&&opts.live); @@ -16912,6 +17107,23 @@ function _processWakeupCardHtml(info, rawText, extras){ return `
${li('chevron-right',12)}${li('terminal',13)}${esc(t('process_wakeup_label'))}${cmdHtml}${chip}${extras.timeHtml||''}
${extras.filesHtml||''}${patternRow}${cmdRow}
${outHtml}
${extras.footHtml||''}
`; } +// #2051: parse into a