diff --git a/.env.example b/.env.example index 3a3d4c13..a96b87f5 100644 --- a/.env.example +++ b/.env.example @@ -132,6 +132,13 @@ # Extra sources for the notes drawer (see docs for the expected format). # HERMES_WEBUI_EXTERNAL_NOTES_SOURCES= +# Sidebar recency window: how many recent sessions are listed (default: 20). +# Also caps how many delegated subagent children can nest at once; raise it +# for wide fan-outs. Invalid or non-positive values fall back to 20. +# Values above 200 are clamped. Resolved before profile init, so a profile +# .env cannot override this server-wide bound. +# HERMES_WEBUI_VISIBLE_SESSION_LIMIT=20 + # Directory scanned for WebUI plugins (default: ~/.hermes/plugins). # HERMES_WEBUI_PLUGINS_DIR=~/.hermes/plugins diff --git a/.github/workflows/conversation-lifecycle.yml b/.github/workflows/conversation-lifecycle.yml index e952d34f..6f49c3c8 100644 --- a/.github/workflows/conversation-lifecycle.yml +++ b/.github/workflows/conversation-lifecycle.yml @@ -13,6 +13,7 @@ on: - 'server.py' - 'tests/browser_conversation_lifecycle.py' - 'tests/browser_historical_transcript_hydration.py' + - 'tests/browser_reconnect_scene_redraw.py' - 'requirements*.txt' - 'pyproject.toml' - '.github/workflows/conversation-lifecycle.yml' @@ -24,6 +25,7 @@ on: - 'server.py' - 'tests/browser_conversation_lifecycle.py' - 'tests/browser_historical_transcript_hydration.py' + - 'tests/browser_reconnect_scene_redraw.py' - 'requirements*.txt' - 'pyproject.toml' - '.github/workflows/conversation-lifecycle.yml' @@ -32,7 +34,8 @@ jobs: live-to-final: runs-on: ubuntu-latest timeout-minutes: 10 - continue-on-error: true + # Keep existing proof rows informational; reconnect regressions fail CI. + continue-on-error: ${{ matrix.name != 'reconnect-scene-redraw' }} strategy: fail-fast: false matrix: @@ -45,6 +48,9 @@ jobs: scenario: terminal-error - name: historical-transcript-hydration script: tests/browser_historical_transcript_hydration.py + - name: reconnect-scene-redraw + script: tests/browser_reconnect_scene_redraw.py + browsers: chromium steps: - uses: actions/checkout@v4 @@ -69,6 +75,7 @@ jobs: LIFECYCLE_ARTIFACT_DIR: lifecycle-artifacts HISTORICAL_HYDRATION_ARTIFACT_DIR: lifecycle-artifacts LIFECYCLE_SCENARIO: ${{ matrix.scenario }} + BROWSERS: ${{ matrix.browsers }} run: python ${{ matrix.script }} - name: Run settlement frame proof diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 72be8bc8..2ce017cf 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -237,6 +237,29 @@ Session is a plain Python class (not a dataclass, not SQLAlchemy): title_from(): takes messages list, finds first user message, returns first 64 chars. Called after run_conversation() completes to set the session title retroactively. +#### Session transcript reconciliation with `state.db` + +`reconciled_state_db_messages_for_session()` uses +`merge_session_messages_append_only()` to combine a WebUI sidecar or context +projection with active Agent `state.db` rows. Its explicit +`incoming_provenance="state_db"` fence permits a state-only row whose timestamp +predates the sidecar tail to use a safe chronological slot. Paginated +`msg_limit` consumers rely on this merged order directly rather than applying a +later timestamp sort. + +The same merge helper also stitches ordered child sidecars onto archived +compression parents. Those calls leave incoming provenance unverified, so their +stable message sequence remains append-only even when parent rows carry later, +restamped timestamps. Timestamp alone is never authority to move a continuation +inside its parent transcript. + +If an older row could only be placed before the first surviving sidecar/context +row, the insertion helper declines it to avoid resurrecting compacted history. +Reconciliation then appends that row instead of dropping it; rows without a +usable timestamp and rows at or after the sidecar tail also append normally. +The fallback therefore preserves an accepted state-only row when exact ordering +is ambiguous, while safely placeable recovery rows remain chronological. + #### Imported `state.db` sidebar projection `api.models.get_cli_sessions()` projects conversations from the active Hermes diff --git a/CHANGELOG.md b/CHANGELOG.md index 60f5ee11..9e4b74bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,91 @@ ## [Unreleased] +### Added + +- **The sidebar's recent-session window is configurable.** `HERMES_WEBUI_VISIBLE_SESSION_LIMIT` sets + how many recent sessions the sidebar lists (default 20). It also bounds how many delegated subagent + children can nest at once, so raise it for wide fan-outs. Invalid or non-positive values fall back to + 20, values above 200 are clamped, and it is resolved before profile init so a profile `.env` cannot + override it. (#7631 by @carlotestor) + +### Performance + +- **Opening a session while its task is still running is much faster.** Rebuilding the live + snapshot from the run journal parsed the journal twice, walked every metering row and grew the + reasoning text with repeated string concatenation, which is quadratic on long runs. The journal + is now parsed once, metering rows are skipped (their timestamp watermark is kept) and reasoning + deltas are joined once per segment. On a 15 MB / 22.7k-event journal the author measured the + rebuild going from 4.34 s to 0.49 s with a byte-identical snapshot. The interim-echo check now matches a + compact-equivalent suffix without a fixed window, so an echo stretched by interior whitespace is + no longer shown twice. (#7310, #7569 by @happy5318) + ### Fixed -- **A rejected request no longer poisons the next one on the same connection.** `server.py` +- **Reconnecting to a running session no longer redraws every tool card over and over.** After a + reconnect restored the live activity scene from the run journal, the client also replayed its older + cached in-flight tool list on top, so a turn with N tool cards redrew them N times. When the + journal-backed scene restores successfully the replay is now skipped (newer rows still arrive + through the reattached stream); the replay stays for legacy restores and for a failed or + unavailable scene. (#7436, @atchisonbrent) + +- **A settled assistant answer is no longer shown twice (#2051).** Two client-side paths could put a + second copy of a finished answer on screen after a turn settled: a stale live-turn node that was + still treated as live, and a settled rebuild branch that appended a turn instead of replacing it. + The same fix stops an interrupted turn's notice from appearing twice, and stops the composer's + model picker from listing one configured custom-provider model twice. The stored transcript was + always correct; only the rendering duplicated it. (#7374, @Thireus) + +- **A hidden browser tab stops polling a session that was deleted.** When a tab is in the + background, its stream poll kept asking for a session every few seconds after it was deleted or + archived away, forever. A `404`/`410` for the polled session now stops that poll; `503` and network + errors keep retrying, a late `404` for one session cannot stop another session's poll, and making + the tab visible again reopens the live stream. (#7299, #7716 by @happy5318) +- **The OpenRouter setup no longer offers a model OpenRouter doesn't serve.** Onboarding offered + `z-ai/glm-4.5-flash`, which is not in OpenRouter's catalog, so picking it failed on the first + message. That slot now offers `z-ai/glm-4.5-air`, Z.AI's nearest light model. The direct Z.AI + setup still offers GLM-4.5 Flash. (#7520, #7734 by @MuhammadUsamaMX) +- **Opening the sidebar can no longer stall the agent's writes to `state.db`.** Several WebUI + paths that only read the agent's `state.db` could quietly become writers. The read-only opener + fell back to a writable connection when `mode=ro` failed. The session listing self-healed a + missing `idx_messages_session` with `CREATE INDEX` through its own writable connection, which + holds the SQLite writer lock for minutes on a large `messages` table. The cron sidebar, insights + and deep-health checks opened the database with a bare `sqlite3.connect()`. Every one of these + readers now opens strictly read-only (`file:...?mode=ro`) and never creates an index. A missing + index falls back to the existing pre-aggregated listing. The gateway watcher also no longer polls + the database when no client is subscribed. Operators with an older agent can create the read + indexes in a drained maintenance window with `scripts/ensure_state_db_read_indexes.py`; see + `docs/troubleshooting.md`. (#7445 by @ruizanthony) + +- **A failed chat launch no longer leaves the session stuck "running".** If the worker thread + couldn't start, its stream ownership and the session's pending-stream fields were never cleared, + so the session could look permanently active and the registries grew. Launch failure now clears + exactly that stream's records, after releasing the chat-start lock, and never evicts a successor + stream. (#6869, #6937 by @jbdrak) +- **A compressed conversation no longer shows up twice in the sidebar.** When context + compression started the continuation a few milliseconds before the parent was marked ended, the + continuation wasn't recognised. The sidebar then showed a duplicate same-title row plus a + spurious child-session entry, and opening the conversation didn't stitch the transcript. Both the + sidebar and the transcript stitcher now accept a bounded (2 s) early start when every other + lineage signal agrees: same source, direct parent link, a compression (or `cli_close`) end reason and not a fork. + (#6931, #7021 by @webtecnica) +- **The profile switcher no longer 500s in a two-container Docker setup.** With the agent + source not mounted (`HERMES_WEBUI_CHAT_BACKEND=gateway`), `GET /api/profiles` fell through to + a skills-stats fallback that imported `agent.skill_utils` unguarded, so the missing module + surfaced as an error on every profile-list load. The import is now guarded and the skill + stats report as unknown in that case, so the picker stays usable (the skills line is simply + omitted). Thanks @webtecnica. (#7305, #7312) + +- **A title that mixes Chinese, Japanese or Korean with English terms is no longer rejected.** + The cross-script guard that stops a generated title from drifting into the wrong language + treated the borrowed Latin words in a CJK title as drift, so a valid title like + `WeChat Pay 回调失败排查` or `Python 代码修复` was thrown away and the session kept its + fallback title. Latin terms are now accepted when the title also contains CJK text, while + an all-Latin title for a CJK conversation is still rejected. Thanks @MuhammadUsamaMX. + (#7693, #7727) + +- **A late-arriving prompt no longer renders below the reply it asked for.** When a message + reached the transcript from `state.db` after the sidecar had already been merged⟪HERMES-CONTEXT-COMPRESSION: 809 of 1,009 chars omitted here by Hermes's context compressor. This is NOT part of the original tool call and must never be reproduced in new output — always write full, untruncated content.⟫- **A rejected request no longer poisons the next one on the same connection.** `server.py` is a raw HTTP/1.1 handler where `rfile` is the socket itself, so answering a request before reading its body left those bytes queued. The next request on a keep-alive connection was then parsed starting mid-body, and the client got diff --git a/README.md b/README.md index 7e73b9f1..76429aea 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,7 @@ Full list of environment variables: | `HERMES_HOME` | Windows: `%LOCALAPPDATA%\hermes`; POSIX: `~/.hermes` | Base directory for Hermes state (affects all paths) | | `HERMES_CONFIG_PATH` | `$HERMES_HOME/config.yaml` | Path to Hermes config file | | `HERMES_WEBUI_SERVER_CWD` | *(unset)* | Working directory for the server process. Defaults to the agent dir; point it at a writable workspace when the agent dir is read-only so fallback relative writes land somewhere writable | +| `HERMES_WEBUI_VISIBLE_SESSION_LIMIT` | `20` | Size of the sidebar's interactive recency window (how many recent non-cron/webhook sessions are listed). Also bounds how many delegated subagent children can nest at once, since a child only renders when its row wins a slot in the window — raise it for wide fan-outs. Non-integer or non-positive values fall back to the default. Values above 200 are clamped. Resolved before profile init, so a profile `.env` cannot override it | | `HERMES_WEBUI_AGENT_CACHE_MAX` | `25` | Max live agent instances kept warm in the in-memory LRU. Each pins a full conversation transcript, so this is the dominant lever on resident memory — lower it on installs with many long sessions to cap RAM (at the cost of more cold reloads) | | `HERMES_WEBUI_SESSIONS_MAX` | `100` | Legacy operator override for the max compact `Session` objects held in the in-memory LRU. Prefer the `webui.sessions_cache_max` key in `config.yaml` (which takes precedence); this env var remains a fallback. Bounds resident memory so long-running installs cannot accumulate every session ever touched and eventually crash (#4765/#2233/#4633). Eviction only ever drops clean, persisted, non-active sessions; an evicted session lazily reloads from its JSON sidecar on next access | diff --git a/TESTING.md b/TESTING.md index af111974..f96a4f88 100644 --- a/TESTING.md +++ b/TESTING.md @@ -154,12 +154,35 @@ HISTORICAL_HYDRATION_TEST_BITE=break-tool-link \ python tests/browser_historical_transcript_hydration.py ``` -The dedicated `Conversation lifecycle (informational)` workflow runs the current -proof rows (`normal`, `terminal-error`, and `historical-transcript-hydration`) and -stays non-blocking while the public -matrix expands to additional behavior rows. The maintainer's private QA harness -remains broader; later public slices will add session switching, reconnect/replay, -cancellation, compression, and recovery. +The dedicated `Conversation lifecycle (informational)` workflow keeps the existing +proof rows (`normal`, `terminal-error`, and `historical-transcript-hydration`) +non-blocking while the public matrix expands. The Chromium +`reconnect-scene-redraw` row does **not** allow failures: it exercises the real +`loadSession` reconnect path and fails the workflow on a regression. Required +merge checks remain a maintainer-controlled repository setting. +The maintainer's private QA harness remains broader; later public slices will +add cancellation, compression, and recovery coverage. + +### Active-session reconnect redraw gate + +`tests/browser_reconnect_scene_redraw.py` is an opt-in deterministic browser +gate for reconnecting to a running session with a large Anchor activity scene. +It uses an isolated temporary server/home and fixture SSE events—no agent, +provider credentials, or production state. By default it checks Chromium and +WebKit, desktop and 390px viewports, and all three activity display modes: + +```bash +pip install playwright +python -m playwright install chromium webkit +python tests/browser_reconnect_scene_redraw.py +``` + +`TOOL_COUNT`, `BROWSERS`, and `MODES` narrow the matrix. `MEASURE_BASELINE=1` +records an unoptimized baseline without enforcing the redraw ceiling; +`METRICS_FILE` saves JSON metrics and `SCREENSHOT_DIR` saves generated-fixture +screenshots. This gate tests page reconstruction, journal-cursor resume, and +subsequent fixture SSE updates. It does not test a real provider/network or PWA +service-worker cache behavior. ### Streaming reader intent diff --git a/api/agent_sessions.py b/api/agent_sessions.py index 97086795..ed3ab538 100644 --- a/api/agent_sessions.py +++ b/api/agent_sessions.py @@ -1,8 +1,12 @@ """Shared helpers for reading Hermes Agent sessions from state.db.""" +import json import logging +import os import sqlite3 +import sys from contextlib import closing -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath +from urllib.parse import quote, quote_from_bytes logger = logging.getLogger(__name__) @@ -18,36 +22,71 @@ logger = logging.getLogger(__name__) _SOURCE_COLUMN_WARNED_DB_PATHS: set[str] = set() +def state_db_file_uri(db_path, platform: str | None = None) -> str: + """Build the ``file:`` URI (no query string) for an absolute ``state.db`` path. + + ``Path.as_uri()`` is not usable here: for a UNC share it emits + ``file://server/share/state.db`` and SQLite rejects any non-local URI + authority unless compiled with ``SQLITE_ALLOW_URI_AUTHORITY`` (the + CPython 3.11-3.13 Windows builds are not). SQLite instead accepts the + *empty*-authority spelling ``file:////server/share/state.db``, whose path + ``//server/share/...`` the Windows VFS opens as the UNC name. Drive-letter + paths keep the documented ``file:///C:/...`` form and POSIX paths are + unchanged. Only ``/`` survives unescaped, so ``?`` and ``#`` in path + components cannot leak into the query string. + + ``platform`` defaults to the running interpreter; tests pass it explicitly + so the Windows shapes are checked from any host. + """ + platform = platform or sys.platform + if platform == "win32": + win = PureWindowsPath(str(db_path)) + drive = win.drive + if drive.startswith("\\\\?\\"): + # Extended-length prefix: "\\?\C:" or "\\?\UNC\server\share". + drive = drive[4:] + if drive.upper().startswith("UNC\\"): + drive = "\\\\" + drive[4:] + parts = win.parts[1:] # drop the anchor, keep the path components + if drive.startswith("\\\\"): + host_share = drive[2:].replace("\\", "/") + posix_path = "//" + "/".join((host_share, *parts)) + else: + posix_path = "/" + "/".join((drive, *parts)) + # ``:`` stays literal so the drive letter keeps SQLite's documented + # ``file:///C:/...`` shape (``Path.as_uri()`` leaves it unescaped too). + return "file://" + quote(posix_path, safe="/:") + posix_path = PurePosixPath(str(db_path)).as_posix() + # Quote the filesystem BYTES, not the str: a POSIX path component that is + # not valid UTF-8 is carried in the str as ``surrogateescape`` code points, + # which ``quote(str)`` rejects with UnicodeEncodeError (the caller then + # treats the db as unreadable and every agent-backed session vanishes). + # ``Path.as_uri()`` — what master used — percent-encodes os.fsencode() + # bytes; this keeps that behavior. + return "file://" + quote_from_bytes(os.fsencode(posix_path), safe="/") + + +def state_db_readonly_uri(db_path, platform: str | None = None) -> str: + """Strict read-only (``mode=ro``) form of :func:`state_db_file_uri`.""" + return state_db_file_uri(db_path, platform=platform) + "?mode=ro" + + def open_state_db_readonly(db_path: Path, log: logging.Logger | None = None) -> sqlite3.Connection: """Open the live agent ``state.db`` read-only for a pure-read projection. Same rationale as the session-listing path (#5455): a write-capable handle on the multi-GB, WAL ``state.db`` while the agent streams into it adds needless checkpoint/lock surface. The read-only ``file:...?mode=ro`` URI - avoids that. Falls back to a writable connection (and warns) if the - read-only open fails, so callers never lose data on exotic filesystems. + avoids that. Read failures propagate; a reader never upgrades to a writer. The caller must ensure ``db_path`` exists — this raises ``FileNotFoundError`` - for a missing path rather than letting the writable fallback below create an - empty, writable ``state.db`` there (a ghost DB in the agent's HOME). The - fallback is only for an *existing* DB whose read-only open fails on an exotic - filesystem, so a real read never loses data. + for a missing path rather than creating a ghost database. Callers own the returned connection (wrap it in ``contextlib.closing``). """ - log = log or logger if not db_path.exists(): raise FileNotFoundError(f"agent state.db not found: {db_path}") - read_only_uri = f"{db_path.resolve().as_uri()}?mode=ro" - try: - return sqlite3.connect(read_only_uri, uri=True) - except sqlite3.Error as exc: - log.warning( - "agent state.db read-only open failed for %s; falling back to writable connection: %s", - db_path, - exc, - ) - return sqlite3.connect(str(db_path)) + return sqlite3.connect(state_db_readonly_uri(db_path.resolve()), uri=True) MESSAGING_SOURCES = { @@ -65,6 +104,15 @@ MESSAGING_SOURCES = { CLI_MIN_UNTITLED_MESSAGE_COUNT = 6 CLI_MIN_UNTITLED_USER_MESSAGE_COUNT = 2 +# Sub-second scheduling/write-order races during a compression/cli_close +# handoff can persist the continuation row with ``started_at`` a few +# milliseconds BEFORE the parent's ``ended_at`` lands (#6931). The tolerance +# below lets those rows still classify as the next segment; the fork / +# model_config branch-marker / cross-source / end_reason guards in +# ``_is_continuation_session`` already rule out unrelated rows, so a small +# window cannot collapse genuinely concurrently started children. +CONTINUATION_STARTED_AT_TOLERANCE_SECONDS = 2.0 + SOURCE_LABELS = { 'acp': 'ACP', 'api_server': 'API', @@ -318,6 +366,30 @@ def is_cli_session_row_visible(row: dict) -> bool: return _count_user_turns(row) >= CLI_MIN_UNTITLED_USER_MESSAGE_COUNT +def _branch_markers(row: dict | None) -> tuple[str | None, str | None]: + """Return ``(_branched_from, _delegate_from)`` from a row's ``model_config``. + + Hermes Agent stamps explicit branches and delegate/subagent runs in the + ``model_config`` JSON column (``_branched_from`` / ``_delegate_from``); + ``session_source='fork'`` alone only covers WebUI-created forks. Missing, + empty, or unparsable ``model_config`` degrades to ``(None, None)``. + """ + if not row: + return None, None + raw = row.get('model_config') + if isinstance(raw, dict): + config = raw + elif isinstance(raw, str) and raw.strip(): + try: + parsed = json.loads(raw) + except (TypeError, ValueError): + parsed = None + config = parsed if isinstance(parsed, dict) else {} + else: + config = {} + return config.get('_branched_from'), config.get('_delegate_from') + + def _is_continuation_session(parent: dict | None, child: dict | None) -> bool: """Return True when ``child`` is the next segment of the same conversation. @@ -325,17 +397,36 @@ def _is_continuation_session(parent: dict | None, child: dict | None) -> bool: by ``hermes -c`` also records a new child session; for sidebar projection it should continue the same visible conversation rather than becoming a separate child-session row. Plain parent/child links that started before the - parent's ended boundary remain child sessions. + parent's ended boundary remain child sessions. A small scheduling/write-order + overlap between the handoff boundary timestamps is tolerated (#6931). Do not collapse lineage across raw sources. A WebUI session that continues from a Telegram/CLI/etc. parent must remain visible as its own surface-owned conversation; otherwise the tip inherits the root's title/source metadata and can disappear under messaging/sidebar policies. + + Explicit branches/delegates never continue: a child whose ``model_config`` + ``_branched_from`` / ``_delegate_from`` points at this parent is a fork of + the conversation and must stay visible. Beyond the bounded tolerance window + there is no reliable continuation signal — titles are user-controlled and + non-unique, so they are never used to widen the window (#7021 re-gate). """ if not parent or not child: return False if str(child.get('session_source') or '').strip().lower() == 'fork': return False + # Real Agent branches/delegates are marked in model_config (not + # session_source, which only WebUI-created forks carry). The marker must + # reference THIS parent: compression continuations inherit the rotated + # agent's model_config verbatim, so presence alone (a delegate's + # continuation still carries the delegate's own ``_delegate_from``) would + # misclassify real continuations. + child_branched_from, child_delegate_from = _branch_markers(child) + parent_id = parent.get('id') + if parent_id and ( + child_branched_from == parent_id or child_delegate_from == parent_id + ): + return False parent_source = str(parent.get('source') or '').strip().lower() child_source = str(child.get('source') or '').strip().lower() if parent_source and child_source and parent_source != child_source: @@ -349,9 +440,16 @@ def _is_continuation_session(parent: dict | None, child: dict | None) -> bool: # continuations when no boundary timestamp is available. return True try: - return float(child.get('started_at') or 0) >= float(ended_at) + child_started = float(child.get('started_at') or 0) + parent_ended = float(ended_at) except (TypeError, ValueError): return False + if child_started >= parent_ended - CONTINUATION_STARTED_AT_TOLERANCE_SECONDS: + return True + # Beyond the bounded early-side window the child is a genuine concurrent + # session — an exact title match is not evidence (titles are user-controlled + # and non-unique, so it cannot extend the tolerance). + return False def _continuation_root_id(rows_by_id: dict[str, dict], session_id: str | None) -> str | None: @@ -549,21 +647,7 @@ def read_importable_agent_session_rows( return [] log = log or logger - # Open read-only for this projection/listing path: it is a pure read, and - # holding a write-capable handle on the live (multi-GB, WAL) state.db while - # the agent streams into it adds needless checkpoint/lock surface (#5455). - # The defensive index self-heal below still runs, but through a separate - # short-lived writable connection on the rare missing-index path only. - read_only_uri = f"{db_path.resolve().as_uri()}?mode=ro" - try: - conn = sqlite3.connect(read_only_uri, uri=True) - except sqlite3.Error as exc: - log.warning( - "agent session listing read-only open failed for %s; falling back to writable connection: %s", - db_path, - exc, - ) - conn = sqlite3.connect(str(db_path)) + conn = open_state_db_readonly(db_path, log=log) with closing(conn): conn.row_factory = sqlite3.Row cur = conn.cursor() @@ -588,6 +672,7 @@ def read_importable_agent_session_rows( parent_expr = _optional_col('parent_session_id', session_cols) session_source_expr = _optional_col('session_source', session_cols) + model_config_expr = _optional_col('model_config', session_cols) ended_expr = _optional_col('ended_at', session_cols) end_reason_expr = _optional_col('end_reason', session_cols) user_id_expr = _optional_col('user_id', session_cols) @@ -613,11 +698,7 @@ def read_importable_agent_session_rows( use_messages_join = messages_has_session_id count_col = 'id' if 'id' in message_cols else 'session_id' - # Defensive index prime (#3887). The normal candidate-ordering shape uses - # the agent's standard ``idx_messages_session ON messages(session_id, - # timestamp)`` index; without it, large cron-only scans degrade badly. - # Writable dbs self-heal by recreating the index. Read-only or locked dbs - # fall back to the pre-aggregated cron-only path below instead of failing. + # Index creation belongs to explicit drained maintenance, never a read. messages_index_present = False if messages_has_session_id and messages_has_timestamp: try: @@ -625,21 +706,7 @@ def read_importable_agent_session_rows( messages_index_present = any(str(row[1]) == "idx_messages_session" for row in cur.fetchall()) except sqlite3.Error: messages_index_present = False - if not messages_index_present: - # Self-heal via a separate writable connection so the common - # (index-present) path keeps its read-only handle. On a truly - # read-only/locked db this fails and we degrade to the - # pre-aggregated cron-only path below, exactly as before. - try: - with closing(sqlite3.connect(str(db_path))) as _heal: - _heal.execute( - "CREATE INDEX IF NOT EXISTS idx_messages_session " - "ON messages(session_id, timestamp)" - ) - _heal.commit() - messages_index_present = True - except sqlite3.Error: - pass # read-only db / locked / older schema — degrade gracefully + if use_messages_join: actual_count_expr = f"COUNT(m.{count_col})" @@ -665,7 +732,6 @@ def read_importable_agent_session_rows( where_clauses = ["s.source IS NOT NULL"] params: list[object] = [] - included = () if include_sources: included = tuple(str(source) for source in include_sources if source) if included: @@ -679,10 +745,16 @@ def read_importable_agent_session_rows( where_clauses.append(f"s.source NOT IN ({placeholders})") params.extend(excluded) + # Without ``idx_messages_session`` the correlated ``MAX(mx.timestamp)`` + # candidate ordering rescans ``messages`` once per session (seconds on + # a few thousand sessions). Every missing-index projection — default + # sidebar, gateway watcher, cron/webhook/kanban views — orders through + # one read-only pre-aggregation pass instead; the listing never creates + # the index itself (that is drained maintenance, see + # ``scripts/ensure_state_db_read_indexes.py``). use_preaggregated_candidate_order = ( use_messages_join and messages_has_timestamp - and included == ("cron",) and not messages_index_present ) if use_preaggregated_candidate_order: @@ -709,6 +781,7 @@ def read_importable_agent_session_rows( SELECT s.id, s.title, s.model, s.message_count, s.started_at, s.source, {session_source_expr}, + {model_config_expr}, {user_id_expr}, {chat_id_expr}, {chat_type_expr}, @@ -791,6 +864,11 @@ def read_importable_agent_session_rows( params, ) projected = _project_agent_session_rows([dict(row) for row in cur.fetchall()]) + # model_config is selected only so the continuation classifier can + # read the real _branched_from/_delegate_from markers; it is internal + # agent state and must not leak into the /api/sessions response. + for row in projected: + row.pop('model_config', None) projected = [_with_normalized_source(row) for row in projected] projected = [row for row in projected if is_cli_session_row_visible(row)] if limit is None: @@ -893,6 +971,7 @@ def read_session_lineage_report(db_path: Path, session_id: str | None, max_hops: source_expr = _optional_col('source', session_cols) session_source_expr = _optional_col('session_source', session_cols) + model_config_expr = _optional_col('model_config', session_cols) title_expr = _optional_col('title', session_cols) started_expr = _optional_col('started_at', session_cols, '0') ended_expr = _optional_col('ended_at', session_cols) @@ -907,6 +986,7 @@ def read_session_lineage_report(db_path: Path, session_id: str | None, max_hops: SELECT s.id, {source_expr}, {session_source_expr}, + {model_config_expr}, {title_expr}, {started_expr}, {parent_expr}, @@ -953,6 +1033,7 @@ def read_session_lineage_report(db_path: Path, session_id: str | None, max_hops: SELECT s.id, {source_expr}, {session_source_expr}, + {model_config_expr}, {title_expr}, {started_expr}, {parent_expr}, @@ -1030,6 +1111,7 @@ def read_session_lineage_metadata(db_path: Path, session_ids: list[str] | set[st return {} session_source_expr = _optional_col('session_source', session_cols) source_expr = _optional_col('source', session_cols) + model_config_expr = _optional_col('model_config', session_cols) message_count_expr = _optional_col('message_count', session_cols, '0') # Scoped fetch via PRIMARY KEY + idx_sessions_parent rather than a # full table scan. The sessions table grows unbounded over time @@ -1066,7 +1148,7 @@ def read_session_lineage_metadata(db_path: Path, session_ids: list[str] | set[st placeholders = ','.join('?' * len(chunk)) cur.execute( f""" - SELECT s.id, {source_expr}, {session_source_expr}, s.title, s.started_at, s.parent_session_id, s.ended_at, s.end_reason, {message_count_expr} + SELECT s.id, {source_expr}, {session_source_expr}, {model_config_expr}, s.title, s.started_at, s.parent_session_id, s.ended_at, s.end_reason, {message_count_expr} FROM sessions s WHERE s.id IN ({placeholders}) """, @@ -1095,7 +1177,7 @@ def read_session_lineage_metadata(db_path: Path, session_ids: list[str] | set[st placeholders = ','.join('?' * len(chunk)) cur.execute( f""" - SELECT s.id, {source_expr}, {session_source_expr}, s.title, s.started_at, s.parent_session_id, s.ended_at, s.end_reason, {message_count_expr} + SELECT s.id, {source_expr}, {session_source_expr}, {model_config_expr}, s.title, s.started_at, s.parent_session_id, s.ended_at, s.end_reason, {message_count_expr} FROM sessions s WHERE s.parent_session_id IN ({placeholders}) """, diff --git a/api/config.py b/api/config.py index 58afc932..5c38d375 100644 --- a/api/config.py +++ b/api/config.py @@ -68,6 +68,21 @@ def _env_int(name: str, default: int, *, minimum: int = 1) -> int: return default return value if value >= minimum else default + +def _env_int_clamped(name: str, default: int, *, minimum: int = 1, maximum: int) -> int: + """Like ``_env_int``, then clamp a valid override to ``maximum``.""" + value = _env_int(name, default, minimum=minimum) + if not str(os.getenv(name) or "").strip(): + return value + return min(value, maximum) + + +# Sidebar recency window. Resolved here, before profile init, so a profile +# .env cannot override a server-wide resource bound. Clamped at 200. +CLI_VISIBLE_SESSION_LIMIT = _env_int_clamped( + "HERMES_WEBUI_VISIBLE_SESSION_LIMIT", 20, maximum=200, +) + # ── TLS/HTTPS config (optional, env-overridable) ──────────────────────────── TLS_CERT = os.getenv("HERMES_WEBUI_TLS_CERT", "").strip() or None TLS_KEY = os.getenv("HERMES_WEBUI_TLS_KEY", "").strip() or None diff --git a/api/gateway_watcher.py b/api/gateway_watcher.py index 635969ec..dc93ecb9 100644 --- a/api/gateway_watcher.py +++ b/api/gateway_watcher.py @@ -328,6 +328,12 @@ class GatewayWatcher: protects projection fields (notably role-derived CLI visibility) that the agent's existing index cannot see. """ + with self._sub_lock: + has_subscribers = bool(self._subscribers) + if not has_subscribers: + # Force fresh projection on the next subscription, without touching DB. + self._last_full_projection_at = None + return False db_path = self._state_db_path # A watcher may start before the agent has created state.db. Publishing an # empty first snapshot would make an already-rendered sidebar disappear; diff --git a/api/models.py b/api/models.py index 84a91b9c..dd775c68 100644 --- a/api/models.py +++ b/api/models.py @@ -48,7 +48,11 @@ from api.agent_sessions import ( from api.process_event_utils import stamp_message_source logger = logging.getLogger(__name__) -CLI_VISIBLE_SESSION_LIMIT = 20 +# Size of the interactive sidebar recency window. Also bounds how many +# delegated subagent children can be rendered at once, since a child only +# nests when it wins a slot in this window. Resolved in api.config before +# profile init; override with HERMES_WEBUI_VISIBLE_SESSION_LIMIT (clamped 1–200). +CLI_VISIBLE_SESSION_LIMIT = _cfg.CLI_VISIBLE_SESSION_LIMIT # How many messageful cron sessions to surface in the project-chip layer. # Needs to exceed CLI_VISIBLE_SESSION_LIMIT so older cron runs stay # addressable even when many newer non-cron sessions dominate the default @@ -8581,9 +8585,26 @@ def get_state_db_session_messages( cur.execute("PRAGMA table_info(sessions)") session_cols = {str(row['name']) for row in cur.fetchall()} if {'parent_session_id', 'end_reason', 'started_at', 'source'}.issubset(session_cols): + # session_source/model_config carry the fork/delegate boundary + # identity consumed by _is_continuation_session (#6931/#7021). + # Select them when present so the branch-marker guard applies + # to the stitch path too; older schemas degrade to NULL. + identity_cols = [] + if 'session_source' in session_cols: + identity_cols.append('session_source') + else: + identity_cols.append('NULL AS session_source') + if 'model_config' in session_cols: + identity_cols.append('model_config') + else: + identity_cols.append('NULL AS model_config') + lineage_select = ( + "id, source, started_at, parent_session_id, ended_at, end_reason" + + ", " + ", ".join(identity_cols) + ) cur.execute( - """ - SELECT id, source, started_at, parent_session_id, ended_at, end_reason + f""" + SELECT {lineage_select} FROM sessions WHERE id = ? """, @@ -8601,8 +8622,8 @@ def get_state_db_session_messages( if not parent_id or parent_id in seen: break cur.execute( - """ - SELECT id, source, started_at, parent_session_id, ended_at, end_reason + f""" + SELECT {lineage_select} FROM sessions WHERE id = ? """, @@ -10382,16 +10403,22 @@ def _insert_state_message_chronologically(messages: list, msg: dict) -> bool: # preceding assistant's tool_calls, not just the first — a multi-tool # turn has several adjacent tool results, and inserting between any of # them splits the block (assistant, tool, , tool). - if ( - idx < len(messages) - and messages[idx].get("role") == "tool" - and idx > 0 - and messages[idx - 1].get("role") == "assistant" - and messages[idx - 1].get("tool_calls") - ): - while idx < len(messages) and messages[idx].get("role") == "tool": - idx += 1 - advanced = True + if idx < len(messages) and messages[idx].get("role") == "tool": + # Walk back over any contiguous tool rows already emitted for + # this block, so the owning assistant is found even when idx + # lands on the SECOND result of a multi-tool turn (where + # messages[idx - 1] is another tool row, not the assistant). + owner = idx - 1 + while owner >= 0 and messages[owner].get("role") == "tool": + owner -= 1 + if ( + owner >= 0 + and messages[owner].get("role") == "assistant" + and messages[owner].get("tool_calls") + ): + while idx < len(messages) and messages[idx].get("role") == "tool": + idx += 1 + advanced = True # (b) Skip past an equal-timestamp run whose left neighbour shares # this message's role — inserting there would re-order an # already-matched same-role turn (user, , assistant). @@ -10420,6 +10447,7 @@ def merge_session_messages_append_only( *, truncation_watermark=None, truncation_boundary=None, + incoming_provenance: Literal["unverified", "state_db"] = "unverified", ) -> list: """Merge sidecar/context and state.db messages without deleting local rows. @@ -10433,6 +10461,7 @@ def merge_session_messages_append_only( state_messages, truncation_watermark=truncation_watermark, truncation_boundary=truncation_boundary, + incoming_provenance=incoming_provenance, ) finally: _STRUCTURED_IDENTITY_MEMO.reset(token) @@ -10444,6 +10473,7 @@ def _merge_session_messages_append_only_impl( *, truncation_watermark=None, truncation_boundary=None, + incoming_provenance=None, ) -> list: """Merge sidecar/context and state.db messages without deleting local rows. @@ -11038,7 +11068,21 @@ def _merge_session_messages_append_only_impl( seen_dedup_keys.add(dedup_key) seen_content_keys.add(content_key) seen_visible_keys.add(visible_key) - merged_messages.append(msg) + # This terminal path is shared by state.db reconciliation and by ordered + # sidecar stitching (notably compression continuations). Only the caller + # that read state.db may authorize timestamp-based recovery placement; + # stable child-sidecar sequence remains authoritative even when an + # archived parent was restamped later. + if ( + incoming_provenance == "state_db" + and max_sidecar_timestamp is not None + and timestamp is not None + and timestamp < max_sidecar_timestamp + ): + if not _insert_state_message_chronologically(merged_messages, msg): + merged_messages.append(msg) + else: + merged_messages.append(msg) _remember_merged_message(msg, source="state") return merged_messages @@ -11158,6 +11202,7 @@ def reconciled_state_db_messages_for_session( state_messages, truncation_watermark=getattr(session, "truncation_watermark", None), truncation_boundary=getattr(session, "truncation_boundary", None), + incoming_provenance="state_db", ) return _state_db_session_messages_result( reconciled_messages, diff --git a/api/onboarding.py b/api/onboarding.py index f09651c4..44ecabfe 100644 --- a/api/onboarding.py +++ b/api/onboarding.py @@ -71,7 +71,9 @@ _SUPPORTED_PROVIDER_SETUPS = { "default_model": "anthropic/claude-sonnet-4.6", "requires_base_url": False, "models": [ - {"id": _to_openrouter_namespace(model["id"]), "label": model["label"]} + {"id": "z-ai/glm-4.5-air", "label": "GLM-4.5 Air"} + if model["id"] == "zai/glm-4.5-flash" + else {"id": _to_openrouter_namespace(model["id"]), "label": model["label"]} for model in _FALLBACK_MODELS ], "category": "easy_start", diff --git a/api/profiles.py b/api/profiles.py index 78f18735..a4b5d8f7 100644 --- a/api/profiles.py +++ b/api/profiles.py @@ -198,7 +198,11 @@ def _unwrap_profile_home_to_base(home: Path) -> Path: # are operator/deployment-level postures, not per-profile toggles. Letting a # profile .env set HERMES_WEBUI_ISOLATED_PROFILE=0 would let a contained user # escape isolation (#4589). -_PROTECTED_ENV_KEYS = frozenset({'HERMES_WEBUI_ISOLATED_PROFILE'}) +_PROTECTED_ENV_KEYS = frozenset({ + 'HERMES_WEBUI_ISOLATED_PROFILE', + # Server-wide sidebar window; a profile .env must not widen the per-request query. + 'HERMES_WEBUI_VISIBLE_SESSION_LIMIT', +}) def _isolated_profile_opt_in() -> bool: @@ -934,6 +938,7 @@ _BLOCKED_RUNTIME_ENV_KEYS = { # #4589: operator/deployment isolation posture — never overridable by a # profile's own env on any runtime/gateway-parity path. 'HERMES_WEBUI_ISOLATED_PROFILE', + 'HERMES_WEBUI_VISIBLE_SESSION_LIMIT', } @@ -1866,7 +1871,19 @@ def _compute_profile_skills_stats(profile_dir: Path) -> tuple[int, int]: except Exception: pass - from agent.skill_utils import iter_skill_index_files, parse_frontmatter, skill_matches_platform + try: + from agent.skill_utils import iter_skill_index_files, parse_frontmatter, skill_matches_platform + except ImportError as exc: + logger.debug("agent.skill_utils unavailable; reporting skill stats as unknown: %s", exc) + # agent source not mounted (two-container Docker, + # HERMES_WEBUI_CHAT_BACKEND=gateway): this must never 500 GET + # /api/profiles (#7305). Report the skill stats as unknown — a stable + # (0, 0) — instead of keeping a partial shadow of agent.skill_utils + # here: a local re-implementation cannot preserve the index walk's + # exclusions, frontmatter-name identity or platform filtering, so any + # count it produced would be inaccurate. The UI omits the skills line + # when the total is 0, so the profile picker stays fully usable. + return (0, 0) seen_names = set() enabled_count = 0 diff --git a/api/routes.py b/api/routes.py index daca7c80..5f5893ba 100644 --- a/api/routes.py +++ b/api/routes.py @@ -46,6 +46,7 @@ from api.agent_sessions import ( _looks_like_default_cli_title, is_cli_session_row, is_cli_session_row_visible, + open_state_db_readonly, read_session_lineage_report, ) from api.compression_anchor import visible_messages_for_anchor @@ -320,7 +321,7 @@ def _latest_cron_session_info_for_jobs( if not db_path or not Path(db_path).exists(): return {jid: {"session_id": "", "message_count": None} for jid in requested} try: - with closing(sqlite3.connect(str(db_path))) as conn: + with closing(open_state_db_readonly(db_path)) as conn: conn.row_factory = sqlite3.Row cur = conn.cursor() cur.execute("PRAGMA table_info(sessions)") @@ -2543,8 +2544,8 @@ def _build_session_list_cache_payload( ) if show_cli_sessions: diag_stage("cli_cap") - archived_scoped = _cap_recent_cli_sessions(archived_scoped, cli_cap=CLI_VISIBLE_SESSION_CAP) - visible_scoped = _cap_recent_cli_sessions(visible_scoped, cli_cap=CLI_VISIBLE_SESSION_CAP) + archived_scoped = _cap_recent_cli_sessions(archived_scoped) + visible_scoped = _cap_recent_cli_sessions(visible_scoped) if visible_only: archived_scoped = [ s for s in archived_scoped if _session_has_server_visible_messages(s) @@ -3484,16 +3485,31 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | emit_error=False, ): return None - summary = find_run_summary(stream_id) - if not summary: - return None - session_id = str(summary.get("session_id") or "") - if not session_id: - return None + # Locate the run journal WITHOUT parsing it first. find_run_summary reads + # and parses the whole file (tens of thousands of rows on a long live + # run), and read_run_events below then parsed it AGAIN — two full passes + # cost ~0.6s per rebuild. Derive the durable summary from the single + # parse below instead; its last_seq / last_event_id are rebuilt from the + # same rows the snapshot consumes. + located = find_run_file(stream_id) + if located: + session_id, _journal_path = located + else: + # No journal file on disk for this run id: fall back to the + # historical summary-based lookup seam (defensive — a real miss + # returns None exactly like before; this is also the seam that + # long-standing tests stub with a handler-side summary). + fallback_summary = find_run_summary(stream_id) + if not fallback_summary: + return None + session_id = str(fallback_summary.get("session_id") or "") + if not session_id: + return None journal = read_run_events(session_id, stream_id) events = [event for event in (journal.get("events") or []) if isinstance(event, dict)] if not events: return None + summary = _summary_from_events(session_id, stream_id, events) event_run_ids: set[str] = set() malformed_envelope_run_id = False for event in events: @@ -3513,6 +3529,18 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | assistant_text = "" reasoning_text = "" + # Reasoning deltas arrive as thousands of tiny append events. Growing the + # transcript with ``reasoning_text += chunk`` copies the full (multi- + # hundred-KB) string on every append inside these closures — quadratic + # (~2.3s of a 4.6s rebuild on a 9.7k-chunk run). Collect chunks and + # materialize lazily; readers (echo probes, strip, final message) join at + # most once per interim segment. + reasoning_parts: list[str] = [] + reasoning_dirty = False + # Incremental folded index over the reasoning transcript: the per-interim + # echo probe consults this instead of re-walking the raw text (see + # ``_CompactEchoIndex``). + reasoning_index = _CompactEchoIndex() messages: list[dict] = [] tool_calls: list[dict] = [] activity_burst_anchors: list[dict] = [] @@ -3521,6 +3549,13 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | last_ts = None reasoning_first_tool_count: int | None = None + def _materialize_reasoning_text() -> str: + nonlocal reasoning_text, reasoning_dirty + if reasoning_dirty: + reasoning_text = "".join(reasoning_parts) + reasoning_dirty = False + return reasoning_text + def mark_boundary() -> int: nonlocal current_activity_burst_id text_end = len(assistant_text) @@ -3585,22 +3620,42 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | tool_calls.append(call) def reasoning_echo_tail_matches(text: str) -> bool: - candidate = _compact_for_echo_compare(text) - if not candidate: - return False - return _compact_for_echo_compare(reasoning_text).endswith(candidate) + # Indexed tail match (no raw-text walk): the incremental index folds + # each reasoning chunk once as it is appended, so this probe costs + # O(len(text)) regardless of how much interior whitespace stretches + # the raw span. The previous raw backward walk re-walked that span + # once per interim event, which turned the replay quadratic on a + # whitespace-heavy transcript (#7569 review, ~55x slower than master + # on a production-shaped journal). + return reasoning_index.matches_tail(text) def strip_reasoning_echo_tail(text: str) -> bool: - nonlocal reasoning_text, reasoning_first_tool_count - next_reasoning, did_remove = _strip_compact_echo_suffix(reasoning_text, text) - if did_remove: - reasoning_text = next_reasoning - if not _compact_for_echo_compare(reasoning_text): - reasoning_first_tool_count = None - return did_remove + nonlocal reasoning_text, reasoning_dirty, reasoning_first_tool_count + cut = reasoning_index.cut_to(text) + if cut is None: + return False + next_reasoning = _materialize_reasoning_text()[:cut].rstrip() + reasoning_text = next_reasoning + reasoning_parts.clear() + reasoning_parts.append(next_reasoning) + reasoning_dirty = False + # Re-index the truncated transcript so later probes match against it + # instead of the pre-strip tail (the index is the match authority). + reasoning_index.reset() + reasoning_index.append(next_reasoning) + if not _compact_for_echo_compare(reasoning_text): + reasoning_first_tool_count = None + return True for event in events: event_name = str(event.get("event") or event.get("type") or "") + if event_name == "metering": + # Metering rows are the bulk of a long live journal (~10k rows, + # ~60% of its bytes) and project nothing onto the snapshot; skip + # their per-row work while preserving the last_ts watermark they + # carry. + last_ts = event.get("created_at", last_ts) + continue payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} last_ts = event.get("created_at", last_ts) if event_name == "token": @@ -3611,9 +3666,12 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | continue if event_name == "reasoning": text = str(payload.get("text") or "") - if text and reasoning_first_tool_count is None: - reasoning_first_tool_count = len(tool_calls) - reasoning_text += text + if text: + if reasoning_first_tool_count is None: + reasoning_first_tool_count = len(tool_calls) + reasoning_parts.append(text) + reasoning_index.append(text) + reasoning_dirty = True continue if event_name == "interim_assistant": visible = str(payload.get("text") or "").strip() @@ -3658,6 +3716,8 @@ def _run_journal_live_snapshot(stream_id: str | None, *, handler=None) -> dict | update_completed_tool(payload) fresh_segment = True + reasoning_text = _materialize_reasoning_text() + if assistant_text or reasoning_text: message = { "role": "assistant", @@ -9296,6 +9356,7 @@ def _limited_webui_messages_for_display_with_sidecar( state_db_messages, truncation_watermark=getattr(session, "truncation_watermark", None), truncation_boundary=getattr(session, "truncation_boundary", None), + incoming_provenance="state_db", ) if cache_key is not None: _state_key = cache_key[4] @@ -10344,11 +10405,16 @@ def _dedupe_cli_sidebar_sessions_for_api( return _include_project_hidden_background_sidebar_sessions(candidates, visible) -CLI_VISIBLE_SESSION_CAP = 20 +def _cli_visible_session_cap() -> int: + """Shared sidebar window, not a second hard-coded 20.""" + from api.config import CLI_VISIBLE_SESSION_LIMIT + return CLI_VISIBLE_SESSION_LIMIT -def _cap_recent_cli_sessions(sessions: list[dict], cli_cap: int = CLI_VISIBLE_SESSION_CAP) -> list[dict]: +def _cap_recent_cli_sessions(sessions: list[dict], cli_cap: int | None = None) -> list[dict]: """Keep only the most recent CLI-visible sessions after filtering.""" + if cli_cap is None: + cli_cap = _cli_visible_session_cap() if cli_cap <= 0: return sessions kept = [] @@ -10769,12 +10835,14 @@ from api.streaming import ( _materialize_pending_user_turn_before_error, generate_session_title_for_session, _compact_for_echo_compare, - _strip_compact_echo_suffix, + _CompactEchoIndex, ) from api.gateway_chat import _run_gateway_chat_streaming, webui_gateway_chat_enabled from api.run_journal import ( _parse_run_journal_event_id as _shared_parse_run_journal_event_id, + _summary_from_events, bound_run_journal_snapshot_args, + find_run_file, find_run_summary, journal_replay_visible, read_run_events, @@ -11969,7 +12037,7 @@ def _handle_insights(handler, parsed) -> bool: from api.models import _active_state_db_path db_path = _active_state_db_path() if db_path and db_path.exists(): - with closing(sqlite3.connect(str(db_path))) as conn: + with closing(open_state_db_readonly(db_path)) as conn: conn.row_factory = sqlite3.Row cur = conn.cursor() # cache_read_tokens may not exist on older agent state DBs; @@ -12604,7 +12672,7 @@ def _deep_health_checks(stream_check: dict | None = None) -> tuple[dict, bool]: "ms": round((time.time() - t0) * 1000, 1), } else: - with closing(sqlite3.connect(str(db_path))) as conn: + with closing(open_state_db_readonly(db_path)) as conn: conn.execute("PRAGMA schema_version").fetchone() checks["state_db"] = { "status": "ok", @@ -22962,6 +23030,54 @@ def _prepare_chat_start_session_for_stream( s.save() +def _cleanup_chat_start_launch_failure(session, stream_id: str) -> None: + """Release state registered before a worker thread successfully starts.""" + clear_session_writeback_owner_if_owned(session.session_id, stream_id) + unregister_stream_owner(stream_id) + with STREAMS_LOCK: + STREAMS.pop(stream_id, None) + STREAM_GOAL_RELATED.pop(stream_id, None) + # The session-field reset needs the same concurrency discipline as the + # registry half: hold the per-session lock and re-resolve the canonical + # session before clearing anything. Mutating the passed-in stale object + # could wipe a concurrent successor turn's pending fields, and saving it + # could resurrect a session deleted while the launch was failing. Same + # pattern as the #1533 race fix (routes.py:3077) and the anchor-scene + # write guard (routes.py:5140). + # + # This runs while the original launch failure is being handled, so it must + # never raise. Lock acquisition and session resolution can fail on their own + # (I/O, deserialization), and an escaping error here would mask the launch + # failure the caller is about to report while leaving the reset half done. + try: + with _get_session_agent_lock(session.session_id): + try: + canonical = get_session(session.session_id) + except KeyError: + return # session deleted while the thread launch was failing + if getattr(canonical, "active_stream_id", None) != stream_id: + return # a successor turn already owns the session + canonical.active_stream_id = None + canonical.pending_user_message = None + canonical.pending_attachments = [] + canonical.pending_started_at = None + canonical.pending_user_source = None + try: + canonical.save() + except Exception: + logger.debug( + "Failed to persist chat-start cleanup after worker launch failure for %s", + stream_id, + exc_info=True, + ) + except Exception: + logger.debug( + "Failed to reset session state after worker launch failure for %s", + stream_id, + exc_info=True, + ) + + def _is_hidden_empty_session(s) -> bool: return ( getattr(s, "title", "Untitled") == "Untitled" @@ -23545,6 +23661,7 @@ def _start_chat_stream_for_session( _clear_gateway_run_starting(stream_id) except Exception: logger.debug("Failed to record gateway run-start failure for stream %s", stream_id, exc_info=True) + _cleanup_chat_start_launch_failure(s, stream_id) raise response = { "stream_id": stream_id, diff --git a/api/run_journal.py b/api/run_journal.py index a088c13d..087a08ba 100644 --- a/api/run_journal.py +++ b/api/run_journal.py @@ -620,6 +620,27 @@ def find_run_summary(run_id: str, *, session_dir: Path | None = None) -> dict | return None +def find_run_file(run_id: str, *, session_dir: Path | None = None) -> tuple[str, Path] | None: + """Locate a run journal file by run id WITHOUT parsing its body. + + Hot callers that immediately read the full journal (the live-snapshot + rebuild) must not pay :func:`find_run_summary`'s full-file parse first: + on a long live run the file holds tens of thousands of rows and parsing + it twice per rebuild dominated the snapshot cost. Returns + ``(session_id, path)`` for the first match, or ``None`` when the run id + is invalid or no journal exists. + """ + try: + rid = _validate_id(run_id, "run_id") + except ValueError: + return None + root = Path(session_dir) if session_dir is not None else _default_session_dir() + journal_root = root / RUN_JOURNAL_DIR_NAME + for path in journal_root.glob(f"*/{rid}.jsonl"): + return path.parent.name, path + return None + + def read_session_run_events( session_id: str, *, diff --git a/api/streaming.py b/api/streaming.py index e924dc10..b61490fb 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -197,41 +197,156 @@ def _compact_for_echo_compare(value: str) -> str: return re.sub(r'\s+', '', str(value or '')) -def _strip_compact_echo_suffix(value: str, suffix: str, *, search_window: int = 4096) -> tuple[str, bool]: +class _CompactEchoIndex: + """Incremental whitespace-folded index over a growing text buffer. + + The journal-rebuild echo check runs once per interim event against the + whole accumulated reasoning transcript. Rescanning the raw buffer each + time walks every whitespace character between the tail and the first + non-whitespace character — on a whitespace-heavy transcript that span is + megabytes, and the per-interim walk turns the replay quadratic (the + ``_find_compact_echo_suffix_start`` shape measured ~55x slower than + master on a production-shaped journal in the #7569 review). + + This class keeps the folded view and the raw cut offsets *incrementally*: + + * ``append`` folds each new chunk once and records, per folded character, + its raw index — so the cost of a chunk is proportional to the chunk, + never to the buffer. + * ``matches_tail`` compares only the candidate against the end of the + folded view: O(len(candidate)), with no raw-text walk at all. + * ``cut_to`` converts a folded length back to a raw index by bisecting + the recorded offsets, so one call returns the same cut point the raw + backward walk would have produced. + + ``str.isspace`` is used for folding instead of the ``\\s`` pattern used + by :func:`_compact_for_echo_compare`. The two agree on every Unicode code + point, so the indexed view and the regex-folded view stay consistent. + """ + + __slots__ = ('_compact', '_offsets', '_raw_len') + + def __init__(self) -> None: + self._compact: list[str] = [] + self._offsets: list[int] = [] + self._raw_len = 0 + + def append(self, text: str) -> None: + """Fold ``text`` onto the end of the index (one pass, no rescan).""" + raw = str(text or '') + if not raw: + return + compact = self._compact + offsets = self._offsets + base = self._raw_len + for i, ch in enumerate(raw): + if not ch.isspace(): + compact.append(ch) + offsets.append(base + i) + self._raw_len = base + len(raw) + + def reset(self) -> None: + """Drop all indexed state (used after the raw text is truncated).""" + self._compact.clear() + self._offsets.clear() + self._raw_len = 0 + + @property + def compact_length(self) -> int: + return len(self._compact) + + def matches_tail(self, suffix: str) -> bool: + """True when the folded ``suffix`` equals the folded tail.""" + candidate = _compact_for_echo_compare(suffix) + if not candidate: + return False + n = len(candidate) + if n > len(self._compact): + return False + start = len(self._compact) - n + compact = self._compact + for i in range(n): + if compact[start + i] != candidate[i]: + return False + return True + + def cut_to(self, suffix: str) -> int | None: + """Raw index at which the echo of ``suffix`` starts, else ``None``. + + The returned index is the first raw character of the echo, matching + the leftmost-cut semantics of the raw backward walk. Trailing + whitespace before the echo is left to the caller's ``rstrip``. + """ + candidate = _compact_for_echo_compare(suffix) + if not candidate: + return None + n = len(candidate) + if n > len(self._compact): + return None + start = len(self._compact) - n + compact = self._compact + for i in range(n): + if compact[start + i] != candidate[i]: + return None + return self._offsets[start] + + def compact_view(self) -> str: + """The folded text (materialized on demand — never kept as a string).""" + return ''.join(self._compact) + + +def _find_compact_echo_suffix_start(value: str, suffix: str) -> int | None: + """Return the index where a whitespace-folded ``suffix`` starts at the + end of ``value``, or ``None`` when the tail does not echo it. + + The match walks ``value`` and ``suffix`` from the end, skipping + whitespace in ``value``; only the echo span itself is inspected, so the + cost is linear in the echo length and allocation-free. Unlike a fixed + fold window, the walk cannot miss a compact-equivalent suffix whose raw + span is stretched by interior whitespace. + + ``str.isspace`` is used for the walk instead of the ``\\s`` pattern used + by :func:`_compact_for_echo_compare`. The two agree on every Unicode code + point, so the folded view and the walk stay consistent. + """ + candidate = _compact_for_echo_compare(suffix) + if not candidate: + return None + i = len(value) - 1 + j = len(candidate) - 1 + while j >= 0: + while i >= 0 and value[i].isspace(): + i -= 1 + if i < 0 or value[i] != candidate[j]: + return None + i -= 1 + j -= 1 + return i + 1 + + +def _strip_compact_echo_suffix(value: str, suffix: str) -> tuple[str, bool]: """Remove ``suffix`` from ``value`` when they match after whitespace folding. - The search window is folded once and the cut point is then located by - walking backwards across the echo itself. The previous implementation - probed every candidate cut index and re-folded the whole remaining tail for - each probe, which is quadratic in the window size: a 6000-character final - message cost seconds of CPU, held under the GIL, stalling every other - stream in the process. + The cut point is located by the same backward walk as + :func:`_find_compact_echo_suffix_start`: no fixed search window is + involved, so a compact-equivalent suffix is removed no matter how much + interior whitespace stretches its raw span. The previous windowed + implementation folded a bounded tail on every call; its retired probing + variant re-folded the remaining tail per candidate cut index, which is + quadratic: a 6000-character final message cost seconds of CPU, held under + the GIL, stalling every other stream in the process. - ``str.isspace`` is used for the backwards walk instead of the ``\\s`` - pattern used by :func:`_compact_for_echo_compare`. The two agree on every - Unicode code point, so the folded view and the walk stay consistent. + Whitespace sitting between the kept text and the echo is removed by + ``rstrip``, which lands on the same result as the leftmost cut index the + probing loop used to return. """ raw = str(value or '') - candidate = _compact_for_echo_compare(suffix) - if not raw or not candidate: + if not raw: return raw, False - tail = raw[-max(len(str(suffix or '')) * 3, search_window):] - offset = len(raw) - len(tail) - compact_tail = _compact_for_echo_compare(tail) - if len(candidate) > len(compact_tail) or not compact_tail.endswith(candidate): + start = _find_compact_echo_suffix_start(raw, suffix) + if start is None: return raw, False - # Consume exactly as many non-whitespace characters as the folded suffix - # holds; ``idx`` then sits on the first character of the echo. Whitespace - # sitting between the kept text and the echo is removed by ``rstrip``, - # which is why this lands on the same result as the leftmost cut index the - # probing loop used to return. - remaining = len(candidate) - idx = len(tail) - while remaining and idx: - idx -= 1 - if not tail[idx].isspace(): - remaining -= 1 - return raw[: offset + idx].rstrip(), True + return raw[:start].rstrip(), True def _redacted_session_payload_with_full_messages(session, *, tool_calls=None) -> dict | None: @@ -4379,9 +4494,10 @@ def _title_language_mismatch(user_text: str, title: str) -> bool: short and frequently embed a borrowed Latin technical term (e.g. a CJK title containing the word "Python"), the title side uses a proportion threshold (>=35% of the title's alphabetic characters in a non-start - script, min 2 chars) rather than a strict majority -- so a CJK title with - one English word still trips, while an English title with a single - foreign place-name does not. + script, min 2 chars) rather than a strict majority. CJK titles with + borrowed Latin terms are allowed when the title also contains CJK + characters (#7693), but pure-Latin titles for CJK conversations are + still rejected. 2. The legacy German-start → English-title heuristic, preserved verbatim so the original behavior keeps working for same-script (latin) drift that the script check can't see. @@ -4391,13 +4507,26 @@ def _title_language_mismatch(user_text: str, title: str) -> bool: return False # (1) Cross-script mismatch — language-agnostic. + # CJK text routinely borrows Latin product/technical terms (e.g. "WeChat + # Pay", "Python", "ProRes RAW"), so when the user writes in CJK, Latin + # characters in the title are acceptable as long as the title also + # contains CJK — i.e. the title is genuinely mixed, not pure drift. + # A pure-Latin title for a CJK conversation is still rejected. + # Unrelated scripts (Cyrillic, Arabic, Greek …) are always flagged. user_script = _dominant_script(user_text) if user_script: title_counts = _script_counts(candidate) title_total = sum(title_counts.values()) if title_total >= 2: for script, n in title_counts.items(): - if script != user_script and n >= 2 and (n / title_total) >= 0.35: + if script == user_script: + continue + # When user writes in CJK, Latin in the title is a borrowed + # term as long as the title also contains CJK characters. + if user_script == 'cjk' and script == 'latin': + if title_counts.get('cjk', 0) >= 2: + continue + if n >= 2 and (n / title_total) >= 0.35: return True # (2) Legacy same-script German→English heuristic. @@ -10006,6 +10135,14 @@ def _run_agent_streaming( # (#3587) replaces the flat _reasoning_text string so each intermediate # assistant turn (before tool calls) keeps its own reasoning segment. _reasoning_segments: dict = {} + # Incremental folded indexes mirroring each reasoning buffer. The + # live echo strip consults them instead of re-walking the raw + # buffers: an unbounded raw scan of the growing transcript is + # quadratic on whitespace-heavy output (#7569 review). Each index + # is fed on every append and re-indexed after a strip. + _reasoning_segment_indexes: dict = {} + _stream_reasoning_index = _CompactEchoIndex() + _reasoning_buffer_index = _CompactEchoIndex() _current_reasoning_idx = 0 _tool_boundary_advanced = False _live_tool_calls = [] # tool progress fallback when final messages omit tool IDs @@ -10028,6 +10165,10 @@ def _run_agent_streaming( if _reasoning_buffer[0]: put('reasoning', {'text': _reasoning_buffer[0]}) _reasoning_buffer[0] = '' + # The folded index mirrors this buffer 1:1 — dropping the + # text without dropping the index would leave it describing + # text that is no longer there. + _reasoning_buffer_index.reset() def _emit_metering(): @@ -10067,30 +10208,37 @@ def _run_agent_streaming( nonlocal _reasoning_segments removed = False if stream_id in STREAM_REASONING_TEXT: - next_text, did_remove = _strip_compact_echo_suffix( - STREAM_REASONING_TEXT.get(stream_id, ''), - text, - ) - if did_remove: + cut = _stream_reasoning_index.cut_to(text) + if cut is not None: + next_text = STREAM_REASONING_TEXT.get(stream_id, '')[:cut].rstrip() STREAM_REASONING_TEXT[stream_id] = next_text + _stream_reasoning_index.reset() + _stream_reasoning_index.append(next_text) removed = True - next_buffer, did_remove_buffer = _strip_compact_echo_suffix(_reasoning_buffer[0], text) - if did_remove_buffer: + cut = _reasoning_buffer_index.cut_to(text) + if cut is not None: + next_buffer = _reasoning_buffer[0][:cut].rstrip() _reasoning_buffer[0] = next_buffer + _reasoning_buffer_index.reset() + _reasoning_buffer_index.append(next_buffer) removed = True for idx in (_current_reasoning_idx, _current_reasoning_idx - 1): if idx not in _reasoning_segments: continue - next_segment, did_remove_segment = _strip_compact_echo_suffix( - _reasoning_segments.get(idx, ''), - text, - ) - if not did_remove_segment: + segment_index = _reasoning_segment_indexes.get(idx) + if segment_index is None: continue + cut = segment_index.cut_to(text) + if cut is None: + continue + next_segment = _reasoning_segments.get(idx, '')[:cut].rstrip() if next_segment: _reasoning_segments[idx] = next_segment + segment_index.reset() + segment_index.append(next_segment) else: _reasoning_segments.pop(idx, None) + _reasoning_segment_indexes.pop(idx, None) removed = True break return removed @@ -10154,15 +10302,21 @@ def _run_agent_streaming( _reasoning_segments[_current_reasoning_idx] = ( _reasoning_segments.get(_current_reasoning_idx, '') + reasoning_delta ) + # Keep the folded index in step with the segment text. + _reasoning_segment_indexes.setdefault( + _current_reasoning_idx, _CompactEchoIndex() + ).append(reasoning_delta) # Mirror full concatenation to shared dict so cancel_stream() can persist # it (#1361 §A). Cancel only creates one partial message, so the flat # concatenation is correct there. # Lock-free GIL-atomic mirror — see the STREAMS_LOCK contract in on_token. if stream_id in STREAM_REASONING_TEXT: STREAM_REASONING_TEXT[stream_id] += reasoning_delta + _stream_reasoning_index.append(reasoning_delta) # Accumulate into a coalescing buffer so every delta reaches the # browser — reasoning deltas are incremental, not idempotent. _reasoning_buffer[0] += reasoning_delta + _reasoning_buffer_index.append(reasoning_delta) # Throttle reasoning SSE events to ~10 Hz to avoid overwhelming the # frontend renderer. Each event triggers _parseStreamState() which # scans the full accumulated text — 10k+ reasoning tokens/second @@ -10173,6 +10327,10 @@ def _run_agent_streaming( _reasoning_last_put[0] = now put('reasoning', {'text': _reasoning_buffer[0]}) _reasoning_buffer[0] = '' + # The folded index mirrors this buffer 1:1 — dropping the + # text without dropping the index would leave it describing + # text that is no longer there. + _reasoning_buffer_index.reset() # Track reasoning deltas in the meter so live TPS reflects all AI output. _metering_reasoning_deltas[0] += 1 meter().record_reasoning(stream_id, _metering_reasoning_deltas[0]) @@ -10281,10 +10439,14 @@ def _run_agent_streaming( _reasoning_segments[_current_reasoning_idx] = ( _reasoning_segments.get(_current_reasoning_idx, '') + reason_delta ) + _reasoning_segment_indexes.setdefault( + _current_reasoning_idx, _CompactEchoIndex() + ).append(reason_delta) # Mirror full concatenation to shared dict (#1361 §A) # Lock-free GIL-atomic mirror — see STREAMS_LOCK contract in on_token. if stream_id in STREAM_REASONING_TEXT: STREAM_REASONING_TEXT[stream_id] += reason_delta + _stream_reasoning_index.append(reason_delta) put('reasoning', {'text': reason_delta}) _metering_reasoning_deltas[0] += 1 meter().record_reasoning(stream_id, _metering_reasoning_deltas[0]) diff --git a/docs/rfcs/stable-assistant-turn-anchors.md b/docs/rfcs/stable-assistant-turn-anchors.md index 9c522519..39be9aa4 100644 --- a/docs/rfcs/stable-assistant-turn-anchors.md +++ b/docs/rfcs/stable-assistant-turn-anchors.md @@ -574,6 +574,38 @@ early renderers can avoid chasing `S.messages` during every paint. If settlement later rewrites the transcript message, the anchor must be refreshed from that message instead of allowing the two copies to drift silently. +### Settled ownership of a restored live turn + +A preserved or snapshotted live-turn node is not evidence of a running turn. The +settled assistant message is persisted a few milliseconds before the stream's +terminal event clears the active stream id, so a live node whose stream has +already ended can survive a transcript rebuild or a session switch and be +appended under the settled answer, putting the same answer on screen twice +(#6948 follow-up; upstream symptom report #2051). + +A live-turn node is therefore discarded only where the settled transcript is +proved to own it: the transcript ends with a settled assistant message, no +message still carries a live-projection marker, that settled message's +persisted stream identity equals the id of the stream that built the live node +— or, only where no scene identity was persisted, the markdown the stream +produced equals the persisted message source — and the node carries nothing the +settled rebuild could not have produced, meaning no unpersisted tool card, +reasoning row or transparent-stream row and no second live segment. + +Ownership is a state proof, never a comparison of two rendered DOM trees: the +live body comes from the streaming parser and the settled body from the +markdown renderer, and an answer that reads the same as the previous one must +not delete a genuinely live turn. An unrecognised node shape keeps the turn. +Only the two branches that ADD a turn are gated; a branch that replaces a node +cannot duplicate one. + +A settled activity scene projects one answer as well. A `process_prose` row +whose text duplicates the scene's final answer — the same text, or a prefix of +it within ten percent of its length — is dropped where both settled renderers +take their rows, so the answer is not rebuilt as a second assistant segment +above itself. Live rendering is unchanged: while the turn streams, the inline +live segment is hidden and the prose row is the visible answer. + ## Replay, Reload, And Reconstruction Replay/reload should reconstruct the same Assistant Turn Anchor from durable diff --git a/docs/rfcs/webui-run-state-consistency-contract.md b/docs/rfcs/webui-run-state-consistency-contract.md index 9b6d02ec..d1362548 100644 --- a/docs/rfcs/webui-run-state-consistency-contract.md +++ b/docs/rfcs/webui-run-state-consistency-contract.md @@ -131,6 +131,23 @@ and 5; it does not mark every run-state boundary implemented. Visible interim assistant progress must remain visible timeline content; a compact Activity disclosure may summarize adjacent tool/debug detail, but it must not be the only place where the user can see emitted progress text. + Interim assistant text that duplicates the tail of the accumulated reasoning + transcript is stripped from the reasoning copy so the restored snapshot + shows the content once. That echo match is whitespace-insensitive and + carries no fixed search window: a compact-equivalent suffix is recognized + however much interior whitespace stretches its raw span. Both consumers + (the live-stream echo path and the journal replay in `api/routes.py`) + match through an incremental folded index (`_CompactEchoIndex` in + `api/streaming.py`): the folded view and its raw cut offsets are built as + each chunk is appended, so a probe costs O(len(candidate)) and never + rescans the transcript's whitespace. The retired windowed variants could + drop the strip when the span exceeded the window, duplicating the interim + text; the retired raw backward walk was correct but re-walked the span per + interim event, quadratic on whitespace-heavy transcripts. Regressions: + `tests/test_live_snapshot_echo_dedup.py` pins the single-occurrence + result, `tests/test_live_snapshot_echo_scan_scaling.py` pins the scaling + property (a fixed-size fixture cannot catch a per-interim rescan), and + `tests/test_compact_echo_index.py` pins index/oracle equivalence. 6. **Compression is not current intent.** Automatic compression summaries and reference cards are recovery/handoff material. They must not be treated as a new user request, active-turn content, or the default visible explanation for diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 92481a53..bb59d431 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -231,6 +231,36 @@ For a foreground `python3 bootstrap.py`, stop it with Ctrl-C and start it again. --- +## Agent sessions list slowly (or the `state.db` read index is missing) + +**Symptom.** The sidebar's imported/CLI session list takes seconds per refresh on a large Hermes profile, or a log line says a `state.db` read failed. Sessions still appear; nothing is lost. + +**Why.** Every WebUI reader of the agent's `state.db` (session listing, transcript reads, lineage, gateway watcher, cron sidebar, insights, health) opens it strictly read-only (`file:...?mode=ro`). A reader never upgrades to a write-capable handle and never creates an index: on a multi-GiB `messages` table `CREATE INDEX` holds the SQLite writer lock for minutes and stalls the agent streaming into the same WAL database. When the agent's standard `idx_messages_session` index is missing (older agent, hand-rebuilt or re-imported DB), listings degrade to a bounded one-pass pre-aggregation — slower than the indexed seek, but read-only. A read-only open failure propagates to the caller's existing error boundary (the listing returns empty for that profile) instead of silently reopening the file writable. + +**Diagnostic.** + +```bash +sqlite3 "file:$HOME/.hermes/state.db?mode=ro" "PRAGMA index_list(messages)" +``` + +`idx_messages_session` should be listed. If it is not, the agent has not created it and WebUI will not create it for you. + +**Fix.** Create the covering read indexes in an explicit drained maintenance window: stop the agent (and any gateway/cron runner) writing to that `state.db`, then run: + +```bash +python3 scripts/ensure_state_db_read_indexes.py --db ~/.hermes/state.db --confirm-drained +``` + +- `--confirm-drained` is mandatory: it is your assertion that no agent turn is running against the database. The tool does not verify it. +- `--lock-file PATH` (optional) additionally holds an exclusive non-blocking lock on `PATH` (`flock` on POSIX, `msvcrt.locking` on Windows) for deployments that serialise agent turns on a lock file; a held lock makes the tool exit without touching the database. Without `--lock-file` no lock primitive is required, so the script runs on native Windows as well. On Windows the lock covers byte 0 of `PATH`; a new or empty lock file is initialised with one byte first (an existing lock file is never rewritten). +- The tool opens the database `mode=rw` (never `rwc`): a mistyped path raises instead of creating an empty database. Indexes are created inside one `BEGIN IMMEDIATE` transaction and rolled back on any error. +- It is idempotent and prints a JSON status per index (`created` / `existing` / `skipped`). `skipped` means this database's schema lacks a column that index keys on (an older agent); the indexes the schema does support are still created. An existing index with a different table, key shape or collation is reported as `Incompatible index` and never replaced; an index that is not covering (`EXPLAIN QUERY PLAN`) is reported as `Index is not covering`. +- Windows UNC profiles (`HERMES_HOME=\\server\share\...`) are supported: readers and this tool build the empty-authority URI `file:////server/share/state.db` that the bundled SQLite accepts. + +**When to file a bug.** File a WebUI bug if the listing stays slow after the tool reports `existing` for `idx_messages_session`, if the tool reports `Incompatible index` on an untouched agent-created database, or if a read-only open fails on a local path. Include the tool's JSON output, the `PRAGMA index_list(messages)` result, and the sanitized error text. + +--- + ## 404 after login when password auth is enabled **Symptom.** After enabling password authentication (`HERMES_WEBUI_PASSWORD`), logging in redirects to `/sessions` and the browser shows a `404 not found` error instead of the chat interface. diff --git a/scripts/ensure_state_db_read_indexes.py b/scripts/ensure_state_db_read_indexes.py new file mode 100644 index 00000000..0d3cbfe0 --- /dev/null +++ b/scripts/ensure_state_db_read_indexes.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Create the covering read indexes on the agent ``state.db`` in an explicit, +drained maintenance window. + +The WebUI read paths (session listing, lineage reads, gateway watcher, cron +sidebar, insights, health) never create indexes themselves: ``CREATE INDEX`` +on a multi-GiB ``messages`` table holds the SQLite writer lock for minutes, +which stalls the agent streaming into the same WAL database. Index +maintenance is therefore an operator action run while the agent is idle. + +Usage:: + + python scripts/ensure_state_db_read_indexes.py --db ~/.hermes/state.db \\ + --confirm-drained [--lock-file /path/to/agent-activity.lock] + +``--confirm-drained`` is mandatory. When ``--lock-file`` is given, the tool +takes an exclusive non-blocking lock on that path (``flock`` on POSIX, +``msvcrt.locking`` on Windows) so that a deployment which serialises agent +turns on a lock file cannot start a turn mid-rebuild. Without ``--lock-file`` +no lock primitive is needed, so the script runs on every supported platform. + +Existing indexes are verified for table, key shape and collation and must be +covering (``EXPLAIN QUERY PLAN``); an incompatible index is reported, never +silently replaced. +""" +import argparse +from contextlib import closing, nullcontext +import json +import os +from pathlib import Path +import sqlite3 +import sys + +try: # POSIX + import fcntl +except ImportError: # native Windows + fcntl = None +try: # Windows + import msvcrt +except ImportError: # POSIX + msvcrt = None + +_REPO_ROOT = Path(__file__).resolve().parents[1] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) +from api.agent_sessions import state_db_file_uri # same UNC-aware URI as the readers + +INDEXES = { + "idx_messages_session": ("messages", (("session_id", "BINARY"), ("timestamp", "BINARY"))), + "idx_messages_session_role": ("messages", (("session_id", "BINARY"), ("role", "NOCASE"))), + "idx_sessions_webui_fingerprint": ("sessions", tuple((c, "BINARY") for c in + ("source", "id", "message_count", "last_activity_at"))), +} + +# The only schema gap an older Agent is known to have: sessions.last_activity_at +# arrived later than the other keyed columns. An index whose missing columns are +# all listed here is reported "skipped"; any other missing column means this is +# not an Agent state.db (or a damaged one), and the tool fails closed. +_LEGACY_OPTIONAL_COLUMNS = { + "idx_sessions_webui_fingerprint": frozenset({"last_activity_at"}), +} + + +def _require_agent_schema(db) -> None: + """Fail closed unless this is an Agent ``state.db``. + + Matching column names alone would accept a look-alike chat database. Every + Agent ``state.db`` since the first schema has carried a ``schema_version`` + row and ``sessions.id`` as the primary key, so require both before any + ``CREATE INDEX``. + """ + has_marker = db.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_version'" + ).fetchone() + version = None + if has_marker: + cols = {r[1] for r in db.execute("PRAGMA table_info(schema_version)")} + if "version" in cols: + row = db.execute("SELECT version FROM schema_version LIMIT 1").fetchone() + version = row[0] if row else None + if not isinstance(version, int) or version < 1: + raise RuntimeError( + "state.db has no Agent schema_version marker; not an Agent state.db, refusing to continue" + ) + pk = [r[1] for r in db.execute("PRAGMA table_info(sessions)") if r[5]] + if pk != ["id"]: + raise RuntimeError( + "state.db sessions table is not keyed on id; not an Agent state.db, refusing to continue" + ) + + +def _exclusive_lock(lock_file): + if lock_file is None: + return nullcontext() + if fcntl is None and msvcrt is None: + raise RuntimeError("--lock-file needs fcntl (POSIX) or msvcrt (Windows); " + "neither is available on this interpreter") + handle = open(lock_file, "a+") + try: + if fcntl is not None: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + else: + # Lock one byte at offset 0 without blocking; raises OSError when held. + # ``a+`` may have just created an empty file, and Windows cannot lock + # a range beyond EOF, so make sure byte 0 exists first. An existing + # (deployment-owned) lock file is left untouched. Write the byte on + # the descriptor: a text-mode ``handle.write("\n")`` becomes two + # bytes (``\r\n``) on native Windows. + if os.fstat(handle.fileno()).st_size == 0: + os.write(handle.fileno(), b"\0") + handle.flush() + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + except BaseException: + handle.close() + raise + return closing(handle) + + +def ensure_read_indexes(db_path, *, confirmed_drained=False, lock_file=None): + if not confirmed_drained: + raise RuntimeError("requires --confirm-drained") + db_path = Path(db_path).resolve(strict=True) + statuses = {} + with _exclusive_lock(lock_file): + # mode=rw (not rwc): never create a ghost database at a mistyped path. + with closing(sqlite3.connect(state_db_file_uri(db_path) + "?mode=rw", uri=True)) as db: + db.execute("BEGIN IMMEDIATE") + try: + # Validate the whole Agent schema before creating anything, so a + # wrong database fails closed without a partial index set. + _require_agent_schema(db) + missing_by_index = {} + for name, (table, keys) in INDEXES.items(): + present = {r[1] for r in db.execute(f"PRAGMA table_info({table})")} + if not present: + raise RuntimeError(f"state.db has no {table!r} table; refusing to continue") + missing = {col for col, _ in keys if col not in present} + if missing - _LEGACY_OPTIONAL_COLUMNS.get(name, frozenset()): + raise RuntimeError( + f"state.db {table!r} table lacks Agent column(s) " + f"{sorted(missing)}; not an Agent state.db, refusing to continue" + ) + missing_by_index[name] = missing + for name, (table, keys) in INDEXES.items(): + existing = db.execute("SELECT tbl_name FROM sqlite_master WHERE name=?", (name,)).fetchone() + if missing_by_index[name]: + # Known legacy gap. A same-named index here cannot be the + # shape we expect (it keys on the absent column), so it is + # incompatible, not something to skip past. + if existing: + raise RuntimeError(f"Incompatible index: {name}") + statuses[name] = "skipped" + continue + if existing: + actual = tuple((r[2], r[4]) for r in db.execute(f"PRAGMA index_xinfo({name})") if r[5]) + flags = next((r for r in db.execute(f"PRAGMA index_list({table})") if r[1] == name), None) + if existing[0] != table or actual != keys or not flags or flags[2] or flags[4]: + raise RuntimeError(f"Incompatible index: {name}") + statuses[name] = "existing" + else: + columns = ", ".join(f"{col} COLLATE {collation}" for col, collation in keys) + db.execute(f"CREATE INDEX {name} ON {table}({columns})") + statuses[name] = "created" + columns = ", ".join(col for col, _ in keys) + plan = db.execute(f"EXPLAIN QUERY PLAN SELECT {columns} FROM {table} INDEXED BY {name}").fetchall() + if not any(f"COVERING INDEX {name}" in row[3] for row in plan): + raise RuntimeError(f"Index is not covering: {name}") + db.commit() + except BaseException: + db.rollback() + raise + return statuses + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--db", type=Path, required=True, help="path to the agent state.db") + parser.add_argument("--lock-file", type=Path, default=None, + help="optional lock file to hold exclusively (flock) while indexes are built") + parser.add_argument("--confirm-drained", action="store_true", + help="assert that no agent turn is running against this database") + args = parser.parse_args() + result = ensure_read_indexes(args.db, confirmed_drained=args.confirm_drained, + lock_file=args.lock_file) + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/static/sessions.js b/static/sessions.js index c6a302a4..dd6a6368 100644 --- a/static/sessions.js +++ b/static/sessions.js @@ -2169,6 +2169,12 @@ async function loadSession(sid){ S.activeStreamId=activeStreamId; const liveToolReplayId=(tc)=>String(tc&&(tc.tid||tc.id||tc.tool_call_id||tc.tool_use_id||tc.call_id||'')||'').trim(); const replayPersistedLiveToolCards=(opts)=>{ + // The journal-backed Anchor scene is authoritative through its resume + // cursor; newer rows arrive through the reattached SSE stream. Replaying + // the older INFLIGHT tool cache after a successful scene restore would + // redraw all N rows N times. Keep the #3707 replay only for legacy HTML + // restoration or a failed/unavailable scene render. + if(restoredAnchorScene) return; const liveToolCalls=Array.isArray(S.toolCalls) ? S.toolCalls : (Array.isArray(INFLIGHT[sid]&&INFLIGHT[sid].toolCalls)?INFLIGHT[sid].toolCalls:[]); diff --git a/static/ui.js b/static/ui.js index 675bd66d..610ad14c 100644 --- a/static/ui.js +++ b/static/ui.js @@ -3949,6 +3949,27 @@ function _isEquivalentConfiguredModelEntry(modelId,badge,entries){ // providers (@custom:name:model) without collapsing matching model IDs from // different providers. const rawId=String(modelId||''); + // `/` is another routing spelling of ``, so its badge + // key must not become a second picker row. configured_model_badges holds every + // spelling of a configured model and renderModelDropdown() synthesises a row + // for each key this predicate does not recognise. The provider-qualified + // spelling was missed because _normalizeConfiguredModelKey() strips only one + // leading slash segment (#3360 keeps `vendor_a/x` and `vendor_b/y/x` distinct), + // so `acme/example-model` and `custom/acme/example-model` normalise to + // different keys and the picker lists one model twice. + // Match it the way the `@provider:` rule below does: the badge declares a + // provider, the key starts with that provider's `/` prefix, and an + // existing row from the same provider normalises equal to the remainder. Two + // different models never satisfy the last clause, so this can only drop a + // duplicate of a row the catalog already produced. + const slashPrefix=provider?`${provider}/`:''; + if(slashPrefix&&rawId.toLowerCase().startsWith(slashPrefix)){ + const slashRoutedId=rawId.slice(slashPrefix.length); + if(slashRoutedId&&(entries||[]).some(entry=> + String(entry.providerId||'').toLowerCase()===provider + &&_normalizeConfiguredModelKey(entry.value)===_normalizeConfiguredModelKey(slashRoutedId) + )) return true; + } const prefix=provider?`@${provider}:`:''; if(!prefix||!rawId.toLowerCase().startsWith(prefix)) return false; const routedId=rawId.slice(prefix.length); @@ -9928,6 +9949,99 @@ function _liveAssistantSegmentTextLength(seg){ return String(body.textContent||'').trim().length; } +// ── #6948 follow-up: settled-transcript ownership of a live-turn node ────── +// (duplicate assistant answer; upstream symptom report #2051) +// +// Both live-turn insertion paths — the renderMessages() re-attach and +// restoreLiveTurnHtmlForSession() — have one branch that ADDS a turn the +// settled rebuild did not produce. The assistant row is persisted a few ms +// BEFORE the stream's terminal event clears S.activeStreamId, so a dead live +// node left behind by an INFLIGHT entry that outlived its stream is re-attached +// on top of the settled transcript: the same answer twice, self-sustaining +// across later renders (the node keeps id=liveAssistantTurn and is re-preserved +// by the #6948 guard), healed only by a reload. +// +// Dropping a live node must be an OWNERSHIP proof, never "the rendered text +// looks the same": +// * the live body is built by the streaming smd parser and the settled body +// by renderMd + post-processing, so their textContent differs for anything +// past single-paragraph plain text (lists, fenced code with its language +// label, tables, `_underscore_` emphasis, the injected copy button) — a +// text comparison would miss most real answers; and +// * an identical answer earlier in the history (a repeated "Done.", a +// greeting, a same-prompt resend) would delete a GENUINELY LIVE turn: while +// the current turn is unpersisted, the last settled assistant message IS the +// previous turn's answer. +// So ownership is proved from state: the transcript must END with a settled +// assistant message that THIS stream produced, and the node must carry nothing +// the settled rebuild could not have produced. + +// Stream identity persisted on a settled assistant message. The server stamps +// _anchor_stream_id from the anchor-scene sidecar record (api/routes.py); the +// client stamps _anchor_stream_id and scene.identity.stream_id at stream end +// (_attachProjectedAnchorSceneToLastAssistant, static/messages.js). +function _settledAssistantStreamId(message){ + if(!message) return ''; + const scene=message._anchor_activity_scene||null; + const identity=(scene&&scene.identity)||null; + return String(message._anchor_stream_id||(scene&&scene.stream_id)||(identity&&identity.stream_id)||''); +} +// The live-projection markers the #6948 preserve guard already treats as proof +// of a live owner. A message the client still considers live is never evidence +// that the turn has settled. +function _messageHasLiveAssistantProjection(m){ + return !!(m&&m.role==='assistant'&&(m._live||m._activityBurstId!==undefined||m._liveSegmentSeq!==undefined)); +} +// True when the live turn holds something the settled rebuild cannot have +// produced from S.messages — an unpersisted tool card, reasoning/thinking row, +// transparent-stream row, or a second live segment. Such a node is never +// dropped: #3714's whole premise is that the live DOM can be AHEAD of +// S.messages, and a tail-segment match must not discard the tool card or the +// earlier segment above it. An unknown node shape fails closed (keep the turn). +function _liveTurnCarriesUnsettledContent(turn){ + if(!turn||typeof turn.querySelectorAll!=='function') return true; + if(turn.querySelectorAll( + '.tool-card-row,.wl-reason,.agent-activity-thinking,.thinking-card-row,.transparent-event-row' + ).length) return true; + return turn.querySelectorAll('[data-live-assistant="1"]').length>1; +} +function _settledTranscriptOwnsLiveTurn(sid, turn){ + if(!turn) return false; + const msgs=(typeof S!=='undefined'&&S&&Array.isArray(S.messages))?S.messages:null; + if(!msgs||!msgs.length) return false; + // The transcript must END with a settled assistant answer. A trailing user + // turn means the live turn IS the current turn — nothing of it is persisted + // yet, so it can never be a leftover (this is the case a text comparison gets + // wrong when the previous answer happens to read the same), and a live + // projection anywhere means the rebuild still owns a live turn of its own. + const last=msgs[msgs.length-1]; + if(!last||last.role!=='assistant') return false; + if(msgs.some(_messageHasLiveAssistantProjection)) return false; + const inflight=(typeof INFLIGHT!=='undefined'&&INFLIGHT)?INFLIGHT[sid]:null; + // Owner of the live DOM: INFLIGHT carries the id of the stream that built it + // (attachLiveStream), and S.activeStreamId is that same id until the terminal + // event clears it. + const liveStreamId=String((inflight&&inflight.streamId)||(typeof S!=='undefined'&&S&&S.activeStreamId)||''); + const settledStreamId=_settledAssistantStreamId(last); + if(settledStreamId){ + // Identity recorded on both sides decides — renderer- and text-independent, + // so code blocks, lists and tables are handled like plain prose. + if(!liveStreamId||settledStreamId!==liveStreamId) return false; + }else{ + // No persisted identity on the tail (a turn whose scene was not worklog + // worthy). Compare SOURCE with SOURCE — the markdown this stream produced + // (INFLIGHT.lastAssistantText) against the persisted message content — + // never two independently rendered DOM trees. Regeneration cannot reach + // this branch: startRegeneration() truncates S.messages at the user turn, + // so the tail is a user message while a regenerated answer streams. + const streamed=String((inflight&&inflight.lastAssistantText)||'').replace(/\s+/g,' ').trim(); + if(!streamed) return false; + const settled=String((typeof msgContent==='function'?msgContent(last):last.content)||'').replace(/\s+/g,' ').trim(); + if(!settled||streamed!==settled) return false; + } + return !_liveTurnCarriesUnsettledContent(turn); +} + function _mergeRestoredLiveAssistantSegment(restored, existing){ if(!restored||!existing) return; const existingLive=existing.querySelector('[data-live-assistant="1"]'); @@ -9964,6 +10078,18 @@ function restoreLiveTurnHtmlForSession(sid){ const existing=$('liveAssistantTurn'); _mergeRestoredLiveAssistantSegment(restored, existing); if(existing) existing.replaceWith(restored); + // #6948 follow-up (duplicate assistant answer; #2051): only this branch ADDS a + // turn — replacing an existing live turn cannot duplicate. When the settled + // transcript already owns the stream this snapshot belongs to, appending it + // pins a SECOND copy of the same answer under the settled one. Release the + // stale snapshot (no other consumer can use it — it would be re-refused on + // every later restore) and report "nothing restored", so loadSession takes the + // same path it already takes for an INFLIGHT entry that never carried one. + else if(typeof _settledTranscriptOwnsLiveTurn==='function' + &&_settledTranscriptOwnsLiveTurn(sid, restored)){ + inflight.liveTurnHtml=null; + return false; + } else inner.appendChild(restored); // Transparent Stream: liveTurnHtml is restored via template.innerHTML, which // drops the property-bound onclick/onkeydown handlers wired by @@ -11469,6 +11595,15 @@ function _assistantMessageBelongsInWorklog(m, rawIdx, toolCallAssistantIdxs, vis const isTurnFinalAssistant=!!(opts&&opts.isTurnFinalAssistant); const visibleText=String(visibleContent!==undefined?visibleContent:msgContent(m)||'').trim(); const hasVisibleText=!!visibleText&&!_isAssistantEmptyPlaceholderContent(m, visibleText); + // The caller only consults this predicate once the turn has settled (it gates + // on `!S.busy`), so an `_live` marker seen here is a leftover of the + // live-snapshot projection, not an ongoing stream. Folding on it hid the turn's + // final answer inline and echoed it into the Worklog, leaving the turn with no + // visible content until the #3875 blank-turn fail-safe revealed both copies. A + // turn-final assistant message with visible text is the answer, so keep it out + // of the fold; every other `_live` message folds as before. Kept as a separate + // guard so the live rule below keeps its position. + if(m._live&&hasVisibleText&&isTurnFinalAssistant) return false; if(m._live) return true; if(hasVisibleText&&m._anchor_activity_scene) return false; if(hasVisibleText&&isTurnFinalAssistant) return false; @@ -13151,6 +13286,17 @@ function _anchorSceneRowsForRendering(scene, opts){ const rows=Array.isArray(scene&&scene.activity_rows)?scene.activity_rows:[]; const settled=!!(opts&&opts.settled); const live=!settled; + // #6948 follow-up (duplicate assistant answer; #2051): on a SETTLED turn the + // assistant segment owns the final answer, so a process_prose row carrying + // that same text must not also be rebuilt into the activity scene. The + // transparent-stream renderer suppresses it per row + // (_anchorSceneTransparentNodeForRow), but the compact-worklog row builder + // (_anchorSceneNodeForRow) has no such guard and renders the row as a second + // .assistant-segment above the settled one. Drop it here — the single place + // BOTH settled renderers (and the #5839 deferred-rows path) get their rows. + // LIVE rendering is untouched: while streaming, the inline live segment is + // hidden and the prose row IS the visible answer. + const settledFinalAnswer=settled?String((scene&&scene.final_answer)||'').trim():''; const out=[]; const byKey=new Map(); const liveProseTextKeys=new Map(); @@ -13173,6 +13319,9 @@ function _anchorSceneRowsForRendering(scene, opts){ if(_anchorSceneIsSettledSuccessfulCompression(row,settled)) continue; const text=String(row.text||'').trim(); if((row.role==='prose'||row.role==='thinking')&&!text) continue; + if(settledFinalAnswer&&row.role==='prose' + &&typeof _anchorSceneProseDuplicatesFinalAnswer==='function' + &&_anchorSceneProseDuplicatesFinalAnswer(text,settledFinalAnswer)) continue; // #6948 follow-up const key=keyFor(row); if(byKey.has(key)){ const index=byKey.get(key); @@ -13447,6 +13596,26 @@ function _anchorSceneProseMatchesFinalAnswer(proseText, finalAnswer){ const shorter=Math.min(a.length,b.length), longer=Math.max(a.length,b.length); return shorter>=80 && (shorter/longer)>=0.9; } +// #6948 follow-up (duplicate assistant answer; #2051): the same near-equality +// test as _anchorSceneProseMatchesFinalAnswer, minus its absolute `shorter>=80` +// floor. That floor makes the matcher a no-op for every SHORT final answer +// (greetings, "Done.", one-liners), so a prose row that is the answer minus its +// last streamed token — e.g. "Hey! What can I help you with today" vs +// "...today?" — was rebuilt as a SECOND assistant-segment beside the settled +// one. The >=0.9 length ratio is what actually keeps a distinct short +// intermediate sentence from being swallowed by a long final answer (Codex +// #4568), so it alone is kept. Deliberately a separate helper: the per-row +// render-time matcher above, and the tests pinning its exact predicate, stay +// untouched. +function _anchorSceneProseDuplicatesFinalAnswer(proseText, finalAnswer){ + if(_anchorSceneProseMatchesFinalAnswer(proseText,finalAnswer)) return true; + const norm=(s)=>String(s||'').replace(/\s+/g,' ').trim(); + const a=norm(proseText), b=norm(finalAnswer); + if(!a||!b) return false; + if(!(a.startsWith(b)||b.startsWith(a))) return false; + const shorter=Math.min(a.length,b.length), longer=Math.max(a.length,b.length); + return (shorter/longer)>=0.9; +} function _anchorSceneWorklogGroup(blocks, opts){ if(!blocks) return null; const live=!!(opts&&opts.live); @@ -16922,6 +17091,23 @@ function _processWakeupCardHtml(info, rawText, extras){ return `
${li('chevron-right',12)}${li('terminal',13)}${esc(t('process_wakeup_label'))}${cmdHtml}${chip}${extras.timeHtml||''}
${extras.filesHtml||''}${patternRow}${cmdRow}
${outHtml}
${extras.footHtml||''}
`; } +// #2051: parse into a