From c7a389e0a731c722184e888428d1edb8ba6a3dcf Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Sat, 6 Jun 2026 18:16:07 -0700 Subject: [PATCH] =?UTF-8?q?Release=20v0.51.305=20=E2=80=94=20Release=20JU?= =?UTF-8?q?=20(stage-p2b=20=E2=80=94=20dormant=20unified-SessionDB=20adapt?= =?UTF-8?q?er)=20(#3759)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(sessions): add dormant JSON-backed SessionDB adapter (#3720, #3383) First, lowest-risk slice of the unified-session-db migration: a SessionDB-shaped adapter over the existing WebUI JSON store, behind a dormant experimental flag (experimental.unified_session_db, default false). No runtime call site is rewired — is_unified_session_db_enabled() has no live callers, so persistence behavior is unchanged until a later migration PR opts in. Includes the adapter, the dormant config flag + _apply_config_defaults wiring, an architecture doc, and adapter tests. Also adds docs/architecture/ to the .gitignore docs allowlist (the docs/* rule excludes subdirectories; the new architecture doc and its presence test would otherwise be silently dropped on a clean checkout). Co-authored-by: rodboev <[email protected]> * docs(changelog): stamp v0.51.305 — Release JU (stage-p2b #3720) --------- Co-authored-by: nesquena-hermes <[email protected]> --- .gitignore | 2 + CHANGELOG.md | 5 + api/config.py | 30 +++ api/webui_session_db.py | 250 ++++++++++++++++++++++++ docs/architecture/unified-session-db.md | 98 ++++++++++ tests/test_webui_session_db_adapter.py | 162 +++++++++++++++ 6 files changed, 547 insertions(+) create mode 100644 api/webui_session_db.py create mode 100644 docs/architecture/unified-session-db.md create mode 100644 tests/test_webui_session_db_adapter.py diff --git a/.gitignore b/.gitignore index 0d26b7ad..ab6ed33c 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,8 @@ docs/* !docs/ui-ux/** !docs/rfcs/ !docs/rfcs/** +!docs/architecture/ +!docs/architecture/** # Local-only AI assistant context — never committed even under docs/. docs/AGENTS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dfce27d..a6cffdd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ## [Unreleased] +## [v0.51.305] — 2026-06-06 — Release JU (stage-p2b — dormant unified-SessionDB adapter groundwork) + +### Changed +- **Added the first, dormant slice of the unified-SessionDB migration (no runtime behavior change).** A new `SessionDB`-shaped adapter over the existing WebUI JSON session store lands behind an experimental flag (`experimental.unified_session_db`, default `false`). No runtime session call site is rewired — `is_unified_session_db_enabled()` has no live callers — so WebUI persistence is byte-for-byte unchanged until a later migration PR deliberately opts in. This is intentionally the lowest-risk first step toward expressing WebUI session access behind a stable adapter contract before any CLI/WebUI storage unification. Ships the adapter, the dormant config flag, an architecture doc, and adapter tests. (#3720 advances #3383, @rodboev) + ## [v0.51.304] — 2026-06-06 — Release JT (stage-p2a — un-held terminal reaper fix + opt-in Docker GPU image) ### Fixed diff --git a/api/config.py b/api/config.py index 89f4b4bb..a2e6d0ec 100644 --- a/api/config.py +++ b/api/config.py @@ -320,6 +320,22 @@ def _get_config_path() -> Path: _WEBUI_SESSION_SAVE_MODES = {"deferred", "eager"} _DEFAULT_WEBUI_SESSION_SAVE_MODE = "deferred" +_DEFAULT_EXPERIMENTAL_CONFIG = { + # Dormant first slice for the unified SessionDB migration. Runtime WebUI + # session call sites must continue using the existing JSON paths unless a + # later PR deliberately enables and wires this flag. + "unified_session_db": False, +} + + +def _apply_config_defaults(config_data: dict) -> None: + """Populate documented default-only config keys in-place.""" + experimental = config_data.get("experimental") + if not isinstance(experimental, dict): + experimental = {} + config_data["experimental"] = experimental + for key, value in _DEFAULT_EXPERIMENTAL_CONFIG.items(): + experimental.setdefault(key, value) def get_config() -> dict: @@ -367,6 +383,19 @@ def get_webui_session_save_mode(config_data: dict | None = None) -> str: return _DEFAULT_WEBUI_SESSION_SAVE_MODE +def is_unified_session_db_enabled(config_data: dict | None = None) -> bool: + """Return the dormant unified-session-db feature flag. + + The default is intentionally false so adding the JSON adapter cannot change + runtime persistence until a later migration PR switches call sites. + """ + active_cfg = config_data if isinstance(config_data, dict) else cfg + experimental = active_cfg.get("experimental", {}) if isinstance(active_cfg, dict) else {} + if not isinstance(experimental, dict): + return False + return experimental.get("unified_session_db") is True + + def reload_config() -> None: """Reload config.yaml from the active profile's directory.""" global _cfg_mtime, _cfg_path, _cfg_fingerprint @@ -391,6 +420,7 @@ def reload_config() -> None: _cfg_mtime = 0.0 except Exception: logger.debug("Failed to load yaml config from %s", config_path) + _apply_config_defaults(_cfg_cache) _cfg_fingerprint = _fingerprint_config(_cfg_cache) # Bust the models cache so the next request sees fresh config values. # Only delete the disk cache when config has actually changed -- not on diff --git a/api/webui_session_db.py b/api/webui_session_db.py new file mode 100644 index 00000000..1d3a1190 --- /dev/null +++ b/api/webui_session_db.py @@ -0,0 +1,250 @@ +"""Dormant JSON-backed SessionDB-shaped adapter for WebUI sessions. + +This module intentionally does not replace existing WebUI runtime call sites. +It provides a small compatibility surface over the current JSON sidecars so the +unified SessionDB contract can be tested without changing persistence behavior. +""" + +from __future__ import annotations + +import copy +import json +import os +import threading +from pathlib import Path +from typing import Any + +import api.models as models + + +_METADATA_FIELDS = frozenset( + { + "title", + "workspace", + "model", + "model_provider", + "created_at", + "updated_at", + "pinned", + "archived", + "project_id", + "profile", + "input_tokens", + "output_tokens", + "estimated_cost", + "cache_read_tokens", + "cache_write_tokens", + "personality", + "active_stream_id", + "pending_user_message", + "pending_attachments", + "pending_started_at", + "compression_anchor_visible_idx", + "compression_anchor_message_key", + "compression_anchor_summary", + "pre_compression_snapshot", + "context_engine", + "compression_anchor_engine", + "compression_anchor_mode", + "compression_anchor_details", + "context_engine_state", + "context_length", + "threshold_tokens", + "last_prompt_tokens", + "truncation_watermark", + "gateway_routing", + "gateway_routing_history", + "llm_title_generated", + "manual_title", + "parent_session_id", + "worktree_path", + "worktree_branch", + "worktree_repo_root", + "worktree_created_at", + "is_cli_session", + "source_tag", + "raw_source", + "session_source", + "source_label", + "read_only", + "enabled_toolsets", + "composer_draft", + } +) + +_UNSAFE_FIELDS = frozenset({"session_id", "messages", "tool_calls", "message_count"}) + + +class WebUIJsonSessionDB: + """Small SessionDB-like facade over existing WebUI session JSON files.""" + + def __init__(self, session_dir: Path | str | None = None): + self._session_dir = Path(session_dir).expanduser().resolve() if session_dir else None + + @property + def session_dir(self) -> Path: + return self._session_dir or models.SESSION_DIR + + def list_sessions(self) -> list[dict[str, Any]]: + """Return compact metadata for persisted WebUI JSON sessions. + + Reads are direct JSON loads and never call ``Session.load()``, because + that path may self-heal and write repaired transcripts. + """ + rows: list[dict[str, Any]] = [] + if not self.session_dir.exists(): + return rows + for path in self.session_dir.glob("*.json"): + if path.name.startswith("_"): + continue + data = self._read_path(path) + if not isinstance(data, dict): + continue + sid = str(data.get("session_id") or path.stem) + if not models.is_safe_session_id(sid): + continue + rows.append(self._metadata_row(sid, data)) + rows.sort(key=lambda row: (bool(row.get("pinned")), self._sort_timestamp(row)), reverse=True) + return rows + + def read_session(self, sid: str) -> dict[str, Any] | None: + """Return the full JSON session payload for ``sid`` without mutation.""" + path = self._path_for_sid(sid) + if path is None or not path.exists(): + return None + data = self._read_path(path) + if not isinstance(data, dict): + return None + return copy.deepcopy(data) + + def update_metadata(self, sid: str, fields: dict[str, Any]) -> dict[str, Any]: + """Persist allowlisted metadata fields while preserving messages. + + This dormant adapter method is for migration experiments and tests only. + Runtime wiring must add Session lock/cache/index parity before using it + from live WebUI routes. + """ + if not isinstance(fields, dict): + raise TypeError("fields must be a dict") + unsafe = sorted((set(fields) & _UNSAFE_FIELDS) | (set(fields) - _METADATA_FIELDS)) + if unsafe: + raise ValueError(f"Unsafe session metadata fields: {', '.join(unsafe)}") + + path = self._existing_path_for_sid(sid) + data = self._read_writable_session(path) + data.update(copy.deepcopy(fields)) + data["message_count"] = len(data["messages"]) + self._atomic_write(path, data) + return self._metadata_row(str(data.get("session_id") or sid), data) + + def archive(self, sid: str, archived: bool = True) -> dict[str, Any]: + """Set the archived metadata flag without touching transcript messages.""" + return self.update_metadata(sid, {"archived": bool(archived)}) + + def write_session(self, session: dict[str, Any]) -> dict[str, Any]: + """Write a full session payload for tests and migration experiments.""" + if not isinstance(session, dict): + raise TypeError("session must be a dict") + sid = session.get("session_id") + path = self._path_for_sid(sid) + if path is None: + raise ValueError(f"Unsafe session_id {sid!r}") + messages = session.get("messages") + if not isinstance(messages, list): + raise ValueError("session payload must include a messages list") + payload = copy.deepcopy(session) + payload["message_count"] = len(messages) + path.parent.mkdir(parents=True, exist_ok=True) + self._atomic_write(path, payload) + return copy.deepcopy(payload) + + def _path_for_sid(self, sid: str) -> Path | None: + if not models.is_safe_session_id(sid): + return None + return self.session_dir / f"{sid}.json" + + def _existing_path_for_sid(self, sid: str) -> Path: + path = self._path_for_sid(sid) + if path is None: + raise ValueError(f"Unsafe session_id {sid!r}") + if not path.exists(): + raise KeyError(sid) + return path + + def _read_writable_session(self, path: Path) -> dict[str, Any]: + data = self._read_path(path) + if not isinstance(data, dict): + raise ValueError(f"Malformed session JSON: {path.name}") + sid = data.get("session_id") + if not models.is_safe_session_id(sid): + raise ValueError(f"Unsafe session_id {sid!r}") + if not isinstance(data.get("messages"), list): + raise ValueError(f"Refusing to write metadata-only session stub: {sid!r}") + return data + + @staticmethod + def _read_path(path: Path) -> dict[str, Any] | None: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + + @staticmethod + def _metadata_row(sid: str, data: dict[str, Any]) -> dict[str, Any]: + messages = data.get("messages") + message_count = data.get("message_count") + if not isinstance(message_count, int): + message_count = len(messages) if isinstance(messages, list) else 0 + row = {field: copy.deepcopy(data.get(field)) for field in _METADATA_FIELDS if field in data} + row["session_id"] = sid + row["message_count"] = message_count + row["last_message_at"] = data.get("last_message_at") or data.get("updated_at") or data.get("created_at") + return row + + @staticmethod + def _sort_timestamp(row: dict[str, Any]) -> float: + for key in ("last_message_at", "updated_at", "created_at"): + value = row.get(key) + if value is None or value == "": + continue + try: + return float(value) + except (TypeError, ValueError): + continue + return 0.0 + + @staticmethod + def _atomic_write(path: Path, data: dict[str, Any]) -> None: + payload = json.dumps(data, ensure_ascii=False, indent=2) + tmp = path.with_suffix(f".tmp.{os.getpid()}.{threading.current_thread().ident}") + try: + with open(tmp, "w", encoding="utf-8") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + finally: + try: + tmp.unlink(missing_ok=True) + except OSError: + pass + + +def list_sessions() -> list[dict[str, Any]]: + return WebUIJsonSessionDB().list_sessions() + + +def read_session(sid: str) -> dict[str, Any] | None: + return WebUIJsonSessionDB().read_session(sid) + + +def update_metadata(sid: str, fields: dict[str, Any]) -> dict[str, Any]: + return WebUIJsonSessionDB().update_metadata(sid, fields) + + +def archive(sid: str, archived: bool = True) -> dict[str, Any]: + return WebUIJsonSessionDB().archive(sid, archived) + + +def write_session(session: dict[str, Any]) -> dict[str, Any]: + return WebUIJsonSessionDB().write_session(session) diff --git a/docs/architecture/unified-session-db.md b/docs/architecture/unified-session-db.md new file mode 100644 index 00000000..76489daa --- /dev/null +++ b/docs/architecture/unified-session-db.md @@ -0,0 +1,98 @@ +# Unified SessionDB Adapter Spike + +WebUI currently persists conversations as JSON files under the WebUI session +directory, while the CLI uses its own session database. The first safe slice of +unification is a dormant adapter that presents a small SessionDB-shaped API over +the existing WebUI JSON files without changing runtime call sites or file +format. + +## Adapter Contract + +`api.webui_session_db.WebUIJsonSessionDB` exposes: + +- `list_sessions()` returns compact metadata rows for persisted WebUI JSON + sessions. +- `read_session(sid)` returns a full session JSON payload or `None`. +- `update_metadata(sid, fields)` writes only allowlisted metadata fields and + rejects unsafe keys such as `session_id`, `messages`, `tool_calls`, and + `message_count`. +- `archive(sid, archived=True)` is a convenience metadata update for the + archived flag. +- `write_session(session)` exists for tests and migration experiments that need + to materialize a complete JSON payload. + +Read operations must not call `Session.load()` or `all_sessions()`, because +those paths can repair indexes or transcripts. Metadata writes must load the +complete JSON payload, verify that a real `messages` list is present, update only +safe fields, recompute `message_count`, and atomically replace the file. The +adapter must never write a metadata-only stub that could drop transcript +messages. + +## Why JSON-Backed And Dormant + +The selected first slice is infrastructure only. Keeping the adapter backed by +the current JSON sidecars validates the API shape while preserving all current +WebUI behavior, backups, and import paths. The feature flag defaults to: + +```yaml +experimental: + unified_session_db: false +``` + +No UI exposes this flag, and no runtime session route switches to the adapter in +this slice. + +## Runtime Wiring Preconditions + +Before any route uses this adapter for live metadata changes, a follow-up PR must +prove parity with the existing `Session.save()` path: + +- take the same per-session mutation locks used by streaming and session routes, + so metadata writes cannot replace a newer transcript with a stale copy; +- refresh or invalidate the in-memory `Session` cache and `_index.json`, so + sidebar rows and later `Session.save()` calls cannot overwrite adapter changes; +- match `Session.compact()` sidebar semantics for pending first turns, + `has_pending_user_message`, `pending_started_at`, and real non-tool + `last_message_at` ordering. + +Until those invariants are implemented, `update_metadata()` and `archive()` are +test/migration helpers, not runtime persistence replacements. + +## Planned Migration Sequence + +1. Land the dormant JSON adapter and contract tests. +2. Add parity tests that compare adapter reads with existing WebUI sidebar and + session payloads. +3. Introduce an opt-in dual-read or shadow-read mode for development builds. +4. Add a migration path that can write unified SessionDB records without + deleting or rewriting JSON sidecars. +5. Switch selected call sites behind the flag only after parity and rollback + behavior are proven. +6. Make the unified store authoritative in a later release after import, + archive, pin, profile, project, and recovery semantics match WebUI JSON. + +## Authoritative Fields And Open Questions + +The JSON sidecar remains authoritative for `messages`, `tool_calls`, metadata +display fields, profile/project ownership, archive and pin state, token/cost +totals, pending stream recovery fields, worktree metadata, and composer draft +state during this spike. + +Open questions for later slices: + +- Whether `updated_at` should reflect metadata-only changes such as archive and + pin operations or only transcript changes. +- How to resolve conflicts when CLI and WebUI update titles, archive state, or + project/profile ownership concurrently. +- Whether imported CLI sessions remain read-only projections or become editable + unified records. +- How unified records should preserve WebUI recovery safeguards such as backup + creation before transcript shrinkage. +- Which store owns sidebar ordering once JSON and SessionDB records coexist. + +## Out Of Scope + +This spike does not switch runtime WebUI call sites, migrate existing session +files, expose a UI setting, alter CLI storage, change session import behavior, or +remove any JSON sidecars. It is a contract and safety test bed for future +migration work. diff --git a/tests/test_webui_session_db_adapter.py b/tests/test_webui_session_db_adapter.py new file mode 100644 index 00000000..81bef46d --- /dev/null +++ b/tests/test_webui_session_db_adapter.py @@ -0,0 +1,162 @@ +import json +from pathlib import Path + +import pytest + +import api.config as config +import api.models as models +import api.webui_session_db as session_db +from api.webui_session_db import WebUIJsonSessionDB + + +@pytest.fixture +def session_dir(tmp_path, monkeypatch): + path = tmp_path / "sessions" + path.mkdir() + monkeypatch.setattr(models, "SESSION_DIR", path) + return path + + +def _write_json_session(session_dir, sid="session_1", **overrides): + payload = { + "session_id": sid, + "title": "Adapter Session", + "workspace": str(session_dir.parent), + "model": "gpt-test", + "model_provider": "openai", + "created_at": 100.0, + "updated_at": 200.0, + "pinned": False, + "archived": False, + "profile": "default", + "messages": [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "world"}, + ], + "tool_calls": [{"id": "tool-1", "name": "demo"}], + } + payload.update(overrides) + payload["message_count"] = len(payload["messages"]) + path = session_dir / f"{sid}.json" + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + return payload, path + + +def test_list_and_read_existing_json_sessions(session_dir): + payload, _path = _write_json_session(session_dir) + db = WebUIJsonSessionDB() + + rows = db.list_sessions() + loaded = db.read_session(payload["session_id"]) + + assert [row["session_id"] for row in rows] == [payload["session_id"]] + assert rows[0]["title"] == payload["title"] + assert rows[0]["message_count"] == 2 + assert loaded == payload + + +def test_metadata_update_survives_reload_and_preserves_messages(session_dir): + payload, path = _write_json_session(session_dir) + db = WebUIJsonSessionDB() + + db.update_metadata(payload["session_id"], {"title": "Renamed", "pinned": True}) + reloaded = json.loads(path.read_text(encoding="utf-8")) + + assert reloaded["title"] == "Renamed" + assert reloaded["pinned"] is True + assert reloaded["messages"] == payload["messages"] + assert reloaded["tool_calls"] == payload["tool_calls"] + assert reloaded["message_count"] == len(payload["messages"]) + + +def test_metadata_update_rejects_unsafe_fields(session_dir): + payload, path = _write_json_session(session_dir) + before = path.read_text(encoding="utf-8") + db = WebUIJsonSessionDB() + + with pytest.raises(ValueError): + db.update_metadata(payload["session_id"], {"messages": []}) + with pytest.raises(ValueError): + db.update_metadata(payload["session_id"], {"unknown_field": "unsafe"}) + + assert path.read_text(encoding="utf-8") == before + + +def test_metadata_update_refuses_metadata_only_stub(session_dir): + sid = "stub_session" + path = session_dir / f"{sid}.json" + path.write_text( + json.dumps({"session_id": sid, "title": "Stub"}, indent=2), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="metadata-only"): + WebUIJsonSessionDB().update_metadata(sid, {"title": "Nope"}) + + +def test_archive_unarchive_round_trip(session_dir): + payload, path = _write_json_session(session_dir) + db = WebUIJsonSessionDB() + + archived = db.archive(payload["session_id"]) + unarchived = db.archive(payload["session_id"], archived=False) + reloaded = json.loads(path.read_text(encoding="utf-8")) + + assert archived["archived"] is True + assert unarchived["archived"] is False + assert reloaded["archived"] is False + assert reloaded["messages"] == payload["messages"] + + +def test_read_only_operations_do_not_mutate_files(session_dir): + payload, path = _write_json_session(session_dir) + before_text = path.read_text(encoding="utf-8") + before_stat = path.stat() + db = WebUIJsonSessionDB() + + assert db.list_sessions() + assert db.read_session(payload["session_id"]) == payload + + after_stat = path.stat() + assert path.read_text(encoding="utf-8") == before_text + assert after_stat.st_mtime_ns == before_stat.st_mtime_ns + assert after_stat.st_size == before_stat.st_size + + +def test_sort_timestamp_falls_back_past_missing_values(): + assert WebUIJsonSessionDB._sort_timestamp({"last_message_at": None, "updated_at": 25.0}) == 25.0 + assert WebUIJsonSessionDB._sort_timestamp({"last_message_at": "", "created_at": "15.5"}) == 15.5 + + +def test_module_level_write_session_wrapper(session_dir): + payload, _path = _write_json_session(session_dir, sid="wrapper_session") + written = session_db.write_session(payload) + + assert written == payload + assert session_db.read_session(payload["session_id"]) == payload + + +def test_unified_session_db_flag_default_remains_false(monkeypatch, tmp_path): + cfg_path = tmp_path / "missing-config.yaml" + monkeypatch.setattr(config, "_get_config_path", lambda: cfg_path) + + config.reload_config() + + assert config.get_config()["experimental"]["unified_session_db"] is False + assert config.is_unified_session_db_enabled() is False + assert config.is_unified_session_db_enabled({"experimental": {"unified_session_db": True}}) is True + + +def test_adapter_docs_pin_runtime_wiring_preconditions(): + doc = (Path(__file__).resolve().parents[1] / "docs" / "architecture" / "unified-session-db.md").read_text( + encoding="utf-8" + ) + + assert "Runtime Wiring Preconditions" in doc + assert "per-session mutation locks" in doc + assert "in-memory `Session` cache and `_index.json`" in doc + assert "pending first turns" in doc + assert "test/migration helpers, not runtime persistence replacements" in doc + assert "Runtime wiring must add Session lock/cache/index parity" in ( + session_db.WebUIJsonSessionDB.update_metadata.__doc__ or "" + )