stage #4447: opt-in chunked SSE for buffering proxies (#4447) + v0.51.524 CHANGELOG

CHANGELOG.md

@@ -3,6 +3,12 @@
 
 ## [Unreleased]
 
+## [v0.51.524] — 2026-06-19 — Release SI (opt-in chunked SSE for buffering reverse proxies)
+
+### Added
+
+- **Opt-in chunked transfer-encoding for SSE streams, so live streams survive buffering reverse proxies (#4447).** Tornado-based proxies (notably `jupyter-server-proxy`, which fronts the WebUI in JupyterHub/REANA deployments) read the stdlib server's unframed SSE body with read-until-close semantics and buffer the entire stream until the connection dies, so chat appears frozen until the client's watchdog kills it. Setting `HERMES_WEBUI_SSE_CHUNKED=1` makes each SSE event a discrete HTTP/1.1 chunk so it flushes through every hop immediately. Default-off: when unset, the wire format is byte-for-byte unchanged, so directly-served deployments are unaffected. Thanks @tomtong2015.
+
 ## [v0.51.523] — 2026-06-19 — Release SH (cache trusted wiki page listings on browser reads)
 
 ### Changed

README.md

@@ -342,6 +342,7 @@ Full list of environment variables:
 | `HERMES_WEBUI_DEFAULT_MODEL` | *(provider default)* | Optional model override; leave unset to use the active Hermes provider default |
 | `HERMES_WEBUI_PASSWORD` | *(unset)* | Set to enable password authentication |
 | `HERMES_WEBUI_CSP_CONNECT_EXTRA` | *(unset)* | Optional space-separated `http(s)://` or `ws(s)://` origins to append to the report-only CSP `connect-src` directive for reverse-proxy or tunnel deployments |
+| `HERMES_WEBUI_SSE_CHUNKED` | *(unset)* | Set truthy (`1`/`true`/`yes`/`on`) to send SSE with `Transfer-Encoding: chunked`. Needed behind buffering reverse proxies (e.g. `jupyter-server-proxy`) that otherwise buffer the whole stream; harmless but unnecessary for directly-served deployments |
 | `HERMES_WEBUI_EXTENSION_DIR` | *(unset)* | Optional local directory served at `/extensions/`; must point to an existing directory before extension injection is enabled |
 | `HERMES_WEBUI_EXTENSION_SCRIPT_URLS` | *(unset)* | Optional comma-separated same-origin script URLs to inject; see [WebUI Extensions](docs/EXTENSIONS.md) |
 | `HERMES_WEBUI_EXTENSION_STYLESHEET_URLS` | *(unset)* | Optional comma-separated same-origin stylesheet URLs to inject; see [WebUI Extensions](docs/EXTENSIONS.md) |

api/kanban_bridge.py

@@ -14,6 +14,7 @@ Supported operations:
 from __future__ import annotations
 
 import json
+from api.sse_chunked import end_sse_headers
 import time
 from dataclasses import asdict, is_dataclass
 from urllib.parse import parse_qs, unquote
@@ -1065,7 +1066,7 @@ def _handle_events_sse_stream(handler, parsed):
     handler.send_header("Cache-Control", "no-cache")
     handler.send_header("X-Accel-Buffering", "no")
     handler.send_header("Connection", "close")
-    handler.end_headers()
+    end_sse_headers(handler)
 
     # Send an initial frame so the client knows the connection is open
     # and learns the current cursor (in case the server already had a

api/routes.py

@@ -9,6 +9,7 @@ import errno
 import io
 import gzip
 import json
+from api.sse_chunked import end_sse_headers
 import logging
 import os
 import queue
@@ -11404,7 +11405,7 @@ def _stream_runner_run_events(handler, run_id: str, cursor: str | None = None) -
     handler.send_header("Cache-Control", "no-cache")
     handler.send_header("X-Accel-Buffering", "no")
     handler.send_header("Connection", "close")
-    handler.end_headers()
+    end_sse_headers(handler)
     cursor_value = cursor
     try:
         while True:
@@ -11464,7 +11465,7 @@ def _handle_sse_stream(handler, parsed):
         handler.send_header("Cache-Control", "no-cache")
         handler.send_header("X-Accel-Buffering", "no")
         handler.send_header("Connection", "close")
-        handler.end_headers()
+        end_sse_headers(handler)
         try:
             _replay_run_journal(handler, stream_id, _parse_run_journal_after_seq(qs, stream_id))
         except _CLIENT_DISCONNECT_ERRORS:
@@ -11480,7 +11481,7 @@ def _handle_sse_stream(handler, parsed):
     handler.send_header("Cache-Control", "no-cache")
     handler.send_header("X-Accel-Buffering", "no")
     handler.send_header("Connection", "close")
-    handler.end_headers()
+    end_sse_headers(handler)
     _sse_set_write_deadline(handler)  # Defect A: slow tab can't pin this thread
     replay_cutoff_seq = None
     if qs.get("replay", [""])[0] or qs.get("after_seq", [None])[0] not in (None, "") or qs.get("after_event_id", [None])[0]:
@@ -11660,7 +11661,7 @@ def _handle_terminal_output(handler, parsed):
     handler.send_header("Cache-Control", "no-cache")
     handler.send_header("X-Accel-Buffering", "no")
     handler.send_header("Connection", "close")
-    handler.end_headers()
+    end_sse_headers(handler)
     _sse_set_write_deadline(handler)  # Defect A: slow tab can't pin this thread
     try:
         while True:
@@ -11746,7 +11747,7 @@ def _handle_gateway_sse_stream(handler, parsed):
     # connect/sessions_changed snapshot/disconnect that thrashes the
     # session list every ~1s. Letting the server close the socket
     # naturally after the stream ends is sufficient.
-    handler.end_headers()
+    end_sse_headers(handler)
     _sse_set_write_deadline(handler)  # Defect A: slow tab can't pin this thread
 
     q = watcher.subscribe()
@@ -11781,7 +11782,7 @@ def _handle_session_events_stream(handler):
     handler.send_header('X-Accel-Buffering', 'no')
     # #3103: see _handle_gateway_sse_stream — `Connection: close` causes
     # EventSource reconnect storms in browsers on long-lived SSE.
-    handler.end_headers()
+    end_sse_headers(handler)
 
     q = subscribe_session_events()
     try:
@@ -12922,7 +12923,7 @@ def _handle_approval_sse_stream(handler, parsed):
     handler.send_header('Cache-Control', 'no-cache')
     handler.send_header('X-Accel-Buffering', 'no')
     handler.send_header('Connection', 'close')
-    handler.end_headers()
+    end_sse_headers(handler)
     _sse_set_write_deadline(handler)  # Defect A: slow tab can't pin this thread
 
     from api.streaming import _sse
@@ -13024,7 +13025,7 @@ def _handle_clarify_sse_stream(handler, parsed):
     handler.send_header('Cache-Control', 'no-cache')
     handler.send_header('X-Accel-Buffering', 'no')
     handler.send_header('Connection', 'close')
-    handler.end_headers()
+    end_sse_headers(handler)
     _sse_set_write_deadline(handler)  # Defect A: slow tab can't pin this thread
 
     from api.streaming import _sse
@@ -13104,7 +13105,7 @@ def _handle_session_sse_stream(handler, parsed):
         # default, matching the other long-lived SSE handlers (gateway/session
         # events) that fixed the reconnect-storm. An explicit value here is a
         # third, inconsistent approach (greptile flag).
-        handler.end_headers()
+        end_sse_headers(handler)
         _sse_set_write_deadline(handler)  # Defect A: slow tab can't pin this thread
 
         from api.streaming import _sse

api/sse_chunked.py

@@ -0,0 +1,67 @@
+"""Opt-in chunked transfer-encoding for SSE responses.
+
+The stdlib HTTP server emits SSE bodies with no framing at all (no
+Content-Length, no Transfer-Encoding). Tornado-based reverse proxies —
+notably jupyter-server-proxy, which fronts the WebUI in JupyterHub/REANA
+deployments — read such responses with read-until-close semantics and
+buffer the ENTIRE body until the upstream connection dies, so the browser
+receives no events while a stream is live: chat appears frozen, then the
+client's watchdog kills it ("Connection interrupted"). Explicit chunked
+framing lets every event flush through each hop immediately.
+
+This is opt-in via the ``HERMES_WEBUI_SSE_CHUNKED`` environment variable so
+the default wire format is unchanged: directly-served deployments keep the
+historical unframed stream, and only deployments behind a buffering proxy
+need to set the flag. It sits alongside the existing ``X-Accel-Buffering: no``
+proxy-compatibility hint on these same handlers.
+
+Usage: in an SSE handler, replace ``handler.end_headers()`` with
+``end_sse_headers(handler)``. When the flag is set, all subsequent
+``handler.wfile.write()`` calls are framed transparently.
+"""
+
+import os
+
+_TRUTHY = {"1", "true", "yes", "on"}
+
+
+def chunked_sse_enabled() -> bool:
+    """True when ``HERMES_WEBUI_SSE_CHUNKED`` opts into chunked SSE framing."""
+    return os.getenv("HERMES_WEBUI_SSE_CHUNKED", "").strip().lower() in _TRUTHY
+
+
+class _ChunkedSSEWriter:
+    """Wrap ``wfile`` so each write becomes an HTTP/1.1 chunk."""
+
+    def __init__(self, raw):
+        self._raw = raw
+
+    def write(self, data):
+        if not data:
+            return 0
+        payload = bytes(data)
+        self._raw.write(b"%X\r\n" % len(payload) + payload + b"\r\n")
+        return len(payload)
+
+    def flush(self):
+        self._raw.flush()
+
+    def __getattr__(self, name):
+        return getattr(self._raw, name)
+
+
+def end_sse_headers(handler):
+    """Finish SSE response headers, optionally enabling chunked framing.
+
+    When ``HERMES_WEBUI_SSE_CHUNKED`` is set, send ``Transfer-Encoding: chunked``
+    and wrap ``wfile`` so each write is framed as one HTTP/1.1 chunk; otherwise
+    behave exactly like ``handler.end_headers()`` so the default wire format is
+    preserved. Chunked + ``Connection: close`` is legal and unambiguous on the
+    HTTP/1.1 responses these handlers emit.
+    """
+    if chunked_sse_enabled():
+        handler.send_header("Transfer-Encoding", "chunked")
+        handler.end_headers()
+        handler.wfile = _ChunkedSSEWriter(handler.wfile)
+    else:
+        handler.end_headers()

tests/test_session_channel_option_x.py

@@ -909,7 +909,9 @@ def test_session_sse_stream_unsubscribes_on_header_write_failure():
     sub_ix = body.find("subscribe_to_session_channel(")
     assert sub_ix != -1, "subscribe call not found"
     try_ix = body.find("try:", sub_ix)
-    end_headers_ix = body.find("end_headers()", sub_ix)
+    # SSE handlers finish their headers via end_sse_headers() (api/sse_chunked),
+    # which is exactly handler.end_headers() unless chunked framing is opted in.
+    end_headers_ix = body.find("end_sse_headers(handler)", sub_ix)
     finally_ix = body.find("finally:", sub_ix)
     unsub_ix = body.find("ch.unsubscribe(q)", finally_ix if finally_ix != -1 else sub_ix)
 

tests/test_sse_chunked.py

@@ -0,0 +1,78 @@
+"""Tests for opt-in chunked transfer-encoding on SSE responses.
+
+See api/sse_chunked.py: behind a buffering reverse proxy (jupyter-server-proxy)
+unframed SSE is buffered until-close, so HERMES_WEBUI_SSE_CHUNKED opts into
+explicit HTTP/1.1 chunk framing. Default off must preserve the historical
+unframed wire format byte-for-byte.
+"""
+
+import io
+
+from api.sse_chunked import chunked_sse_enabled, end_sse_headers
+
+
+class _Handler:
+    def __init__(self):
+        self.wfile = io.BytesIO()
+        self.headers_sent = []
+        self.ended = False
+
+    def send_header(self, name, value):
+        self.headers_sent.append((name, value))
+
+    def end_headers(self):
+        self.ended = True
+
+
+def _dechunk(raw: bytes) -> bytes:
+    """Decode an HTTP/1.1 chunked body (no trailing zero-length terminator)."""
+    out = bytearray()
+    i = 0
+    while i < len(raw):
+        nl = raw.index(b"\r\n", i)
+        size = int(raw[i:nl], 16)
+        start = nl + 2
+        out += raw[start : start + size]
+        i = start + size + 2  # skip chunk data + trailing CRLF
+    return bytes(out)
+
+
+def test_default_off_is_plain_end_headers(monkeypatch):
+    monkeypatch.delenv("HERMES_WEBUI_SSE_CHUNKED", raising=False)
+    assert chunked_sse_enabled() is False
+
+    h = _Handler()
+    end_sse_headers(h)
+    h.wfile.write(b"id: 1\n")
+    h.wfile.write(b"event: token\ndata: {}\n\n")
+
+    assert h.ended is True
+    # No Transfer-Encoding header and the body is written verbatim (unframed).
+    assert all(name != "Transfer-Encoding" for name, _ in h.headers_sent)
+    assert h.wfile.getvalue() == b"id: 1\nevent: token\ndata: {}\n\n"
+
+
+def test_flag_on_emits_chunked_frames_that_decode_back(monkeypatch):
+    monkeypatch.setenv("HERMES_WEBUI_SSE_CHUNKED", "1")
+    assert chunked_sse_enabled() is True
+
+    h = _Handler()
+    end_sse_headers(h)
+    h.wfile.write(b"id: 1\n")
+    h.wfile.write(b"event: token\ndata: {}\n\n")
+
+    assert ("Transfer-Encoding", "chunked") in h.headers_sent
+    raw = h.wfile._raw.getvalue()
+    # Each write became its own chunk (size prefix present), and the chunked
+    # body decodes back to exactly what was written — what any HTTP client sees.
+    assert raw.startswith(b"6\r\nid: 1\n\r\n")
+    assert _dechunk(raw) == b"id: 1\nevent: token\ndata: {}\n\n"
+
+
+def test_truthy_values(monkeypatch):
+    for val in ("1", "true", "TRUE", "yes", "On"):
+        monkeypatch.setenv("HERMES_WEBUI_SSE_CHUNKED", val)
+        assert chunked_sse_enabled() is True
+    for val in ("", "0", "false", "no", "off"):
+        monkeypatch.setenv("HERMES_WEBUI_SSE_CHUNKED", val)
+        assert chunked_sse_enabled() is False