hermes-webui/tests/test_terminal_idle_reaper.py

"""Regression tests — the idle-terminal reaper closes abandoned shells (#4633).

The fd-leak fix retires a terminal whose shell EXITS, and the cap bounds the
worst case, but an interactive shell abandoned by its client (tab closed without
POSTing /api/terminal/close, browser crash, network drop) keeps running because
there is deliberately no PDEATHSIG. The reaper closes a terminal that has had
zero attached output streams for longer than the idle grace, plus any dead-proc
terminal as a belt-and-suspenders. A tab refresh / brief drop re-attaches within
the grace and keeps the session.
"""
import os
import threading
import time

import pytest

if os.name != "posix":
    pytest.skip("terminal tests require POSIX terminal support", allow_module_level=True)

import api.terminal as terminal


class _FakeProc:
    def __init__(self, pid=515151, alive=True):
        self.pid = pid
        self._alive = alive

    def poll(self):
        return None if self._alive else 0

    def wait(self, timeout=None):
        return 0


def _make_registered_term(monkeypatch, sid, *, alive=True, unwatched_since=None):
    r, w = os.pipe()
    os.close(w)
    term = terminal.TerminalSession(
        session_id=sid, workspace="/tmp", proc=_FakeProc(alive=alive), master_fd=r
    )
    term.unwatched_since = unwatched_since
    with terminal._LOCK:
        terminal._TERMINALS[sid] = term
    return term


@pytest.fixture(autouse=True)
def _clean(monkeypatch):
    monkeypatch.setattr(terminal.os, "killpg", lambda *a, **k: None)
    yield
    with terminal._LOCK:
        sids = list(terminal._TERMINALS)
    for sid in sids:
        try:
            terminal.close_terminal(sid)
        except Exception:
            pass


# ── unwatched_since tracking via subscribe/unsubscribe ───────────────────────

def test_unwatched_since_tracks_viewer_attachment():
    term = _make_registered_term_local()
    # Born unwatched.
    assert term.unwatched_since is not None

    a = term.subscribe()
    assert term.unwatched_since is None  # a viewer attached

    b = term.subscribe()
    term.unsubscribe(a)
    assert term.unwatched_since is None  # b still attached

    term.unsubscribe(b)
    assert term.unwatched_since is not None  # last viewer left


def _make_registered_term_local():
    class _P:
        pid = 1

        def poll(self):
            return None

    return terminal.TerminalSession(session_id="x", workspace="/tmp", proc=_P(), master_fd=-1)


# ── _terminals_to_reap selection ─────────────────────────────────────────────

def test_reaps_unwatched_beyond_grace(monkeypatch):
    now = 10_000.0
    grace = terminal._TERMINAL_IDLE_GRACE_SECONDS
    _make_registered_term(monkeypatch, "old", alive=True, unwatched_since=now - grace - 1)
    victims = {sid for sid, _ in terminal._terminals_to_reap(now)}
    assert "old" in victims


def test_keeps_watched_terminal(monkeypatch):
    now = 10_000.0
    # unwatched_since None => a viewer is attached => never reaped, however old.
    _make_registered_term(monkeypatch, "watched", alive=True, unwatched_since=None)
    victims = {sid for sid, _ in terminal._terminals_to_reap(now)}
    assert "watched" not in victims


def test_keeps_recently_unwatched_within_grace(monkeypatch):
    now = 10_000.0
    # Detached 5s ago — within grace, so a reconnecting tab is not killed.
    _make_registered_term(monkeypatch, "reconnecting", alive=True, unwatched_since=now - 5)
    victims = {sid for sid, _ in terminal._terminals_to_reap(now)}
    assert "reconnecting" not in victims


def test_reaps_dead_process_regardless(monkeypatch):
    now = 10_000.0
    # Dead proc but "watched" — still swept (belt-and-suspenders).
    _make_registered_term(monkeypatch, "dead", alive=False, unwatched_since=None)
    victims = {sid for sid, _ in terminal._terminals_to_reap(now)}
    assert "dead" in victims


# ── _reap_idle_terminals effect ──────────────────────────────────────────────

def test_reap_closes_and_removes(monkeypatch):
    now = 10_000.0
    grace = terminal._TERMINAL_IDLE_GRACE_SECONDS
    term = _make_registered_term(monkeypatch, "reapme", alive=True, unwatched_since=now - grace - 1)
    fd = term.master_fd

    n = terminal._reap_idle_terminals(now)

    assert n == 1
    with terminal._LOCK:
        assert "reapme" not in terminal._TERMINALS
    with pytest.raises(OSError):
        os.fstat(fd)  # fd closed


def test_reaper_thread_starts_idempotently(monkeypatch):
    # Reset reaper state so the ensure actually starts a (dummy) thread.
    monkeypatch.setattr(terminal, "_terminal_reaper_started", False)
    monkeypatch.setattr(terminal, "_terminal_reaper_thread", None)
    started = []

    class _DummyThread:
        def __init__(self, *a, **k):
            self._k = k

        def start(self):
            started.append(1)

        def is_alive(self):
            return True

    monkeypatch.setattr(terminal.threading, "Thread", _DummyThread)
    terminal._ensure_terminal_reaper()
    terminal._ensure_terminal_reaper()  # second call is a no-op (already alive)
    assert started == [1]


# ── Selection → reconnect → close TOCTOU ────────────────────────────────────
# `_terminals_to_reap()` snapshots victims under `_LOCK` and then RELEASES it.
# `subscribe()` clears `unwatched_since` under the terminal's own `_sub_lock`,
# which the selection pass never held, so a viewer can attach in that gap. The
# old close path re-checked object identity only — still true — and killed a
# terminal that now had a live viewer. The claim must re-establish the whole
# selection predicate under both locks.


def _expired_since():
    """A wall-clock stamp old enough to select a terminal for reaping."""
    return time.time() - terminal._TERMINAL_IDLE_GRACE_SECONDS - 5


def test_reconnect_between_selection_and_close_keeps_the_terminal(monkeypatch):
    """Attach a viewer after selection but before the claim: no reap.

    Drives the claim with the *stale snapshot* the selection pass produced.
    Re-running the full `_reap_idle_terminals()` here would not pin anything:
    its own second selection pass no longer sees the terminal as idle, so the
    test would pass even with an identity-only claim — the exact defect.
    """
    sid = "toctou-reconnect"
    now = time.time()
    term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())

    victims = terminal._terminals_to_reap(now)
    assert [s for s, _ in victims] == [sid], "the terminal was not even selected"

    # The gap: a reconnecting EventSource attaches through the production path.
    attached = terminal.attach_terminal(sid)
    assert attached is not None
    _term, output = attached
    assert term.unwatched_since is None

    # The claim runs against the snapshot taken BEFORE the reconnect.
    for victim_sid, victim_term in victims:
        assert terminal._claim_reap_victim(victim_sid, victim_term, now) is None

    with terminal._LOCK:
        assert terminal._TERMINALS.get(sid) is term, "a watched terminal was reaped"
    assert not term.closed.is_set()

    # And it still receives output on that viewer's queue.
    term.put_output("terminal_output", {"data": "hello"})
    assert output.get(timeout=1)[1] == "terminal_output"


def test_reconnect_racing_the_claim_under_a_barrier_keeps_the_terminal(monkeypatch):
    """Same race, driven deterministically across two threads.

    The reaper is paused *after* it has selected its victim and is about to
    claim it; the viewer attaches through the production path in that window;
    then the reaper resumes.
    """
    sid = "toctou-barrier"
    term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())

    selected = threading.Event()
    attached_done = threading.Event()
    real_select = terminal._terminals_to_reap

    def paused_select(now):
        victims = real_select(now)
        selected.set()
        # Hold the reaper here until the viewer is attached.
        assert attached_done.wait(timeout=5)
        return victims

    monkeypatch.setattr(terminal, "_terminals_to_reap", paused_select)

    result = {}

    def reap():
        result["reaped"] = terminal._reap_idle_terminals(time.time())

    reaper = threading.Thread(target=reap)
    reaper.start()
    try:
        assert selected.wait(timeout=5)
        attached = terminal.attach_terminal(sid)
        assert attached is not None, "attach lost to a reaper that had not claimed yet"
        _term, output = attached
    finally:
        attached_done.set()
        reaper.join(timeout=10)
    assert not reaper.is_alive()

    assert result["reaped"] == 0
    with terminal._LOCK:
        assert terminal._TERMINALS.get(sid) is term
    assert not term.closed.is_set()
    term.put_output("terminal_output", {"data": "still alive"})
    assert output.get(timeout=1)[1] == "terminal_output"


def test_reap_that_claims_first_makes_attach_report_not_running(monkeypatch):
    """The inverse: once the reaper has claimed, an attach must fail cleanly.

    It must not hand back a queue for a terminal that is being torn down —
    the caller would commit a 200 and stream from a corpse forever.
    """
    sid = "toctou-reap-wins"
    term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())

    assert terminal._reap_idle_terminals(time.time()) == 1
    assert term.closed.is_set()
    with terminal._LOCK:
        assert sid not in terminal._TERMINALS

    assert terminal.attach_terminal(sid) is None


def test_claim_refuses_a_terminal_whose_grace_was_refreshed(monkeypatch):
    """`unwatched_since` moved forward after selection → not idle any more."""
    sid = "toctou-refreshed"
    term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
    now = time.time()
    assert [s for s, _ in terminal._terminals_to_reap(now)] == [sid]

    # A viewer attached and detached again inside the gap: no subscribers, but
    # the grace restarted.
    term.unwatched_since = time.time()

    assert terminal._claim_reap_victim(sid, term, now) is None
    with terminal._LOCK:
        assert terminal._TERMINALS.get(sid) is term


def test_claim_refuses_a_replaced_registry_entry(monkeypatch):
    """A restart replaced the sid: the old snapshot must not kill the new one."""
    sid = "toctou-replaced"
    old = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
    now = time.time()
    assert [s for s, _ in terminal._terminals_to_reap(now)] == [sid]

    replacement = _make_registered_term(monkeypatch, sid, unwatched_since=None)
    assert terminal._claim_reap_victim(sid, old, now) is None
    with terminal._LOCK:
        assert terminal._TERMINALS.get(sid) is replacement


def test_dead_terminal_is_reaped_even_with_an_attached_viewer(monkeypatch):
    """A viewer watching a corpse must not keep the entry alive forever."""
    sid = "toctou-dead"
    term = _make_registered_term(monkeypatch, sid, alive=False, unwatched_since=None)
    term.subscribe()

    assert terminal._reap_idle_terminals(time.time()) == 1
    with terminal._LOCK:
        assert sid not in terminal._TERMINALS


def test_teardown_runs_without_holding_the_registry_lock(monkeypatch):
    """Process teardown can block for seconds; it must not pin `_LOCK`.

    Otherwise a single hung shell stalls every attach and spawn for the whole
    kill/wait budget.
    """
    sid = "toctou-teardown-lock"
    _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())

    observed = {}

    def slow_killpg(*_a, **_k):
        # `_LOCK` is an RLock, so probing it from the reaping thread itself
        # would succeed even while held. Probe from a separate thread, and
        # acquire/release there — an RLock may only be released by its owner.
        def probe():
            acquired = terminal._LOCK.acquire(timeout=0.5)
            observed["free"] = acquired
            if acquired:
                terminal._LOCK.release()

        holder = threading.Thread(target=probe)
        holder.start()
        holder.join(timeout=5)

    monkeypatch.setattr(terminal.os, "killpg", slow_killpg)
    terminal._reap_idle_terminals(time.time())

    assert observed.get("free") is True, "_LOCK was held across process teardown"


def test_a_header_write_failure_after_attach_still_unsubscribes(monkeypatch):
    """Attaching before the headers must not be able to leak the subscriber.

    `attach_terminal()` subscribes, and only then are the response headers
    written. A client that dropped in that instant makes `end_headers()` raise
    BrokenPipeError; if that escaped before the try/finally, the queue would
    stay in `_subscribers` forever, pin `unwatched_since` at None, and make the
    terminal permanently unreapable — the very leak this reaper exists to
    prevent, self-inflicted.
    """
    import io
    from types import SimpleNamespace

    import api.routes as routes

    sid = "sse-header-failure"
    term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())

    class _Handler:
        headers = {}

        def __init__(self):
            self.wfile = io.BytesIO()
            self.status = None

        def send_response(self, status):
            self.status = status

        def send_header(self, _name, _value):
            pass

        def end_headers(self):
            raise ConnectionResetError("client vanished")

    monkeypatch.setattr(routes, "_embedded_terminal_gate_allows", lambda _handler: True)
    monkeypatch.setattr(routes, "_sse_set_write_deadline", lambda _handler: None)

    routes._handle_terminal_output(
        _Handler(), SimpleNamespace(query=f"session_id={sid}")
    )

    assert term._subscribers == [], "the subscriber leaked on a header-write failure"
    assert term.unwatched_since is not None, "the terminal was pinned as watched forever"
    # The grace restarts from the detach, which is correct — but the terminal
    # is reapable again once it elapses, instead of being pinned forever.
    assert terminal._reap_idle_terminals(
        term.unwatched_since + terminal._TERMINAL_IDLE_GRACE_SECONDS + 1
    ) == 1