fix(terminal): close the reaper's selection→reconnect→close TOCTOU

`_terminals_to_reap()` snapshots victims under `_LOCK` and then releases it,
but `subscribe()` clears `unwatched_since` and appends to `_subscribers` under
the terminal's own `_sub_lock`, which the selection pass never held. A viewer
reconnecting in that gap left object identity — all `close_terminal(expected=…)`
ever checked — still true, so the reaper killed a terminal that now had a live
viewer.

- `_claim_reap_victim()` re-establishes the *whole* selection predicate while
  holding `_LOCK` and the terminal's `_sub_lock`, and removes the registry
  entry in that same critical section: same object, no subscribers, and
  `unwatched_since` still set and still expired. A dead process is still
  reaped unconditionally — it cannot come back, and a viewer watching a corpse
  must not pin the entry forever.
- `attach_terminal()` does the registry lookup and the subscribe atomically
  under `_LOCK`, replacing `get_terminal()` + `term.subscribe()` in the SSE
  route. It takes the same two locks in the same order as the claim, so
  exactly one side wins: the viewer attaches (and the reap is refused), or the
  entry is already gone (and the route answers "terminal not running" before
  committing a 200, instead of streaming from a corpse). Registration is the
  authority — a registered-but-`closed` terminal still attaches so the viewer
  receives `terminal_closed`.
- Teardown moved to `_teardown_terminal()` and runs after both locks are
  released; `killpg`+`wait` can block ~2.5s and must not pin the registry.
- Dropped the duplicated `self.last_activity = time.time()` the rebase left in
  `put_output()`.

Tests: a barrier regression that pauses the reaper after selection, attaches a
viewer through the production path, resumes, and asserts the terminal stays
registered/live and still receives output; the inverse where the reap claims
first and the attach reports not-running; refreshed-grace and replaced-entry
claims; dead-with-viewer; and a probe that `_LOCK` is free during teardown.
Three of them fail against an identity-only claim.
This commit is contained in:
ai-ag2026 committed 2026-07-25 00:33:41 +02:00
1 parent 932e858994
commit 07d110a064
3 files changed
+299 -19

No files matched your search

+186
View File
@@ -9,6 +9,8 @@ terminal as a belt-and-suspenders. A tab refresh / brief drop re-attaches within
the grace and keeps the session.
"""
import os
import threading
import time
import pytest
@@ -154,3 +156,187 @@ def test_reaper_thread_starts_idempotently(monkeypatch):
terminal._ensure_terminal_reaper()
terminal._ensure_terminal_reaper() # second call is a no-op (already alive)
assert started == [1]
# ── Selection → reconnect → close TOCTOU ────────────────────────────────────
# `_terminals_to_reap()` snapshots victims under `_LOCK` and then RELEASES it.
# `subscribe()` clears `unwatched_since` under the terminal's own `_sub_lock`,
# which the selection pass never held, so a viewer can attach in that gap. The
# old close path re-checked object identity only — still true — and killed a
# terminal that now had a live viewer. The claim must re-establish the whole
# selection predicate under both locks.
def _expired_since():
"""A wall-clock stamp old enough to select a terminal for reaping."""
return time.time() - terminal._TERMINAL_IDLE_GRACE_SECONDS - 5
def test_reconnect_between_selection_and_close_keeps_the_terminal(monkeypatch):
"""Attach a viewer after selection but before the claim: no reap.
Drives the claim with the *stale snapshot* the selection pass produced.
Re-running the full `_reap_idle_terminals()` here would not pin anything:
its own second selection pass no longer sees the terminal as idle, so the
test would pass even with an identity-only claim — the exact defect.
"""
sid = "toctou-reconnect"
now = time.time()
term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
victims = terminal._terminals_to_reap(now)
assert [s for s, _ in victims] == [sid], "the terminal was not even selected"
# The gap: a reconnecting EventSource attaches through the production path.
attached = terminal.attach_terminal(sid)
assert attached is not None
_term, output = attached
assert term.unwatched_since is None
# The claim runs against the snapshot taken BEFORE the reconnect.
for victim_sid, victim_term in victims:
assert terminal._claim_reap_victim(victim_sid, victim_term, now) is None
with terminal._LOCK:
assert terminal._TERMINALS.get(sid) is term, "a watched terminal was reaped"
assert not term.closed.is_set()
# And it still receives output on that viewer's queue.
term.put_output("terminal_output", {"data": "hello"})
assert output.get(timeout=1)[1] == "terminal_output"
def test_reconnect_racing_the_claim_under_a_barrier_keeps_the_terminal(monkeypatch):
"""Same race, driven deterministically across two threads.
The reaper is paused *after* it has selected its victim and is about to
claim it; the viewer attaches through the production path in that window;
then the reaper resumes.
"""
sid = "toctou-barrier"
term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
selected = threading.Event()
attached_done = threading.Event()
real_select = terminal._terminals_to_reap
def paused_select(now):
victims = real_select(now)
selected.set()
# Hold the reaper here until the viewer is attached.
assert attached_done.wait(timeout=5)
return victims
monkeypatch.setattr(terminal, "_terminals_to_reap", paused_select)
result = {}
def reap():
result["reaped"] = terminal._reap_idle_terminals(time.time())
reaper = threading.Thread(target=reap)
reaper.start()
try:
assert selected.wait(timeout=5)
attached = terminal.attach_terminal(sid)
assert attached is not None, "attach lost to a reaper that had not claimed yet"
_term, output = attached
finally:
attached_done.set()
reaper.join(timeout=10)
assert not reaper.is_alive()
assert result["reaped"] == 0
with terminal._LOCK:
assert terminal._TERMINALS.get(sid) is term
assert not term.closed.is_set()
term.put_output("terminal_output", {"data": "still alive"})
assert output.get(timeout=1)[1] == "terminal_output"
def test_reap_that_claims_first_makes_attach_report_not_running(monkeypatch):
"""The inverse: once the reaper has claimed, an attach must fail cleanly.
It must not hand back a queue for a terminal that is being torn down —
the caller would commit a 200 and stream from a corpse forever.
"""
sid = "toctou-reap-wins"
term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
assert terminal._reap_idle_terminals(time.time()) == 1
assert term.closed.is_set()
with terminal._LOCK:
assert sid not in terminal._TERMINALS
assert terminal.attach_terminal(sid) is None
def test_claim_refuses_a_terminal_whose_grace_was_refreshed(monkeypatch):
"""`unwatched_since` moved forward after selection → not idle any more."""
sid = "toctou-refreshed"
term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
now = time.time()
assert [s for s, _ in terminal._terminals_to_reap(now)] == [sid]
# A viewer attached and detached again inside the gap: no subscribers, but
# the grace restarted.
term.unwatched_since = time.time()
assert terminal._claim_reap_victim(sid, term, now) is None
with terminal._LOCK:
assert terminal._TERMINALS.get(sid) is term
def test_claim_refuses_a_replaced_registry_entry(monkeypatch):
"""A restart replaced the sid: the old snapshot must not kill the new one."""
sid = "toctou-replaced"
old = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
now = time.time()
assert [s for s, _ in terminal._terminals_to_reap(now)] == [sid]
replacement = _make_registered_term(monkeypatch, sid, unwatched_since=None)
assert terminal._claim_reap_victim(sid, old, now) is None
with terminal._LOCK:
assert terminal._TERMINALS.get(sid) is replacement
def test_dead_terminal_is_reaped_even_with_an_attached_viewer(monkeypatch):
"""A viewer watching a corpse must not keep the entry alive forever."""
sid = "toctou-dead"
term = _make_registered_term(monkeypatch, sid, alive=False, unwatched_since=None)
term.subscribe()
assert terminal._reap_idle_terminals(time.time()) == 1
with terminal._LOCK:
assert sid not in terminal._TERMINALS
def test_teardown_runs_without_holding_the_registry_lock(monkeypatch):
"""Process teardown can block for seconds; it must not pin `_LOCK`.
Otherwise a single hung shell stalls every attach and spawn for the whole
kill/wait budget.
"""
sid = "toctou-teardown-lock"
_make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
observed = {}
def slow_killpg(*_a, **_k):
# `_LOCK` is an RLock, so probing it from the reaping thread itself
# would succeed even while held. Probe from a separate thread, and
# acquire/release there — an RLock may only be released by its owner.
def probe():
acquired = terminal._LOCK.acquire(timeout=0.5)
observed["free"] = acquired
if acquired:
terminal._LOCK.release()
holder = threading.Thread(target=probe)
holder.start()
holder.join(timeout=5)
monkeypatch.setattr(terminal.os, "killpg", slow_killpg)
terminal._reap_idle_terminals(time.time())
assert observed.get("free") is True, "_LOCK was held across process teardown"