fix(terminal): close the reaper's selection→reconnect→close TOCTOU
`_terminals_to_reap()` snapshots victims under `_LOCK` and then releases it, but `subscribe()` clears `unwatched_since` and appends to `_subscribers` under the terminal's own `_sub_lock`, which the selection pass never held. A viewer reconnecting in that gap left object identity — all `close_terminal(expected=…)` ever checked — still true, so the reaper killed a terminal that now had a live viewer. - `_claim_reap_victim()` re-establishes the *whole* selection predicate while holding `_LOCK` and the terminal's `_sub_lock`, and removes the registry entry in that same critical section: same object, no subscribers, and `unwatched_since` still set and still expired. A dead process is still reaped unconditionally — it cannot come back, and a viewer watching a corpse must not pin the entry forever. - `attach_terminal()` does the registry lookup and the subscribe atomically under `_LOCK`, replacing `get_terminal()` + `term.subscribe()` in the SSE route. It takes the same two locks in the same order as the claim, so exactly one side wins: the viewer attaches (and the reap is refused), or the entry is already gone (and the route answers "terminal not running" before committing a 200, instead of streaming from a corpse). Registration is the authority — a registered-but-`closed` terminal still attaches so the viewer receives `terminal_closed`. - Teardown moved to `_teardown_terminal()` and runs after both locks are released; `killpg`+`wait` can block ~2.5s and must not pin the registry. - Dropped the duplicated `self.last_activity = time.time()` the rebase left in `put_output()`. Tests: a barrier regression that pauses the reaper after selection, attaches a viewer through the production path, resumes, and asserts the terminal stays registered/live and still receives output; the inverse where the reap claims first and the attach reports not-running; refreshed-grace and replaced-entry claims; dead-with-viewer; and a probe that `_LOCK` is free during teardown. Three of them fail against an identity-only claim.
This commit is contained in:
1 parent
932e858994
commit
07d110a064
3 files changed
+299
-19
No files matched your search
@@ -9,6 +9,8 @@ terminal as a belt-and-suspenders. A tab refresh / brief drop re-attaches within
|
||||
the grace and keeps the session.
|
||||
"""
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -154,3 +156,187 @@ def test_reaper_thread_starts_idempotently(monkeypatch):
|
||||
terminal._ensure_terminal_reaper()
|
||||
terminal._ensure_terminal_reaper() # second call is a no-op (already alive)
|
||||
assert started == [1]
|
||||
|
||||
|
||||
# ── Selection → reconnect → close TOCTOU ────────────────────────────────────
|
||||
# `_terminals_to_reap()` snapshots victims under `_LOCK` and then RELEASES it.
|
||||
# `subscribe()` clears `unwatched_since` under the terminal's own `_sub_lock`,
|
||||
# which the selection pass never held, so a viewer can attach in that gap. The
|
||||
# old close path re-checked object identity only — still true — and killed a
|
||||
# terminal that now had a live viewer. The claim must re-establish the whole
|
||||
# selection predicate under both locks.
|
||||
|
||||
|
||||
def _expired_since():
|
||||
"""A wall-clock stamp old enough to select a terminal for reaping."""
|
||||
return time.time() - terminal._TERMINAL_IDLE_GRACE_SECONDS - 5
|
||||
|
||||
|
||||
def test_reconnect_between_selection_and_close_keeps_the_terminal(monkeypatch):
|
||||
"""Attach a viewer after selection but before the claim: no reap.
|
||||
|
||||
Drives the claim with the *stale snapshot* the selection pass produced.
|
||||
Re-running the full `_reap_idle_terminals()` here would not pin anything:
|
||||
its own second selection pass no longer sees the terminal as idle, so the
|
||||
test would pass even with an identity-only claim — the exact defect.
|
||||
"""
|
||||
sid = "toctou-reconnect"
|
||||
now = time.time()
|
||||
term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
|
||||
|
||||
victims = terminal._terminals_to_reap(now)
|
||||
assert [s for s, _ in victims] == [sid], "the terminal was not even selected"
|
||||
|
||||
# The gap: a reconnecting EventSource attaches through the production path.
|
||||
attached = terminal.attach_terminal(sid)
|
||||
assert attached is not None
|
||||
_term, output = attached
|
||||
assert term.unwatched_since is None
|
||||
|
||||
# The claim runs against the snapshot taken BEFORE the reconnect.
|
||||
for victim_sid, victim_term in victims:
|
||||
assert terminal._claim_reap_victim(victim_sid, victim_term, now) is None
|
||||
|
||||
with terminal._LOCK:
|
||||
assert terminal._TERMINALS.get(sid) is term, "a watched terminal was reaped"
|
||||
assert not term.closed.is_set()
|
||||
|
||||
# And it still receives output on that viewer's queue.
|
||||
term.put_output("terminal_output", {"data": "hello"})
|
||||
assert output.get(timeout=1)[1] == "terminal_output"
|
||||
|
||||
|
||||
def test_reconnect_racing_the_claim_under_a_barrier_keeps_the_terminal(monkeypatch):
|
||||
"""Same race, driven deterministically across two threads.
|
||||
|
||||
The reaper is paused *after* it has selected its victim and is about to
|
||||
claim it; the viewer attaches through the production path in that window;
|
||||
then the reaper resumes.
|
||||
"""
|
||||
sid = "toctou-barrier"
|
||||
term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
|
||||
|
||||
selected = threading.Event()
|
||||
attached_done = threading.Event()
|
||||
real_select = terminal._terminals_to_reap
|
||||
|
||||
def paused_select(now):
|
||||
victims = real_select(now)
|
||||
selected.set()
|
||||
# Hold the reaper here until the viewer is attached.
|
||||
assert attached_done.wait(timeout=5)
|
||||
return victims
|
||||
|
||||
monkeypatch.setattr(terminal, "_terminals_to_reap", paused_select)
|
||||
|
||||
result = {}
|
||||
|
||||
def reap():
|
||||
result["reaped"] = terminal._reap_idle_terminals(time.time())
|
||||
|
||||
reaper = threading.Thread(target=reap)
|
||||
reaper.start()
|
||||
try:
|
||||
assert selected.wait(timeout=5)
|
||||
attached = terminal.attach_terminal(sid)
|
||||
assert attached is not None, "attach lost to a reaper that had not claimed yet"
|
||||
_term, output = attached
|
||||
finally:
|
||||
attached_done.set()
|
||||
reaper.join(timeout=10)
|
||||
assert not reaper.is_alive()
|
||||
|
||||
assert result["reaped"] == 0
|
||||
with terminal._LOCK:
|
||||
assert terminal._TERMINALS.get(sid) is term
|
||||
assert not term.closed.is_set()
|
||||
term.put_output("terminal_output", {"data": "still alive"})
|
||||
assert output.get(timeout=1)[1] == "terminal_output"
|
||||
|
||||
|
||||
def test_reap_that_claims_first_makes_attach_report_not_running(monkeypatch):
|
||||
"""The inverse: once the reaper has claimed, an attach must fail cleanly.
|
||||
|
||||
It must not hand back a queue for a terminal that is being torn down —
|
||||
the caller would commit a 200 and stream from a corpse forever.
|
||||
"""
|
||||
sid = "toctou-reap-wins"
|
||||
term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
|
||||
|
||||
assert terminal._reap_idle_terminals(time.time()) == 1
|
||||
assert term.closed.is_set()
|
||||
with terminal._LOCK:
|
||||
assert sid not in terminal._TERMINALS
|
||||
|
||||
assert terminal.attach_terminal(sid) is None
|
||||
|
||||
|
||||
def test_claim_refuses_a_terminal_whose_grace_was_refreshed(monkeypatch):
|
||||
"""`unwatched_since` moved forward after selection → not idle any more."""
|
||||
sid = "toctou-refreshed"
|
||||
term = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
|
||||
now = time.time()
|
||||
assert [s for s, _ in terminal._terminals_to_reap(now)] == [sid]
|
||||
|
||||
# A viewer attached and detached again inside the gap: no subscribers, but
|
||||
# the grace restarted.
|
||||
term.unwatched_since = time.time()
|
||||
|
||||
assert terminal._claim_reap_victim(sid, term, now) is None
|
||||
with terminal._LOCK:
|
||||
assert terminal._TERMINALS.get(sid) is term
|
||||
|
||||
|
||||
def test_claim_refuses_a_replaced_registry_entry(monkeypatch):
|
||||
"""A restart replaced the sid: the old snapshot must not kill the new one."""
|
||||
sid = "toctou-replaced"
|
||||
old = _make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
|
||||
now = time.time()
|
||||
assert [s for s, _ in terminal._terminals_to_reap(now)] == [sid]
|
||||
|
||||
replacement = _make_registered_term(monkeypatch, sid, unwatched_since=None)
|
||||
assert terminal._claim_reap_victim(sid, old, now) is None
|
||||
with terminal._LOCK:
|
||||
assert terminal._TERMINALS.get(sid) is replacement
|
||||
|
||||
|
||||
def test_dead_terminal_is_reaped_even_with_an_attached_viewer(monkeypatch):
|
||||
"""A viewer watching a corpse must not keep the entry alive forever."""
|
||||
sid = "toctou-dead"
|
||||
term = _make_registered_term(monkeypatch, sid, alive=False, unwatched_since=None)
|
||||
term.subscribe()
|
||||
|
||||
assert terminal._reap_idle_terminals(time.time()) == 1
|
||||
with terminal._LOCK:
|
||||
assert sid not in terminal._TERMINALS
|
||||
|
||||
|
||||
def test_teardown_runs_without_holding_the_registry_lock(monkeypatch):
|
||||
"""Process teardown can block for seconds; it must not pin `_LOCK`.
|
||||
|
||||
Otherwise a single hung shell stalls every attach and spawn for the whole
|
||||
kill/wait budget.
|
||||
"""
|
||||
sid = "toctou-teardown-lock"
|
||||
_make_registered_term(monkeypatch, sid, unwatched_since=_expired_since())
|
||||
|
||||
observed = {}
|
||||
|
||||
def slow_killpg(*_a, **_k):
|
||||
# `_LOCK` is an RLock, so probing it from the reaping thread itself
|
||||
# would succeed even while held. Probe from a separate thread, and
|
||||
# acquire/release there — an RLock may only be released by its owner.
|
||||
def probe():
|
||||
acquired = terminal._LOCK.acquire(timeout=0.5)
|
||||
observed["free"] = acquired
|
||||
if acquired:
|
||||
terminal._LOCK.release()
|
||||
|
||||
holder = threading.Thread(target=probe)
|
||||
holder.start()
|
||||
holder.join(timeout=5)
|
||||
|
||||
monkeypatch.setattr(terminal.os, "killpg", slow_killpg)
|
||||
terminal._reap_idle_terminals(time.time())
|
||||
|
||||
assert observed.get("free") is True, "_LOCK was held across process teardown"
|
||||
Reference in new issue
Block a user