fix: keep event loop watchdog alive after dump failures

This commit is contained in:
Soulter
2026-07-07 00:12:05 +08:00
parent f683adc74c
commit a18054c8b4
2 changed files with 43 additions and 3 deletions

View File

@@ -149,18 +149,23 @@ async def faulthandler_event_loop_watchdog(
try:
while True:
faulthandler.cancel_dump_traceback_later()
output = dump_file or _open_watchdog_log_file(log_path, max_bytes)
should_close = dump_file is None
output: TextIO | None = None
should_close = False
try:
output = dump_file or _open_watchdog_log_file(log_path, max_bytes)
should_close = dump_file is None
faulthandler.dump_traceback_later(
timeout,
repeat=False,
file=output,
)
await asyncio.sleep(interval)
except Exception as e:
logger.warning("Event loop faulthandler watchdog failed: %s", e)
await asyncio.sleep(interval)
finally:
faulthandler.cancel_dump_traceback_later()
if should_close:
if should_close and output is not None:
output.close()
finally:
faulthandler.cancel_dump_traceback_later()

View File

@@ -97,3 +97,38 @@ async def test_faulthandler_watchdog_writes_rotating_log(tmp_path, monkeypatch):
encoding="utf-8"
) == "x" * 8
assert any(isinstance(call, tuple) and call[0] == "dump" for call in calls)
@pytest.mark.asyncio
async def test_faulthandler_watchdog_survives_dump_failure(tmp_path, monkeypatch):
"""The watchdog should keep running after faulthandler arm failures."""
log_path = tmp_path / "event_loop_watchdog.log"
armed_again = asyncio.Event()
calls = []
class FakeFaultHandler:
def cancel_dump_traceback_later(self):
calls.append("cancel")
def dump_traceback_later(self, timeout, repeat, file):
calls.append(("dump", timeout, repeat, file.name))
if len([call for call in calls if isinstance(call, tuple)]) == 1:
raise RuntimeError("boom")
armed_again.set()
fake_faulthandler = FakeFaultHandler()
monkeypatch.setattr(diagnostics, "faulthandler", fake_faulthandler)
task = asyncio.create_task(
diagnostics.faulthandler_event_loop_watchdog(
timeout=10,
interval=0.01,
dump_path=log_path,
)
)
await asyncio.wait_for(armed_again.wait(), timeout=1)
task.cancel()
await asyncio.gather(task, return_exceptions=True)
dump_calls = [call for call in calls if isinstance(call, tuple)]
assert len(dump_calls) >= 2