mirror of
https://github.com/AstrBotDevs/AstrBot
synced 2026-07-15 17:30:13 +08:00
fix: keep event loop watchdog alive after dump failures
This commit is contained in:
@@ -149,18 +149,23 @@ async def faulthandler_event_loop_watchdog(
|
||||
try:
|
||||
while True:
|
||||
faulthandler.cancel_dump_traceback_later()
|
||||
output = dump_file or _open_watchdog_log_file(log_path, max_bytes)
|
||||
should_close = dump_file is None
|
||||
output: TextIO | None = None
|
||||
should_close = False
|
||||
try:
|
||||
output = dump_file or _open_watchdog_log_file(log_path, max_bytes)
|
||||
should_close = dump_file is None
|
||||
faulthandler.dump_traceback_later(
|
||||
timeout,
|
||||
repeat=False,
|
||||
file=output,
|
||||
)
|
||||
await asyncio.sleep(interval)
|
||||
except Exception as e:
|
||||
logger.warning("Event loop faulthandler watchdog failed: %s", e)
|
||||
await asyncio.sleep(interval)
|
||||
finally:
|
||||
faulthandler.cancel_dump_traceback_later()
|
||||
if should_close:
|
||||
if should_close and output is not None:
|
||||
output.close()
|
||||
finally:
|
||||
faulthandler.cancel_dump_traceback_later()
|
||||
|
||||
@@ -97,3 +97,38 @@ async def test_faulthandler_watchdog_writes_rotating_log(tmp_path, monkeypatch):
|
||||
encoding="utf-8"
|
||||
) == "x" * 8
|
||||
assert any(isinstance(call, tuple) and call[0] == "dump" for call in calls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_faulthandler_watchdog_survives_dump_failure(tmp_path, monkeypatch):
|
||||
"""The watchdog should keep running after faulthandler arm failures."""
|
||||
log_path = tmp_path / "event_loop_watchdog.log"
|
||||
armed_again = asyncio.Event()
|
||||
calls = []
|
||||
|
||||
class FakeFaultHandler:
|
||||
def cancel_dump_traceback_later(self):
|
||||
calls.append("cancel")
|
||||
|
||||
def dump_traceback_later(self, timeout, repeat, file):
|
||||
calls.append(("dump", timeout, repeat, file.name))
|
||||
if len([call for call in calls if isinstance(call, tuple)]) == 1:
|
||||
raise RuntimeError("boom")
|
||||
armed_again.set()
|
||||
|
||||
fake_faulthandler = FakeFaultHandler()
|
||||
monkeypatch.setattr(diagnostics, "faulthandler", fake_faulthandler)
|
||||
|
||||
task = asyncio.create_task(
|
||||
diagnostics.faulthandler_event_loop_watchdog(
|
||||
timeout=10,
|
||||
interval=0.01,
|
||||
dump_path=log_path,
|
||||
)
|
||||
)
|
||||
await asyncio.wait_for(armed_again.wait(), timeout=1)
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
dump_calls = [call for call in calls if isinstance(call, tuple)]
|
||||
assert len(dump_calls) >= 2
|
||||
|
||||
Reference in New Issue
Block a user