diff --git a/astrbot/core/utils/event_loop_diagnostics.py b/astrbot/core/utils/event_loop_diagnostics.py index 47f51c971..50eabea5d 100644 --- a/astrbot/core/utils/event_loop_diagnostics.py +++ b/astrbot/core/utils/event_loop_diagnostics.py @@ -149,18 +149,23 @@ async def faulthandler_event_loop_watchdog( try: while True: faulthandler.cancel_dump_traceback_later() - output = dump_file or _open_watchdog_log_file(log_path, max_bytes) - should_close = dump_file is None + output: TextIO | None = None + should_close = False try: + output = dump_file or _open_watchdog_log_file(log_path, max_bytes) + should_close = dump_file is None faulthandler.dump_traceback_later( timeout, repeat=False, file=output, ) await asyncio.sleep(interval) + except Exception as e: + logger.warning("Event loop faulthandler watchdog failed: %s", e) + await asyncio.sleep(interval) finally: faulthandler.cancel_dump_traceback_later() - if should_close: + if should_close and output is not None: output.close() finally: faulthandler.cancel_dump_traceback_later() diff --git a/tests/unit/test_event_loop_diagnostics.py b/tests/unit/test_event_loop_diagnostics.py index 73275c5e9..0737478f9 100644 --- a/tests/unit/test_event_loop_diagnostics.py +++ b/tests/unit/test_event_loop_diagnostics.py @@ -97,3 +97,38 @@ async def test_faulthandler_watchdog_writes_rotating_log(tmp_path, monkeypatch): encoding="utf-8" ) == "x" * 8 assert any(isinstance(call, tuple) and call[0] == "dump" for call in calls) + + +@pytest.mark.asyncio +async def test_faulthandler_watchdog_survives_dump_failure(tmp_path, monkeypatch): + """The watchdog should keep running after faulthandler arm failures.""" + log_path = tmp_path / "event_loop_watchdog.log" + armed_again = asyncio.Event() + calls = [] + + class FakeFaultHandler: + def cancel_dump_traceback_later(self): + calls.append("cancel") + + def dump_traceback_later(self, timeout, repeat, file): + calls.append(("dump", timeout, repeat, file.name)) + if len([call for call in calls if isinstance(call, tuple)]) == 1: + raise RuntimeError("boom") + armed_again.set() + + fake_faulthandler = FakeFaultHandler() + monkeypatch.setattr(diagnostics, "faulthandler", fake_faulthandler) + + task = asyncio.create_task( + diagnostics.faulthandler_event_loop_watchdog( + timeout=10, + interval=0.01, + dump_path=log_path, + ) + ) + await asyncio.wait_for(armed_again.wait(), timeout=1) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + dump_calls = [call for call in calls if isinstance(call, tuple)] + assert len(dump_calls) >= 2