perf: update FileReadTool description to mention image, PDF and docx support, and enhance modality checking in tool result case (#7506)

* feat: update FileReadTool description to mention image and PDF support

Add explicit mention of image (OCR) and PDF (text extraction) support
to the FileReadTool description for better discoverability.

* feat: update FileReadTool description to include support for docx and epub files; change base64 decoding to utf-8

* feat: enhance ToolLoopAgentRunner to support image and audio modalities; add context sanitization logic
This commit is contained in:
Soulter
2026-04-22 11:38:40 +08:00
committed by GitHub
parent 662b1d3678
commit e6b68e9b09
7 changed files with 361 additions and 286 deletions

View File

@@ -14,6 +14,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")
from astrbot.core.agent.agent import Agent
from astrbot.core.agent.handoff import HandoffTool
from astrbot.core.agent.hooks import BaseAgentRunHooks
from astrbot.core.agent.message import ImageURLPart, Message, TextPart
from astrbot.core.agent.run_context import ContextWrapper
from astrbot.core.agent.runners.tool_loop_agent_runner import ToolLoopAgentRunner
from astrbot.core.agent.tool import FunctionTool, ToolSet
@@ -156,6 +157,25 @@ class MockErrProvider(MockProvider):
)
class CapturingProvider(MockProvider):
def __init__(self, modalities: list[str]):
super().__init__()
self.provider_config["modalities"] = modalities
self.received_contexts = []
self.received_func_tools = []
self.should_call_tools = False
async def text_chat(self, **kwargs) -> LLMResponse:
self.call_count += 1
self.received_contexts.append(kwargs.get("contexts"))
self.received_func_tools.append(kwargs.get("func_tool"))
return LLMResponse(
role="assistant",
completion_text="final",
usage=TokenUsage(input_other=10, output=5),
)
class MockEmptyOutputThenSuccessProvider(MockProvider):
def __init__(self, failures_before_success: int = 1):
super().__init__()
@@ -615,6 +635,99 @@ async def test_tool_result_includes_all_calltoolresult_content(
]
@pytest.mark.asyncio
async def test_runner_replaces_runtime_image_context_before_provider_call(
runner, provider_request, mock_hooks
):
provider = CapturingProvider(modalities=["tool_use"])
await runner.reset(
provider=provider,
request=provider_request,
run_context=ContextWrapper(context=None),
tool_executor=MockToolExecutor,
agent_hooks=mock_hooks,
streaming=False,
)
runner.run_context.messages.append(
Message(
role="user",
content=[
TextPart(text="Review this image"),
ImageURLPart(
image_url=ImageURLPart.ImageURL(
url="data:image/png;base64,dGVzdA=="
)
),
],
)
)
async for _ in runner.step_until_done(1):
pass
assert provider.received_contexts
sent_context = provider.received_contexts[0]
assert sent_context[-1]["content"] == [
{"type": "text", "text": "Review this image"},
{"type": "text", "text": "[Image]"},
]
assert len(runner.run_context.messages[-2].content) == 2
@pytest.mark.asyncio
async def test_runner_builds_placeholder_for_unsupported_request_image(
runner, mock_hooks, tool_set
):
provider = CapturingProvider(modalities=["tool_use"])
request = ProviderRequest(
prompt="Describe it",
image_urls=["/path/that/should/not/be/read.jpg"],
func_tool=tool_set,
contexts=[],
)
await runner.reset(
provider=provider,
request=request,
run_context=ContextWrapper(context=None),
tool_executor=MockToolExecutor,
agent_hooks=mock_hooks,
streaming=False,
)
async for _ in runner.step_until_done(1):
pass
sent_context = provider.received_contexts[0]
assert sent_context[-1]["content"] == [
{"type": "text", "text": "Describe it"},
{"type": "text", "text": "[Image]"},
]
@pytest.mark.asyncio
async def test_runner_clears_tools_for_provider_without_tool_use(
runner, provider_request, mock_hooks, mock_tool_executor
):
provider = CapturingProvider(modalities=["text"])
await runner.reset(
provider=provider,
request=provider_request,
run_context=ContextWrapper(context=None),
tool_executor=mock_tool_executor,
agent_hooks=mock_hooks,
streaming=False,
)
async for _ in runner.step_until_done(1):
pass
assert provider.received_func_tools == [None]
@pytest.mark.asyncio
async def test_same_tool_consecutive_results_include_escalating_guidance(
runner, mock_tool_executor, mock_hooks