perf: update FileReadTool description to mention image, PDF and docx support, and enhance modality checking in tool result case (#7506)

* feat: update FileReadTool description to mention image and PDF support

Add explicit mention of image (OCR) and PDF (text extraction) support
to the FileReadTool description for better discoverability.

* feat: update FileReadTool description to include support for docx and epub files; change base64 decoding to utf-8

* feat: enhance ToolLoopAgentRunner to support image and audio modalities; add context sanitization logic
This commit is contained in:
Soulter
2026-04-22 11:38:40 +08:00
committed by GitHub
parent 662b1d3678
commit e6b68e9b09
7 changed files with 361 additions and 286 deletions

View File

@@ -713,144 +713,6 @@ class TestDecorateLlmRequest:
assert req.prompt == "Hello"
class TestModalitiesFix:
"""Tests for _modalities_fix function."""
def test_modalities_fix_image_not_supported(self, mock_provider):
"""Test modality fix when image is not supported."""
module = ama
mock_provider.provider_config = {"modalities": ["text"]}
req = ProviderRequest(prompt="Hello", image_urls=["/path/to/image.jpg"])
module._modalities_fix(mock_provider, req)
assert "[Image]" in req.prompt
assert req.image_urls == []
def test_modalities_fix_tool_not_supported(self, mock_provider):
"""Test modality fix when tool is not supported."""
module = ama
mock_provider.provider_config = {"modalities": ["text", "image"]}
req = ProviderRequest(prompt="Hello")
req.func_tool = ToolSet()
req.func_tool.add_tool(
FunctionTool(
name="dummy_tool",
description="dummy",
parameters={"type": "object", "properties": {}},
)
)
module._modalities_fix(mock_provider, req)
assert req.func_tool is None
def test_modalities_fix_all_supported(self, mock_provider):
"""Test modality fix when all features are supported."""
module = ama
mock_provider.provider_config = {"modalities": ["image", "tool_use"]}
tool_set = ToolSet()
tool_set.add_tool(
FunctionTool(
name="dummy_tool",
description="dummy",
parameters={"type": "object", "properties": {}},
)
)
req = ProviderRequest(
prompt="Hello",
image_urls=["/path/to/image.jpg"],
func_tool=tool_set,
)
module._modalities_fix(mock_provider, req)
assert req.prompt == "Hello"
assert len(req.image_urls) == 1
assert req.func_tool is not None
class TestSanitizeContextByModalities:
"""Tests for _sanitize_context_by_modalities function."""
def test_sanitize_no_op(self, mock_provider):
"""Test sanitize when disabled or modalities support everything."""
module = ama
config = module.MainAgentBuildConfig(
tool_call_timeout=60, sanitize_context_by_modalities=False
)
mock_provider.provider_config = {"modalities": ["image", "tool_use"]}
req = ProviderRequest(contexts=[{"role": "user", "content": "Hello"}])
module._sanitize_context_by_modalities(config, mock_provider, req)
assert len(req.contexts) == 1
def test_sanitize_removes_tool_messages(self, mock_provider):
"""Test sanitize removes tool messages when tool_use not supported."""
module = ama
config = module.MainAgentBuildConfig(
tool_call_timeout=60, sanitize_context_by_modalities=True
)
mock_provider.provider_config = {"modalities": ["image"]}
req = ProviderRequest(
contexts=[
{"role": "user", "content": "Hello"},
{"role": "tool", "content": "Tool result"},
]
)
module._sanitize_context_by_modalities(config, mock_provider, req)
assert len(req.contexts) == 1
assert req.contexts[0]["role"] == "user"
def test_sanitize_removes_tool_calls(self, mock_provider):
"""Test sanitize removes tool_calls from assistant messages."""
module = ama
config = module.MainAgentBuildConfig(
tool_call_timeout=60, sanitize_context_by_modalities=True
)
mock_provider.provider_config = {"modalities": ["image"]}
req = ProviderRequest(
contexts=[
{
"role": "assistant",
"content": "Response",
"tool_calls": [{"name": "tool"}],
}
]
)
module._sanitize_context_by_modalities(config, mock_provider, req)
assert "tool_calls" not in req.contexts[0]
def test_sanitize_removes_image_blocks(self, mock_provider):
"""Test sanitize removes image blocks when image not supported."""
module = ama
config = module.MainAgentBuildConfig(
tool_call_timeout=60, sanitize_context_by_modalities=True
)
mock_provider.provider_config = {"modalities": ["tool_use"]}
req = ProviderRequest(
contexts=[
{
"role": "user",
"content": [
{"type": "text", "text": "Hello"},
{"type": "image_url", "url": "image.jpg"},
],
}
]
)
module._sanitize_context_by_modalities(config, mock_provider, req)
assert len(req.contexts[0]["content"]) == 1
assert req.contexts[0]["content"][0]["type"] == "text"
class TestPluginToolFix:
"""Tests for _plugin_tool_fix function."""