mirror of
https://github.com/AstrBotDevs/AstrBot
synced 2026-07-15 17:30:13 +08:00
feat: enhance document upload process with batch settings and improved chunk handling
This commit is contained in:
@@ -24,6 +24,21 @@ class BaseVecDB:
|
||||
"""
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
async def insert_batch(
|
||||
self,
|
||||
contents: list[str],
|
||||
metadatas: list[dict] | None = None,
|
||||
ids: list[str] | None = None,
|
||||
batch_size: int = 32,
|
||||
tasks_limit: int = 3,
|
||||
max_retries: int = 3,
|
||||
) -> int:
|
||||
"""
|
||||
批量插入文本和其对应向量,自动生成 ID 并保持一致性。
|
||||
"""
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
async def retrieve(
|
||||
self,
|
||||
|
||||
@@ -36,6 +36,22 @@ class EmbeddingStorage:
|
||||
self.index.add_with_ids(vector.reshape(1, -1), np.array([id]))
|
||||
await self.save_index()
|
||||
|
||||
async def insert_batch(self, vectors: np.ndarray, ids: list[int]):
|
||||
"""批量插入向量
|
||||
|
||||
Args:
|
||||
vectors (np.ndarray): 要插入的向量数组
|
||||
ids (list[int]): 向量的ID列表
|
||||
Raises:
|
||||
ValueError: 如果向量的维度与存储的维度不匹配
|
||||
"""
|
||||
assert self.index is not None, "FAISS index is not initialized."
|
||||
if vectors.shape[1] != self.dimension:
|
||||
raise ValueError(
|
||||
f"向量维度不匹配, 期望: {self.dimension}, 实际: {vectors.shape[1]}"
|
||||
)
|
||||
self.index.add_with_ids(vectors, np.array(ids))
|
||||
|
||||
async def search(self, vector: np.ndarray, k: int) -> tuple:
|
||||
"""搜索最相似的向量
|
||||
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import numpy as np
|
||||
from .document_storage import DocumentStorage
|
||||
from .embedding_storage import EmbeddingStorage
|
||||
from ..base import Result, BaseVecDB
|
||||
from astrbot.core.provider.provider import EmbeddingProvider
|
||||
from astrbot.core.provider.provider import RerankProvider
|
||||
from astrbot import logger
|
||||
|
||||
|
||||
class FaissVecDB(BaseVecDB):
|
||||
@@ -60,6 +62,55 @@ class FaissVecDB(BaseVecDB):
|
||||
await self.embedding_storage.insert(vector, int_id)
|
||||
return int_id
|
||||
|
||||
async def insert_batch(
|
||||
self,
|
||||
contents: list[str],
|
||||
metadatas: list[dict] | None = None,
|
||||
ids: list[str] | None = None,
|
||||
batch_size: int = 32,
|
||||
tasks_limit: int = 3,
|
||||
max_retries: int = 3,
|
||||
) -> list[int]:
|
||||
"""
|
||||
批量插入文本和其对应向量,自动生成 ID 并保持一致性。
|
||||
"""
|
||||
assert self.document_storage.connection is not None, (
|
||||
"Database connection is not initialized."
|
||||
)
|
||||
metadatas = metadatas or [{} for _ in contents]
|
||||
ids = ids or [str(uuid.uuid4()) for _ in contents]
|
||||
|
||||
start = time.time()
|
||||
logger.debug(f"Generating embeddings for {len(contents)} contents...")
|
||||
vectors = await self.embedding_provider.get_embeddings_batch(
|
||||
contents,
|
||||
batch_size=batch_size,
|
||||
tasks_limit=tasks_limit,
|
||||
max_retries=max_retries,
|
||||
)
|
||||
end = time.time()
|
||||
logger.debug(
|
||||
f"Generated embeddings for {len(contents)} contents in {end - start:.2f} seconds."
|
||||
)
|
||||
|
||||
int_ids = []
|
||||
async with self.document_storage.connection.cursor() as cursor:
|
||||
for str_id, content, metadata in zip(ids, contents, metadatas):
|
||||
await cursor.execute(
|
||||
"INSERT INTO documents (doc_id, text, metadata) VALUES (?, ?, ?)",
|
||||
(str_id, content, json.dumps(metadata)),
|
||||
)
|
||||
await self.document_storage.connection.commit()
|
||||
|
||||
for str_id in ids:
|
||||
result = await self.document_storage.get_document_by_doc_id(str_id)
|
||||
int_ids.append(result["id"])
|
||||
|
||||
# 批量插入向量到 FAISS
|
||||
vectors_array = np.array(vectors).astype("float32")
|
||||
await self.embedding_storage.insert_batch(vectors_array, int_ids)
|
||||
return int_ids
|
||||
|
||||
async def retrieve(
|
||||
self,
|
||||
query: str,
|
||||
|
||||
@@ -13,7 +13,7 @@ class BaseChunker(ABC):
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def chunk(self, text: str) -> list[str]:
|
||||
async def chunk(self, text: str, **kwargs) -> list[str]:
|
||||
"""将文本分块
|
||||
|
||||
Args:
|
||||
|
||||
@@ -22,7 +22,7 @@ class FixedSizeChunker(BaseChunker):
|
||||
self.chunk_size = chunk_size
|
||||
self.chunk_overlap = chunk_overlap
|
||||
|
||||
async def chunk(self, text: str) -> list[str]:
|
||||
async def chunk(self, text: str, **kwargs) -> list[str]:
|
||||
"""固定大小分块
|
||||
|
||||
Args:
|
||||
@@ -31,22 +31,25 @@ class FixedSizeChunker(BaseChunker):
|
||||
Returns:
|
||||
list[str]: 分块后的文本列表
|
||||
"""
|
||||
chunk_size = kwargs.get("chunk_size", self.chunk_size)
|
||||
chunk_overlap = kwargs.get("chunk_overlap", self.chunk_overlap)
|
||||
|
||||
chunks = []
|
||||
start = 0
|
||||
text_len = len(text)
|
||||
|
||||
while start < text_len:
|
||||
end = start + self.chunk_size
|
||||
end = start + chunk_size
|
||||
chunk = text[start:end]
|
||||
|
||||
if chunk:
|
||||
chunks.append(chunk)
|
||||
|
||||
# 移动窗口,保留重叠部分
|
||||
start = end - self.chunk_overlap
|
||||
start = end - chunk_overlap
|
||||
|
||||
# 防止无限循环: 如果重叠过大,直接移到end
|
||||
if start >= end or self.chunk_overlap >= self.chunk_size:
|
||||
if start >= end or chunk_overlap >= chunk_size:
|
||||
start = end
|
||||
|
||||
return chunks
|
||||
|
||||
@@ -101,13 +101,18 @@ class KBHelper:
|
||||
file_name: str,
|
||||
file_content: bytes,
|
||||
file_type: str,
|
||||
chunk_size: int = 512,
|
||||
chunk_overlap: int = 50,
|
||||
batch_size: int = 32,
|
||||
tasks_limit: int = 3,
|
||||
max_retries: int = 3,
|
||||
) -> KBDocument:
|
||||
"""上传并处理文档(带原子性保证和失败清理)
|
||||
|
||||
流程:
|
||||
1. 保存原始文件
|
||||
2. 解析文档内容
|
||||
3. 提取多媒体资源 (TODO)
|
||||
3. 提取多媒体资源
|
||||
4. 分块处理
|
||||
5. 生成向量并存储
|
||||
6. 保存元数据(事务)
|
||||
@@ -116,7 +121,6 @@ class KBHelper:
|
||||
await self._ensure_vec_db()
|
||||
doc_id = str(uuid.uuid4())
|
||||
media_paths: list[Path] = []
|
||||
vec_doc_ids = []
|
||||
|
||||
# file_path = self.kb_files_dir / f"{doc_id}.{file_type}"
|
||||
# async with aiofiles.open(file_path, "wb") as f:
|
||||
@@ -144,18 +148,27 @@ class KBHelper:
|
||||
media_paths.append(Path(media.file_path))
|
||||
|
||||
# 分块并生成向量
|
||||
# saved_chunks = []
|
||||
chunks_text = await self.chunker.chunk(text_content)
|
||||
chunks_text = await self.chunker.chunk(
|
||||
text_content, chunk_size=chunk_size, chunk_overlap=chunk_overlap
|
||||
)
|
||||
contents = []
|
||||
metadatas = []
|
||||
for idx, chunk_text in enumerate(chunks_text):
|
||||
vec_doc_id = await self.vec_db.insert(
|
||||
content=chunk_text,
|
||||
metadata={
|
||||
contents.append(chunk_text)
|
||||
metadatas.append(
|
||||
{
|
||||
"kb_id": self.kb.kb_id,
|
||||
"doc_id": doc_id,
|
||||
"chunk_index": idx,
|
||||
},
|
||||
}
|
||||
)
|
||||
vec_doc_ids.append(str(vec_doc_id))
|
||||
await self.vec_db.insert_batch(
|
||||
contents=contents,
|
||||
metadatas=metadatas,
|
||||
batch_size=batch_size,
|
||||
tasks_limit=tasks_limit,
|
||||
max_retries=max_retries,
|
||||
)
|
||||
|
||||
# 保存文档的元数据
|
||||
doc = KBDocument(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import abc
|
||||
import asyncio
|
||||
from typing import List
|
||||
from typing import AsyncGenerator
|
||||
from astrbot.core.agent.tool import ToolSet
|
||||
@@ -203,6 +204,64 @@ class EmbeddingProvider(AbstractProvider):
|
||||
"""获取向量的维度"""
|
||||
...
|
||||
|
||||
async def get_embeddings_batch(
|
||||
self,
|
||||
texts: list[str],
|
||||
batch_size: int = 16,
|
||||
tasks_limit: int = 3,
|
||||
max_retries: int = 3,
|
||||
) -> list[list[float]]:
|
||||
"""批量获取文本的向量,分批处理以节省内存
|
||||
|
||||
Args:
|
||||
texts: 文本列表
|
||||
batch_size: 每批处理的文本数量
|
||||
tasks_limit: 并发任务数量限制
|
||||
max_retries: 失败时的最大重试次数
|
||||
|
||||
Returns:
|
||||
向量列表
|
||||
"""
|
||||
semaphore = asyncio.Semaphore(tasks_limit)
|
||||
all_embeddings: list[list[float]] = []
|
||||
failed_batches: list[tuple[int, list[str]]] = []
|
||||
|
||||
async def process_batch(batch_idx: int, batch_texts: list[str]):
|
||||
async with semaphore:
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
batch_embeddings = await self.get_embeddings(batch_texts)
|
||||
all_embeddings.extend(batch_embeddings)
|
||||
return
|
||||
except Exception as e:
|
||||
if attempt == max_retries - 1:
|
||||
# 最后一次重试失败,记录失败的批次
|
||||
failed_batches.append((batch_idx, batch_texts))
|
||||
raise Exception(
|
||||
f"批次 {batch_idx} 处理失败,已重试 {max_retries} 次: {str(e)}"
|
||||
)
|
||||
# 等待一段时间后重试,使用指数退避
|
||||
await asyncio.sleep(2**attempt)
|
||||
|
||||
tasks = []
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch_texts = texts[i : i + batch_size]
|
||||
batch_idx = i // batch_size
|
||||
tasks.append(process_batch(batch_idx, batch_texts))
|
||||
|
||||
# 收集所有任务的结果,包括失败的任务
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# 检查是否有失败的任务
|
||||
errors = [r for r in results if isinstance(r, Exception)]
|
||||
if errors:
|
||||
error_msg = (
|
||||
f"有 {len(errors)} 个批次处理失败: {'; '.join(str(e) for e in errors)}"
|
||||
)
|
||||
raise Exception(error_msg)
|
||||
|
||||
return all_embeddings
|
||||
|
||||
|
||||
class RerankProvider(AbstractProvider):
|
||||
def __init__(self, provider_config: dict, provider_settings: dict) -> None:
|
||||
|
||||
@@ -428,17 +428,18 @@ class KnowledgeBaseRoute(Route):
|
||||
"""上传文档
|
||||
|
||||
支持两种方式:
|
||||
1. multipart/form-data 文件上传
|
||||
2. JSON 格式 base64 编码上传
|
||||
1. multipart/form-data 文件上传(支持多文件,最多10个)
|
||||
2. JSON 格式 base64 编码上传(支持多文件,最多10个)
|
||||
|
||||
Form Data (multipart/form-data):
|
||||
- kb_id: 知识库 ID (必填)
|
||||
- file: 文件对象 (必填)
|
||||
- file: 文件对象 (必填,可多个,字段名为 file, file1, file2, ... 或 files[])
|
||||
|
||||
JSON Body (application/json):
|
||||
- kb_id: 知识库 ID (必填)
|
||||
- file_name: 文件名 (必填)
|
||||
- file_content: base64 编码的文件内容 (必填)
|
||||
- files: 文件数组 (必填)
|
||||
- file_name: 文件名 (必填)
|
||||
- file_content: base64 编码的文件内容 (必填)
|
||||
"""
|
||||
try:
|
||||
kb_manager = self._get_kb_manager()
|
||||
@@ -446,20 +447,46 @@ class KnowledgeBaseRoute(Route):
|
||||
# 检查 Content-Type
|
||||
content_type = request.content_type
|
||||
kb_id = None
|
||||
chunk_size = None
|
||||
chunk_overlap = None
|
||||
batch_size = 32
|
||||
tasks_limit = 3
|
||||
max_retries = 3
|
||||
files_to_upload = [] # 存储待上传的文件信息列表
|
||||
|
||||
if content_type and "multipart/form-data" in content_type:
|
||||
# 方式 1: multipart/form-data
|
||||
form_data = await request.form
|
||||
files = await request.files
|
||||
if content_type and "multipart/form-data" not in content_type:
|
||||
return (
|
||||
Response().error("Content-Type 须为 multipart/form-data").__dict__
|
||||
)
|
||||
form_data = await request.form
|
||||
files = await request.files
|
||||
|
||||
kb_id = form_data.get("kb_id")
|
||||
if not kb_id:
|
||||
return Response().error("缺少参数 kb_id").__dict__
|
||||
kb_id = form_data.get("kb_id")
|
||||
chunk_size = int(form_data.get("chunk_size", 512))
|
||||
chunk_overlap = int(form_data.get("chunk_overlap", 50))
|
||||
batch_size = int(form_data.get("batch_size", 32))
|
||||
tasks_limit = int(form_data.get("tasks_limit", 3))
|
||||
max_retries = int(form_data.get("max_retries", 3))
|
||||
if not kb_id:
|
||||
return Response().error("缺少参数 kb_id").__dict__
|
||||
|
||||
if "file" not in files:
|
||||
return Response().error("缺少文件").__dict__
|
||||
# 收集所有文件
|
||||
file_list = []
|
||||
# 支持 file, file1, file2, ... 或 files[] 格式
|
||||
for key in files.keys():
|
||||
if key == "file" or key.startswith("file") or key == "files[]":
|
||||
file_items = files.getlist(key)
|
||||
file_list.extend(file_items)
|
||||
|
||||
file = files["file"]
|
||||
if not file_list:
|
||||
return Response().error("缺少文件").__dict__
|
||||
|
||||
# 限制文件数量
|
||||
if len(file_list) > 10:
|
||||
return Response().error("最多只能上传10个文件").__dict__
|
||||
|
||||
# 处理每个文件
|
||||
for file in file_list:
|
||||
file_name = file.filename
|
||||
|
||||
# 保存到临时文件
|
||||
@@ -470,64 +497,83 @@ class KnowledgeBaseRoute(Route):
|
||||
# 异步读取文件内容
|
||||
async with aiofiles.open(temp_file_path, "rb") as f:
|
||||
file_content = await f.read()
|
||||
|
||||
# 提取文件类型
|
||||
file_type = (
|
||||
file_name.rsplit(".", 1)[-1].lower() if "." in file_name else ""
|
||||
)
|
||||
|
||||
files_to_upload.append(
|
||||
{
|
||||
"file_name": file_name,
|
||||
"file_content": file_content,
|
||||
"file_type": file_type,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
# 清理临时文件
|
||||
if os.path.exists(temp_file_path):
|
||||
os.remove(temp_file_path)
|
||||
|
||||
else:
|
||||
# 方式 2: JSON base64
|
||||
import base64
|
||||
|
||||
data = await request.json
|
||||
|
||||
kb_id = data.get("kb_id")
|
||||
file_name = data.get("file_name")
|
||||
file_content_b64 = data.get("file_content")
|
||||
|
||||
if not kb_id or not file_name or not file_content_b64:
|
||||
return (
|
||||
Response()
|
||||
.error("缺少参数 kb_id, file_name 或 file_content")
|
||||
.__dict__
|
||||
)
|
||||
|
||||
try:
|
||||
file_content = base64.b64decode(file_content_b64)
|
||||
except Exception:
|
||||
return (
|
||||
Response()
|
||||
.error("file_content 必须是有效的 base64 编码")
|
||||
.__dict__
|
||||
)
|
||||
|
||||
# 提取文件类型
|
||||
file_type = file_name.rsplit(".", 1)[-1].lower() if "." in file_name else ""
|
||||
|
||||
# 获取知识库
|
||||
kb_helper = await kb_manager.get_kb(kb_id)
|
||||
if not kb_helper:
|
||||
return Response().error("知识库不存在").__dict__
|
||||
|
||||
# 上传文档
|
||||
doc = await kb_helper.upload_document(
|
||||
file_name=file_name,
|
||||
file_content=file_content,
|
||||
file_type=file_type,
|
||||
)
|
||||
# 上传所有文档
|
||||
uploaded_docs = []
|
||||
failed_docs = []
|
||||
|
||||
doc_dict = {
|
||||
"doc_id": doc.doc_id,
|
||||
"kb_id": doc.kb_id,
|
||||
"doc_name": doc.doc_name,
|
||||
"file_type": doc.file_type,
|
||||
"file_size": doc.file_size,
|
||||
"chunk_count": doc.chunk_count,
|
||||
"media_count": doc.media_count,
|
||||
"created_at": doc.created_at.isoformat(),
|
||||
"updated_at": doc.updated_at.isoformat(),
|
||||
for file_info in files_to_upload:
|
||||
try:
|
||||
doc = await kb_helper.upload_document(
|
||||
file_name=file_info["file_name"],
|
||||
file_content=file_info["file_content"],
|
||||
file_type=file_info["file_type"],
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
batch_size=batch_size,
|
||||
tasks_limit=tasks_limit,
|
||||
max_retries=max_retries,
|
||||
)
|
||||
|
||||
doc_dict = {
|
||||
"doc_id": doc.doc_id,
|
||||
"kb_id": doc.kb_id,
|
||||
"doc_name": doc.doc_name,
|
||||
"file_type": doc.file_type,
|
||||
"file_size": doc.file_size,
|
||||
"chunk_count": doc.chunk_count,
|
||||
"media_count": doc.media_count,
|
||||
"created_at": doc.created_at.isoformat(),
|
||||
"updated_at": doc.updated_at.isoformat(),
|
||||
}
|
||||
uploaded_docs.append(doc_dict)
|
||||
except Exception as e:
|
||||
logger.error(f"上传文档 {file_info['file_name']} 失败: {e}")
|
||||
failed_docs.append(
|
||||
{"file_name": file_info["file_name"], "error": str(e)}
|
||||
)
|
||||
|
||||
# 返回结果
|
||||
result = {
|
||||
"uploaded": uploaded_docs,
|
||||
"failed": failed_docs,
|
||||
"total": len(files_to_upload),
|
||||
"success_count": len(uploaded_docs),
|
||||
"failed_count": len(failed_docs),
|
||||
}
|
||||
|
||||
return Response().ok(doc_dict, "上传文档成功").__dict__
|
||||
if failed_docs:
|
||||
message = (
|
||||
f"部分文档上传成功 ({len(uploaded_docs)}/{len(files_to_upload)})"
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
f"所有文档上传成功 ({len(uploaded_docs)}/{len(files_to_upload)})"
|
||||
)
|
||||
|
||||
return Response().ok(result, message).__dict__
|
||||
|
||||
except ValueError as e:
|
||||
return Response().error(str(e)).__dict__
|
||||
|
||||
@@ -46,12 +46,19 @@
|
||||
"selectFile": "Select File",
|
||||
"dropzone": "Drop files here or click to select",
|
||||
"supportedFormats": "Supported formats: TXT, PDF, Markdown",
|
||||
"maxSize": "Max file size: 50MB",
|
||||
"maxSize": "Max file size: 128MB",
|
||||
"chunkSettings": "Chunk Settings",
|
||||
"batchSettings": "Batch Settings",
|
||||
"chunkSize": "Chunk Size",
|
||||
"chunkSizeHint": "Number of characters per chunk (default: 512)",
|
||||
"chunkOverlap": "Chunk Overlap",
|
||||
"chunkOverlapHint": "Overlapping characters between chunks (default: 50)",
|
||||
"batchSize": "Batch Size",
|
||||
"batchSizeHint": "Number of chunks to process in each batch (default: 32)",
|
||||
"tasksLimit": "Concurrent Tasks Limit",
|
||||
"tasksLimitHint": "Maximum number of concurrent upload tasks (default: 3)",
|
||||
"maxRetries": "Max Retries",
|
||||
"maxRetriesHint": "Number of times to retry a failed upload task (default: 3)",
|
||||
"cancel": "Cancel",
|
||||
"submit": "Upload",
|
||||
"fileRequired": "Please select a file to upload"
|
||||
|
||||
@@ -47,12 +47,19 @@
|
||||
"selectFile": "选择文件",
|
||||
"dropzone": "拖放文件到这里或点击选择",
|
||||
"supportedFormats": "支持的格式: TXT, PDF, Markdown",
|
||||
"maxSize": "最大文件大小: 50MB",
|
||||
"maxSize": "最大文件大小: 128MB",
|
||||
"chunkSettings": "分块设置",
|
||||
"batchSettings": "批处理设置",
|
||||
"chunkSize": "分块大小",
|
||||
"chunkSizeHint": "每个文本块的字符数 (默认: 512)",
|
||||
"chunkOverlap": "分块重叠",
|
||||
"chunkOverlapHint": "相邻文本块之间的重叠字符数 (默认: 50)",
|
||||
"batchSize": "批处理大小",
|
||||
"batchSizeHint": "每批处理的文本块数量 (默认: 32)",
|
||||
"tasksLimit": "并发任务限制",
|
||||
"tasksLimitHint": "最大并发上传任务数 (默认: 3)",
|
||||
"maxRetries": "最大重试次数",
|
||||
"maxRetriesHint": "上传失败任务的重试次数 (默认: 3)",
|
||||
"cancel": "取消",
|
||||
"submit": "上传",
|
||||
"fileRequired": "请选择要上传的文件"
|
||||
|
||||
@@ -2,35 +2,16 @@
|
||||
<div class="documents-tab">
|
||||
<!-- 操作栏 -->
|
||||
<div class="action-bar mb-4">
|
||||
<v-btn
|
||||
prepend-icon="mdi-upload"
|
||||
color="primary"
|
||||
variant="elevated"
|
||||
@click="showUploadDialog = true"
|
||||
>
|
||||
<v-btn prepend-icon="mdi-upload" color="primary" variant="elevated" @click="showUploadDialog = true">
|
||||
{{ t('documents.upload') }}
|
||||
</v-btn>
|
||||
<v-text-field
|
||||
v-model="searchQuery"
|
||||
prepend-inner-icon="mdi-magnify"
|
||||
:placeholder="'搜索文档...'"
|
||||
variant="outlined"
|
||||
density="compact"
|
||||
hide-details
|
||||
clearable
|
||||
style="max-width: 300px"
|
||||
/>
|
||||
<v-text-field v-model="searchQuery" prepend-inner-icon="mdi-magnify" :placeholder="'搜索文档...'" variant="outlined"
|
||||
density="compact" hide-details clearable style="max-width: 300px" />
|
||||
</div>
|
||||
|
||||
<!-- 文档列表 -->
|
||||
<v-card elevation="2">
|
||||
<v-data-table
|
||||
:headers="headers"
|
||||
:items="documents"
|
||||
:loading="loading"
|
||||
:search="searchQuery"
|
||||
:items-per-page="10"
|
||||
>
|
||||
<v-data-table :headers="headers" :items="documents" :loading="loading" :search="searchQuery" :items-per-page="10">
|
||||
<template #item.doc_name="{ item }">
|
||||
<div class="d-flex align-center gap-2">
|
||||
<v-icon :color="getFileColor(item.file_type)">
|
||||
@@ -49,20 +30,8 @@
|
||||
</template>
|
||||
|
||||
<template #item.actions="{ item }">
|
||||
<v-btn
|
||||
icon="mdi-eye"
|
||||
variant="text"
|
||||
size="small"
|
||||
color="info"
|
||||
@click="viewDocument(item)"
|
||||
/>
|
||||
<v-btn
|
||||
icon="mdi-delete"
|
||||
variant="text"
|
||||
size="small"
|
||||
color="error"
|
||||
@click="confirmDelete(item)"
|
||||
/>
|
||||
<v-btn icon="mdi-eye" variant="text" size="small" color="info" @click="viewDocument(item)" />
|
||||
<v-btn icon="mdi-delete" variant="text" size="small" color="error" @click="confirmDelete(item)" />
|
||||
</template>
|
||||
|
||||
<template #no-data>
|
||||
@@ -75,9 +44,9 @@
|
||||
</v-card>
|
||||
|
||||
<!-- 上传对话框 -->
|
||||
<v-dialog v-model="showUploadDialog" max-width="600px" persistent>
|
||||
<v-dialog v-model="showUploadDialog" max-width="600px" persistent @after-enter="initUploadSettings">
|
||||
<v-card>
|
||||
<v-card-title class="pa-4">
|
||||
<v-card-title class="pa-4 d-flex align-center">
|
||||
<span class="text-h5">{{ t('upload.title') }}</span>
|
||||
<v-spacer />
|
||||
<v-btn icon="mdi-close" variant="text" @click="closeUploadDialog" />
|
||||
@@ -87,68 +56,74 @@
|
||||
|
||||
<v-card-text class="pa-6">
|
||||
<!-- 文件选择 -->
|
||||
<div
|
||||
class="upload-dropzone"
|
||||
:class="{ 'dragover': isDragging }"
|
||||
@drop.prevent="handleDrop"
|
||||
@dragover.prevent="isDragging = true"
|
||||
@dragleave="isDragging = false"
|
||||
@click="$refs.fileInput.click()"
|
||||
>
|
||||
<div class="upload-dropzone" :class="{ 'dragover': isDragging }" @drop.prevent="handleDrop"
|
||||
@dragover.prevent="isDragging = true" @dragleave="isDragging = false" @click="fileInput?.click()">
|
||||
<v-icon size="64" color="primary">mdi-cloud-upload</v-icon>
|
||||
<p class="mt-4 text-h6">{{ t('upload.dropzone') }}</p>
|
||||
<p class="text-caption text-medium-emphasis mt-2">{{ t('upload.supportedFormats') }}</p>
|
||||
<p class="text-caption text-medium-emphasis">{{ t('upload.maxSize') }}</p>
|
||||
<input
|
||||
ref="fileInput"
|
||||
type="file"
|
||||
hidden
|
||||
accept=".txt,.md,.pdf"
|
||||
@change="handleFileSelect"
|
||||
/>
|
||||
<p class="text-caption text-medium-emphasis">最多可上传 10 个文件</p>
|
||||
<input ref="fileInput" type="file" multiple hidden accept=".txt,.md,.pdf" @change="handleFileSelect" />
|
||||
</div>
|
||||
|
||||
<div v-if="selectedFile" class="mt-4 pa-4 rounded bg-surface-variant">
|
||||
<div class="d-flex align-center justify-space-between">
|
||||
<div class="d-flex align-center gap-2">
|
||||
<v-icon>{{ getFileIcon(selectedFile.name) }}</v-icon>
|
||||
<div>
|
||||
<div class="font-weight-medium">{{ selectedFile.name }}</div>
|
||||
<div class="text-caption">{{ formatFileSize(selectedFile.size) }}</div>
|
||||
<div v-if="selectedFiles.length > 0" class="mt-4">
|
||||
<div class="d-flex align-center justify-space-between mb-2">
|
||||
<span class="text-subtitle-2">已选择 {{ selectedFiles.length }} 个文件</span>
|
||||
<v-btn variant="text" size="small" @click="selectedFiles = []">清空</v-btn>
|
||||
</div>
|
||||
<div class="files-list">
|
||||
<div v-for="(file, index) in selectedFiles" :key="index" class="file-item pa-3 mb-2 rounded bg-surface-variant">
|
||||
<div class="d-flex align-center justify-space-between">
|
||||
<div class="d-flex align-center gap-2">
|
||||
<v-icon>{{ getFileIcon(file.name) }}</v-icon>
|
||||
<div>
|
||||
<div class="font-weight-medium">{{ file.name }}</div>
|
||||
<div class="text-caption">{{ formatFileSize(file.size) }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<v-btn icon="mdi-close" variant="text" size="small" @click="removeFile(index)" />
|
||||
</div>
|
||||
</div>
|
||||
<v-btn icon="mdi-close" variant="text" size="small" @click="selectedFile = null" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 分块设置 -->
|
||||
<v-expansion-panels class="mt-4">
|
||||
<v-expansion-panel>
|
||||
<v-expansion-panel-title>
|
||||
<v-icon start>mdi-cog</v-icon>
|
||||
{{ t('upload.chunkSettings') }}
|
||||
</v-expansion-panel-title>
|
||||
<v-expansion-panel-text>
|
||||
<v-text-field
|
||||
v-model.number="uploadSettings.chunk_size"
|
||||
:label="t('upload.chunkSize')"
|
||||
:hint="t('upload.chunkSizeHint')"
|
||||
type="number"
|
||||
variant="outlined"
|
||||
density="compact"
|
||||
class="mb-2"
|
||||
/>
|
||||
<v-text-field
|
||||
v-model.number="uploadSettings.chunk_overlap"
|
||||
:label="t('upload.chunkOverlap')"
|
||||
:hint="t('upload.chunkOverlapHint')"
|
||||
type="number"
|
||||
variant="outlined"
|
||||
density="compact"
|
||||
/>
|
||||
</v-expansion-panel-text>
|
||||
</v-expansion-panel>
|
||||
</v-expansion-panels>
|
||||
<div class="mt-6">
|
||||
<div class="d-flex align-center mb-4">
|
||||
<h3 class="text-h6">{{ t('upload.chunkSettings') }}</h3>
|
||||
</div>
|
||||
<v-row>
|
||||
<v-col cols="12" sm="6">
|
||||
<v-text-field v-model.number="uploadSettings.chunk_size" :label="t('upload.chunkSize')"
|
||||
:hint="t('upload.chunkSizeHint')" persistent-hint type="number" variant="outlined"
|
||||
density="compact" :placeholder="props.kb?.chunk_size?.toString() || '512'" />
|
||||
</v-col>
|
||||
<v-col cols="12" sm="6">
|
||||
<v-text-field v-model.number="uploadSettings.chunk_overlap" :label="t('upload.chunkOverlap')"
|
||||
:hint="t('upload.chunkOverlapHint')" persistent-hint type="number" variant="outlined"
|
||||
density="compact" :placeholder="props.kb?.chunk_overlap?.toString() || '50'" />
|
||||
</v-col>
|
||||
</v-row>
|
||||
</div>
|
||||
|
||||
<div class="mt-2">
|
||||
<h3 class="text-h6 mb-4">{{ t('upload.batchSettings') }}</h3>
|
||||
<v-row>
|
||||
<v-col cols="12" sm="4">
|
||||
<v-text-field v-model.number="uploadSettings.batch_size" :label="t('upload.batchSize')"
|
||||
hint="每批处理的文本数量" persistent-hint type="number" variant="outlined" density="compact" />
|
||||
</v-col>
|
||||
<v-col cols="12" sm="4">
|
||||
<v-text-field v-model.number="uploadSettings.tasks_limit" :label="t('upload.tasksLimit')"
|
||||
hint="并发任务数量限制" persistent-hint type="number" variant="outlined" density="compact" />
|
||||
</v-col>
|
||||
<v-col cols="12" sm="4">
|
||||
<v-text-field v-model.number="uploadSettings.max_retries" :label="t('upload.maxRetries')"
|
||||
hint="失败时的最大重试次数" persistent-hint type="number" variant="outlined" density="compact" />
|
||||
</v-col>
|
||||
</v-row>
|
||||
</div>
|
||||
|
||||
</v-card-text>
|
||||
|
||||
<v-divider />
|
||||
@@ -158,13 +133,8 @@
|
||||
<v-btn variant="text" @click="closeUploadDialog">
|
||||
{{ t('upload.cancel') }}
|
||||
</v-btn>
|
||||
<v-btn
|
||||
color="primary"
|
||||
variant="elevated"
|
||||
@click="uploadDocument"
|
||||
:loading="uploading"
|
||||
:disabled="!selectedFile"
|
||||
>
|
||||
<v-btn color="primary" variant="elevated" @click="uploadDocument" :loading="uploading"
|
||||
:disabled="selectedFiles.length === 0">
|
||||
{{ t('upload.submit') }}
|
||||
</v-btn>
|
||||
</v-card-actions>
|
||||
@@ -224,9 +194,10 @@ const documents = ref<any[]>([])
|
||||
const searchQuery = ref('')
|
||||
const showUploadDialog = ref(false)
|
||||
const showDeleteDialog = ref(false)
|
||||
const selectedFile = ref<File | null>(null)
|
||||
const selectedFiles = ref<File[]>([])
|
||||
const deleteTarget = ref<any>(null)
|
||||
const isDragging = ref(false)
|
||||
const fileInput = ref<HTMLInputElement | null>(null)
|
||||
|
||||
const snackbar = ref({
|
||||
show: false,
|
||||
@@ -242,10 +213,24 @@ const showSnackbar = (text: string, color: string = 'success') => {
|
||||
|
||||
// 上传设置
|
||||
const uploadSettings = ref({
|
||||
chunk_size: null,
|
||||
chunk_overlap: null
|
||||
chunk_size: null as number | null,
|
||||
chunk_overlap: null as number | null,
|
||||
batch_size: 32,
|
||||
tasks_limit: 3,
|
||||
max_retries: 3
|
||||
})
|
||||
|
||||
// 初始化上传设置
|
||||
const initUploadSettings = () => {
|
||||
uploadSettings.value = {
|
||||
chunk_size: props.kb?.chunk_size || null,
|
||||
chunk_overlap: props.kb?.chunk_overlap || null,
|
||||
batch_size: 32,
|
||||
tasks_limit: 3,
|
||||
max_retries: 3
|
||||
}
|
||||
}
|
||||
|
||||
// 表格列
|
||||
const headers = [
|
||||
{ title: t('documents.name'), key: 'doc_name', sortable: true },
|
||||
@@ -253,7 +238,7 @@ const headers = [
|
||||
{ title: t('documents.size'), key: 'file_size', sortable: true },
|
||||
{ title: t('documents.chunks'), key: 'chunk_count', sortable: true },
|
||||
{ title: t('documents.createdAt'), key: 'created_at', sortable: true },
|
||||
{ title: t('documents.actions'), key: 'actions', sortable: false, align: 'end' }
|
||||
{ title: t('documents.actions'), key: 'actions', sortable: false, align: 'end' as const }
|
||||
]
|
||||
|
||||
// 加载文档列表
|
||||
@@ -277,22 +262,39 @@ const loadDocuments = async () => {
|
||||
// 文件选择
|
||||
const handleFileSelect = (event: Event) => {
|
||||
const target = event.target as HTMLInputElement
|
||||
if (target.files && target.files[0]) {
|
||||
selectedFile.value = target.files[0]
|
||||
if (target.files && target.files.length > 0) {
|
||||
const newFiles = Array.from(target.files)
|
||||
addFiles(newFiles)
|
||||
}
|
||||
}
|
||||
|
||||
// 添加文件(检查数量限制)
|
||||
const addFiles = (files: File[]) => {
|
||||
const totalFiles = selectedFiles.value.length + files.length
|
||||
if (totalFiles > 10) {
|
||||
showSnackbar('最多只能选择 10 个文件', 'warning')
|
||||
return
|
||||
}
|
||||
selectedFiles.value.push(...files)
|
||||
}
|
||||
|
||||
// 移除文件
|
||||
const removeFile = (index: number) => {
|
||||
selectedFiles.value.splice(index, 1)
|
||||
}
|
||||
|
||||
// 拖放上传
|
||||
const handleDrop = (event: DragEvent) => {
|
||||
isDragging.value = false
|
||||
if (event.dataTransfer?.files && event.dataTransfer.files[0]) {
|
||||
selectedFile.value = event.dataTransfer.files[0]
|
||||
if (event.dataTransfer?.files && event.dataTransfer.files.length > 0) {
|
||||
const newFiles = Array.from(event.dataTransfer.files)
|
||||
addFiles(newFiles)
|
||||
}
|
||||
}
|
||||
|
||||
// 上传文档
|
||||
const uploadDocument = async () => {
|
||||
if (!selectedFile.value) {
|
||||
if (selectedFiles.value.length === 0) {
|
||||
showSnackbar(t('upload.fileRequired'), 'warning')
|
||||
return
|
||||
}
|
||||
@@ -300,7 +302,12 @@ const uploadDocument = async () => {
|
||||
uploading.value = true
|
||||
try {
|
||||
const formData = new FormData()
|
||||
formData.append('file', selectedFile.value)
|
||||
|
||||
// 添加所有文件
|
||||
selectedFiles.value.forEach((file, index) => {
|
||||
formData.append(`file${index}`, file)
|
||||
})
|
||||
|
||||
formData.append('kb_id', props.kbId)
|
||||
if (uploadSettings.value.chunk_size) {
|
||||
formData.append('chunk_size', uploadSettings.value.chunk_size.toString())
|
||||
@@ -308,13 +315,25 @@ const uploadDocument = async () => {
|
||||
if (uploadSettings.value.chunk_overlap) {
|
||||
formData.append('chunk_overlap', uploadSettings.value.chunk_overlap.toString())
|
||||
}
|
||||
formData.append('batch_size', uploadSettings.value.batch_size.toString())
|
||||
formData.append('tasks_limit', uploadSettings.value.tasks_limit.toString())
|
||||
formData.append('max_retries', uploadSettings.value.max_retries.toString())
|
||||
|
||||
const response = await axios.post('/api/kb/document/upload', formData, {
|
||||
headers: { 'Content-Type': 'multipart/form-data' }
|
||||
})
|
||||
|
||||
if (response.data.status === 'ok') {
|
||||
showSnackbar(t('documents.uploadSuccess'))
|
||||
const result = response.data.data
|
||||
const successCount = result.success_count || 0
|
||||
const failedCount = result.failed_count || 0
|
||||
|
||||
if (failedCount === 0) {
|
||||
showSnackbar(`成功上传 ${successCount} 个文档`)
|
||||
} else {
|
||||
showSnackbar(`上传完成: ${successCount} 个成功, ${failedCount} 个失败`, 'warning')
|
||||
}
|
||||
|
||||
closeUploadDialog()
|
||||
await loadDocuments()
|
||||
emit('refresh')
|
||||
@@ -332,8 +351,9 @@ const uploadDocument = async () => {
|
||||
// 关闭上传对话框
|
||||
const closeUploadDialog = () => {
|
||||
showUploadDialog.value = false
|
||||
selectedFile.value = null
|
||||
uploadSettings.value = { chunk_size: null, chunk_overlap: null }
|
||||
selectedFiles.value = []
|
||||
// 重置为知识库默认设置
|
||||
initUploadSettings()
|
||||
}
|
||||
|
||||
// 查看文档
|
||||
@@ -428,8 +448,13 @@ onMounted(() => {
|
||||
}
|
||||
|
||||
@keyframes fadeIn {
|
||||
from { opacity: 0; }
|
||||
to { opacity: 1; }
|
||||
from {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.action-bar {
|
||||
@@ -457,13 +482,26 @@ onMounted(() => {
|
||||
transform: scale(1.02);
|
||||
}
|
||||
|
||||
.files-list {
|
||||
max-height: 300px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.file-item {
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.file-item:hover {
|
||||
background: rgba(var(--v-theme-surface-variant), 0.8) !important;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.action-bar {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
.action-bar > * {
|
||||
.action-bar>* {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user