完成请求推送,但组装mes还有问题
This commit is contained in:
@@ -177,8 +177,8 @@ class CharacterService:
|
||||
更新角色卡
|
||||
|
||||
Args:
|
||||
name: 角色名
|
||||
updates: 更新的字段
|
||||
name: 角色名(旧名称,用于定位文件夹)
|
||||
updates: 更新的字段(可以包含 name 字段来重命名)
|
||||
|
||||
Returns:
|
||||
更新后的 CharacterCard 对象
|
||||
@@ -193,6 +193,29 @@ class CharacterService:
|
||||
with open(char_file, 'r', encoding='utf-8') as f:
|
||||
existing_data = json.load(f)
|
||||
|
||||
# 检查是否需要重命名
|
||||
new_name = updates.get('name')
|
||||
needs_rename = new_name and new_name != name
|
||||
|
||||
if needs_rename:
|
||||
# 验证新名称是否合法
|
||||
if not new_name or new_name.strip() == '':
|
||||
raise ValueError("角色名不能为空")
|
||||
|
||||
# 检查新名称是否已存在
|
||||
new_folder = self.characters_dir / new_name
|
||||
if new_folder.exists():
|
||||
raise FileExistsError(f"角色 '{new_name}' 已存在")
|
||||
|
||||
# 重命名文件夹
|
||||
try:
|
||||
import shutil
|
||||
shutil.move(str(char_folder), str(new_folder))
|
||||
char_folder = new_folder
|
||||
char_file = char_folder / "character.json"
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"重命名文件夹失败: {str(e)}")
|
||||
|
||||
# 合并更新
|
||||
existing_data.update(updates)
|
||||
existing_data['updatedAt'] = int(datetime.now().timestamp())
|
||||
|
||||
@@ -57,7 +57,7 @@ class ChatService:
|
||||
logger.error(f"处理角色目录 {role_dir.name} 时出错: {str(e)}")
|
||||
continue
|
||||
|
||||
return {"chat": [{"role_name": role, **chat} for role, chats in result.items() for chat in chats]}
|
||||
return result
|
||||
|
||||
def _get_chat_summary(self, role_name: str, chat_name: str) -> Optional[Dict]:
|
||||
"""
|
||||
@@ -197,7 +197,8 @@ class ChatService:
|
||||
tags = []
|
||||
|
||||
try:
|
||||
character_file = Path("data/characters") / role_name / "character.json"
|
||||
from backend.core.config import settings
|
||||
character_file = settings.CHARACTERS_PATH / role_name / "character.json"
|
||||
if character_file.exists():
|
||||
with open(character_file, 'r', encoding='utf-8') as f:
|
||||
character_data = json.load(f)
|
||||
|
||||
1009
backend/services/chat_workflow_service.py
Normal file
1009
backend/services/chat_workflow_service.py
Normal file
File diff suppressed because it is too large
Load Diff
346
backend/services/image_metadata_service.py
Normal file
346
backend/services/image_metadata_service.py
Normal file
@@ -0,0 +1,346 @@
|
||||
"""
|
||||
图片元数据服务
|
||||
|
||||
负责管理生成图片的元数据,支持绑定到角色/聊天的特定楼层
|
||||
数据持久化到 data/image_metadata 目录
|
||||
"""
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional, Any
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
from backend.models.internal import ImageMetadata
|
||||
from backend.core.config import settings
|
||||
except ImportError:
|
||||
from models.internal import ImageMetadata
|
||||
from core.config import settings
|
||||
|
||||
|
||||
class ImageMetadataService:
|
||||
"""
|
||||
图片元数据服务
|
||||
|
||||
功能:
|
||||
- 记录生成图片的元数据
|
||||
- 按角色/聊天/楼层组织
|
||||
- 支持 swipe(同一楼层多张图片)
|
||||
- 提供画廊查询接口
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.metadata_dir = settings.DATA_PATH / "image_metadata"
|
||||
self.metadata_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 图片存储目录
|
||||
self.images_dir = settings.DATA_PATH / "images"
|
||||
self.images_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _get_chat_metadata_file(self, chat_id: str) -> Path:
|
||||
"""获取指定聊天的元数据文件路径"""
|
||||
# chat_id 格式: role_name/chat_name
|
||||
parts = chat_id.split("/")
|
||||
if len(parts) == 2:
|
||||
role_name, chat_name = parts
|
||||
role_dir = self.metadata_dir / role_name
|
||||
role_dir.mkdir(parents=True, exist_ok=True)
|
||||
return role_dir / f"{chat_name}.json"
|
||||
else:
|
||||
# fallback
|
||||
return self.metadata_dir / f"{chat_id.replace('/', '_')}.json"
|
||||
|
||||
def _load_chat_metadata(self, chat_id: str) -> List[ImageMetadata]:
|
||||
"""加载指定聊天的所有图片元数据"""
|
||||
file_path = self._get_chat_metadata_file(chat_id)
|
||||
|
||||
if not file_path.exists():
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
return [ImageMetadata(**item) for item in data]
|
||||
except Exception as e:
|
||||
print(f"[ImageMetadata] 加载元数据失败: {e}")
|
||||
return []
|
||||
|
||||
def _save_chat_metadata(self, chat_id: str, metadata_list: List[ImageMetadata]):
|
||||
"""保存聊天的所有图片元数据"""
|
||||
file_path = self._get_chat_metadata_file(chat_id)
|
||||
|
||||
try:
|
||||
data = [m.model_dump() for m in metadata_list]
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
except Exception as e:
|
||||
print(f"[ImageMetadata] 保存元数据失败: {e}")
|
||||
|
||||
async def add_image(
|
||||
self,
|
||||
chat_id: str,
|
||||
role_name: str,
|
||||
chat_name: str,
|
||||
floor: int,
|
||||
filename: str,
|
||||
filepath: str,
|
||||
prompt: Optional[str] = None,
|
||||
negative_prompt: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
model: Optional[str] = None,
|
||||
workflow_name: Optional[str] = None,
|
||||
task_id: Optional[str] = None,
|
||||
generation_time: Optional[float] = None,
|
||||
width: Optional[int] = None,
|
||||
height: Optional[int] = None,
|
||||
file_size: Optional[int] = None
|
||||
) -> ImageMetadata:
|
||||
"""
|
||||
添加图片元数据
|
||||
|
||||
Args:
|
||||
chat_id: 聊天ID
|
||||
role_name: 角色名称
|
||||
chat_name: 聊天名称
|
||||
floor: 楼层号
|
||||
filename: 文件名
|
||||
filepath: 文件相对路径
|
||||
prompt: 提示词
|
||||
negative_prompt: 负面提示词
|
||||
seed: 随机种子
|
||||
model: 使用的模型
|
||||
workflow_name: 工作流名称
|
||||
task_id: 任务ID
|
||||
generation_time: 生成耗时
|
||||
width: 图片宽度
|
||||
height: 图片高度
|
||||
file_size: 文件大小
|
||||
|
||||
Returns:
|
||||
ImageMetadata: 创建的元数据
|
||||
"""
|
||||
metadata_list = self._load_chat_metadata(chat_id)
|
||||
|
||||
# 计算 swipe_index
|
||||
same_floor_images = [m for m in metadata_list if m.floor == floor]
|
||||
swipe_index = len(same_floor_images)
|
||||
|
||||
# 如果这是该楼层的第一张图片,将其他图片的 isCurrentSwipe 设为 False
|
||||
if swipe_index == 0:
|
||||
for m in metadata_list:
|
||||
if m.floor == floor:
|
||||
m.isCurrentSwipe = False
|
||||
|
||||
metadata = ImageMetadata(
|
||||
id=str(uuid.uuid4()),
|
||||
chatId=chat_id,
|
||||
roleName=role_name,
|
||||
chatName=chat_name,
|
||||
floor=floor,
|
||||
filename=filename,
|
||||
filepath=filepath,
|
||||
prompt=prompt,
|
||||
negativePrompt=negative_prompt,
|
||||
seed=seed,
|
||||
model=model,
|
||||
workflowName=workflow_name,
|
||||
taskId=task_id,
|
||||
generationTime=generation_time,
|
||||
width=width,
|
||||
height=height,
|
||||
fileSize=file_size,
|
||||
swipeIndex=swipe_index,
|
||||
isCurrentSwipe=True
|
||||
)
|
||||
|
||||
metadata_list.append(metadata)
|
||||
self._save_chat_metadata(chat_id, metadata_list)
|
||||
|
||||
return metadata
|
||||
|
||||
async def get_images_by_chat(
|
||||
self,
|
||||
chat_id: str,
|
||||
floor: Optional[int] = None
|
||||
) -> List[ImageMetadata]:
|
||||
"""
|
||||
获取指定聊天的图片列表
|
||||
|
||||
Args:
|
||||
chat_id: 聊天ID
|
||||
floor: 楼层号(可选,用于过滤)
|
||||
|
||||
Returns:
|
||||
图片元数据列表
|
||||
"""
|
||||
metadata_list = self._load_chat_metadata(chat_id)
|
||||
|
||||
if floor is not None:
|
||||
metadata_list = [m for m in metadata_list if m.floor == floor]
|
||||
|
||||
# 按楼层和 swipe_index 排序
|
||||
metadata_list.sort(key=lambda m: (m.floor, m.swipeIndex))
|
||||
|
||||
return metadata_list
|
||||
|
||||
async def get_images_by_role(self, role_name: str) -> List[ImageMetadata]:
|
||||
"""获取指定角色的所有图片"""
|
||||
all_images = []
|
||||
|
||||
role_dir = self.metadata_dir / role_name
|
||||
if not role_dir.exists():
|
||||
return []
|
||||
|
||||
for chat_file in role_dir.glob("*.json"):
|
||||
chat_name = chat_file.stem
|
||||
chat_id = f"{role_name}/{chat_name}"
|
||||
images = self._load_chat_metadata(chat_id)
|
||||
all_images.extend(images)
|
||||
|
||||
# 按创建时间排序
|
||||
all_images.sort(key=lambda m: m.createdAt, reverse=True)
|
||||
|
||||
return all_images
|
||||
|
||||
async def delete_image(self, chat_id: str, image_id: str) -> bool:
|
||||
"""
|
||||
删除图片元数据(不删除实际文件)
|
||||
|
||||
Args:
|
||||
chat_id: 聊天ID
|
||||
image_id: 图片ID
|
||||
|
||||
Returns:
|
||||
bool: 是否成功删除
|
||||
"""
|
||||
metadata_list = self._load_chat_metadata(chat_id)
|
||||
|
||||
# 找到要删除的图片
|
||||
target_image = None
|
||||
for m in metadata_list:
|
||||
if m.id == image_id:
|
||||
target_image = m
|
||||
break
|
||||
|
||||
if not target_image:
|
||||
return False
|
||||
|
||||
floor = target_image.floor
|
||||
swipe_index = target_image.swipeIndex
|
||||
|
||||
# 删除该图片
|
||||
metadata_list = [m for m in metadata_list if m.id != image_id]
|
||||
|
||||
# 重新调整同一楼层其他图片的 swipe_index
|
||||
same_floor_images = [m for m in metadata_list if m.floor == floor]
|
||||
same_floor_images.sort(key=lambda m: m.swipeIndex)
|
||||
|
||||
for idx, m in enumerate(same_floor_images):
|
||||
m.swipeIndex = idx
|
||||
m.isCurrentSwipe = (idx == 0) # 第一个为当前显示
|
||||
|
||||
self._save_chat_metadata(chat_id, metadata_list)
|
||||
return True
|
||||
|
||||
async def clear_chat_images(self, chat_id: str) -> int:
|
||||
"""
|
||||
清空指定聊天的所有图片元数据
|
||||
|
||||
Args:
|
||||
chat_id: 聊天ID
|
||||
|
||||
Returns:
|
||||
int: 删除的图片数量
|
||||
"""
|
||||
metadata_list = self._load_chat_metadata(chat_id)
|
||||
count = len(metadata_list)
|
||||
|
||||
# 清空元数据文件
|
||||
file_path = self._get_chat_metadata_file(chat_id)
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
|
||||
return count
|
||||
|
||||
async def set_current_swipe(self, chat_id: str, image_id: str) -> bool:
|
||||
"""
|
||||
设置某张图片为当前显示的 swipe
|
||||
|
||||
Args:
|
||||
chat_id: 聊天ID
|
||||
image_id: 图片ID
|
||||
|
||||
Returns:
|
||||
bool: 是否成功设置
|
||||
"""
|
||||
metadata_list = self._load_chat_metadata(chat_id)
|
||||
|
||||
target_image = None
|
||||
for m in metadata_list:
|
||||
if m.id == image_id:
|
||||
target_image = m
|
||||
break
|
||||
|
||||
if not target_image:
|
||||
return False
|
||||
|
||||
floor = target_image.floor
|
||||
|
||||
# 将同一楼层的所有图片设为非当前
|
||||
for m in metadata_list:
|
||||
if m.floor == floor:
|
||||
m.isCurrentSwipe = False
|
||||
|
||||
# 设置目标图片为当前
|
||||
target_image.isCurrentSwipe = True
|
||||
|
||||
self._save_chat_metadata(chat_id, metadata_list)
|
||||
return True
|
||||
|
||||
async def get_gallery_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取画廊统计信息
|
||||
|
||||
Returns:
|
||||
统计信息字典
|
||||
"""
|
||||
stats = {
|
||||
"totalImages": 0,
|
||||
"byRole": {},
|
||||
"byChat": {}
|
||||
}
|
||||
|
||||
if not self.metadata_dir.exists():
|
||||
return stats
|
||||
|
||||
for role_dir in self.metadata_dir.iterdir():
|
||||
if not role_dir.is_dir():
|
||||
continue
|
||||
|
||||
role_name = role_dir.name
|
||||
role_count = 0
|
||||
|
||||
for chat_file in role_dir.glob("*.json"):
|
||||
chat_name = chat_file.stem
|
||||
chat_id = f"{role_name}/{chat_name}"
|
||||
images = self._load_chat_metadata(chat_id)
|
||||
|
||||
chat_count = len(images)
|
||||
role_count += chat_count
|
||||
stats["totalImages"] += chat_count
|
||||
|
||||
if chat_count > 0:
|
||||
stats["byChat"][chat_id] = chat_count
|
||||
|
||||
if role_count > 0:
|
||||
stats["byRole"][role_name] = role_count
|
||||
|
||||
return stats
|
||||
|
||||
def get_image_full_path(self, filepath: str) -> Path:
|
||||
"""获取图片的完整路径"""
|
||||
return self.images_dir / filepath
|
||||
|
||||
|
||||
# 全局实例
|
||||
image_metadata_service = ImageMetadataService()
|
||||
@@ -27,18 +27,35 @@ class LLMModelService:
|
||||
if not base_url:
|
||||
base_url = "https://api.openai.com/v1"
|
||||
|
||||
# 确保 base_url 以 /v1 结尾
|
||||
if not base_url.endswith('/v1'):
|
||||
base_url = base_url.rstrip('/') + '/v1'
|
||||
# 规范化 base_url:确保有协议前缀
|
||||
base_url = base_url.strip()
|
||||
if not base_url.startswith(('http://', 'https://')):
|
||||
base_url = 'https://' + base_url
|
||||
|
||||
response = requests.get(
|
||||
f"{base_url}/models",
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
timeout=10
|
||||
)
|
||||
# 移除末尾的斜杠和常见 endpoint 路径
|
||||
base_url = base_url.rstrip('/')
|
||||
# 移除可能已经存在的 endpoint 路径
|
||||
for endpoint in ['/chat/completions', '/completions', '/embeddings', '/models']:
|
||||
if base_url.endswith(endpoint):
|
||||
base_url = base_url[:-len(endpoint)]
|
||||
break
|
||||
|
||||
# 调用 models API
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{base_url}/models",
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
timeout=10
|
||||
)
|
||||
except requests.exceptions.InvalidSchema as e:
|
||||
raise Exception(f"URL 格式错误: {base_url}/models - 请确保 URL 以 http:// 或 https:// 开头")
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
raise Exception(f"无法连接到 API: {base_url}/models - 请检查网络连接和 API 地址")
|
||||
except requests.exceptions.Timeout as e:
|
||||
raise Exception(f"请求超时: {base_url}/models - 请检查网络连接")
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"HTTP {response.status_code}: {response.text}")
|
||||
@@ -46,14 +63,8 @@ class LLMModelService:
|
||||
data = response.json()
|
||||
models = [model['id'] for model in data.get('data', [])]
|
||||
|
||||
# 过滤出聊天模型(可选)
|
||||
chat_models = [
|
||||
m for m in models
|
||||
if any(keyword in m.lower() for keyword in ['gpt', 'chat'])
|
||||
]
|
||||
|
||||
# 如果没有找到聊天模型,返回所有模型
|
||||
return chat_models if chat_models else models
|
||||
# 返回所有模型,不做过滤
|
||||
return models
|
||||
|
||||
except Exception as e:
|
||||
raise Exception(f"获取 OpenAI 模型列表失败: {str(e)}")
|
||||
@@ -132,6 +143,9 @@ class LLMModelService:
|
||||
return 'anthropic'
|
||||
elif 'ollama' in api_url_lower or 'localhost:11434' in api_url_lower or '127.0.0.1:11434' in api_url_lower:
|
||||
return 'ollama'
|
||||
elif 'bigmodel' in api_url_lower or 'glm' in api_url_lower:
|
||||
# 智谱AI GLM - 兼容 OpenAI API
|
||||
return 'openai'
|
||||
elif 'siliconflow' in api_url_lower or 'silicon.cloud' in api_url_lower:
|
||||
# SiliconFlow 等兼容 OpenAI API 的服务
|
||||
return 'openai'
|
||||
|
||||
332
backend/services/regex_service.py
Normal file
332
backend/services/regex_service.py
Normal file
@@ -0,0 +1,332 @@
|
||||
"""
|
||||
正则规则服务(重构版 - 文件夹结构)
|
||||
|
||||
负责加载、管理和应用正则替换规则。
|
||||
使用文件夹结构组织规则,兼容 SillyTavern 格式。
|
||||
|
||||
文件结构:
|
||||
data/regex/
|
||||
├── global/ # 全局规则
|
||||
│ └── default.json
|
||||
├── characters/ # 角色卡绑定规则
|
||||
│ └── {characterName}.json
|
||||
└── presets/ # 预设绑定规则
|
||||
└── {presetName}.json
|
||||
"""
|
||||
import re
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Dict
|
||||
from uuid import uuid4
|
||||
|
||||
from core.config import settings
|
||||
from models.regex_rules import RegexRule, RegexRuleset, RegexScope, RegexPlacement, SubstituteMode
|
||||
from services.system_settings_service import system_settings_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RegexService:
|
||||
"""
|
||||
正则替换规则服务(文件夹结构版)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.regex_base_path = settings.DATA_PATH / "regex"
|
||||
self.global_path = self.regex_base_path / "global"
|
||||
self.characters_path = self.regex_base_path / "characters"
|
||||
self.presets_path = self.regex_base_path / "presets"
|
||||
|
||||
# 内存缓存
|
||||
self.global_rulesets: Dict[str, RegexRuleset] = {}
|
||||
self.character_rulesets: Dict[str, RegexRuleset] = {} # key: characterName
|
||||
self.preset_rulesets: Dict[str, RegexRuleset] = {} # key: presetName
|
||||
|
||||
self._ensure_directories()
|
||||
self._load_all_rules()
|
||||
|
||||
def _ensure_directories(self):
|
||||
"""确保目录结构存在"""
|
||||
for path in [self.regex_base_path, self.global_path, self.characters_path, self.presets_path]:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _load_all_rules(self):
|
||||
"""加载所有规则"""
|
||||
self._load_global_rules()
|
||||
self._load_character_rules()
|
||||
self._load_preset_rules()
|
||||
|
||||
def _load_global_rules(self):
|
||||
"""加载全局规则"""
|
||||
if not self.global_path.exists():
|
||||
return
|
||||
|
||||
for json_file in self.global_path.glob("*.json"):
|
||||
try:
|
||||
with open(json_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
if isinstance(data, list):
|
||||
# SillyTavern 格式
|
||||
ruleset = self._convert_sillytavern_format(data, json_file.stem)
|
||||
elif isinstance(data, dict) and 'rules' in data:
|
||||
# 我们的规则集格式
|
||||
ruleset = RegexRuleset(**data)
|
||||
else:
|
||||
logger.warning(f"未知的规则文件格式: {json_file}")
|
||||
continue
|
||||
|
||||
self.global_rulesets[ruleset.id] = ruleset
|
||||
logger.info(f"加载全局规则集: {ruleset.name} ({len(ruleset.rules)} 条规则)")
|
||||
except Exception as e:
|
||||
logger.error(f"加载全局规则失败 {json_file}: {e}")
|
||||
|
||||
def _load_character_rules(self):
|
||||
"""加载角色卡绑定规则"""
|
||||
if not self.characters_path.exists():
|
||||
return
|
||||
|
||||
for json_file in self.characters_path.glob("*.json"):
|
||||
try:
|
||||
character_name = json_file.stem
|
||||
with open(json_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
if isinstance(data, list):
|
||||
ruleset = self._convert_sillytavern_format(data, character_name, RegexScope.CHARACTER)
|
||||
elif isinstance(data, dict) and 'rules' in data:
|
||||
ruleset = RegexRuleset(**data)
|
||||
else:
|
||||
continue
|
||||
|
||||
# 确保所有规则的 scope 正确
|
||||
for rule in ruleset.rules:
|
||||
rule.scope = RegexScope.CHARACTER
|
||||
rule.characterName = character_name
|
||||
|
||||
self.character_rulesets[character_name] = ruleset
|
||||
logger.info(f"加载角色规则: {character_name} ({len(ruleset.rules)} 条规则)")
|
||||
except Exception as e:
|
||||
logger.error(f"加载角色规则失败 {json_file}: {e}")
|
||||
|
||||
def _load_preset_rules(self):
|
||||
"""加载预设绑定规则"""
|
||||
if not self.presets_path.exists():
|
||||
return
|
||||
|
||||
for json_file in self.presets_path.glob("*.json"):
|
||||
try:
|
||||
preset_name = json_file.stem
|
||||
with open(json_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
if isinstance(data, list):
|
||||
ruleset = self._convert_sillytavern_format(data, preset_name, RegexScope.PRESET)
|
||||
elif isinstance(data, dict) and 'rules' in data:
|
||||
ruleset = RegexRuleset(**data)
|
||||
else:
|
||||
continue
|
||||
|
||||
# 确保所有规则的 scope 正确
|
||||
for rule in ruleset.rules:
|
||||
rule.scope = RegexScope.PRESET
|
||||
rule.presetName = preset_name
|
||||
|
||||
self.preset_rulesets[preset_name] = ruleset
|
||||
logger.info(f"加载预设规则: {preset_name} ({len(ruleset.rules)} 条规则)")
|
||||
except Exception as e:
|
||||
logger.error(f"加载预设规则失败 {json_file}: {e}")
|
||||
|
||||
def _convert_sillytavern_format(
|
||||
self,
|
||||
st_rules: List[dict],
|
||||
name: str,
|
||||
scope: RegexScope = RegexScope.GLOBAL
|
||||
) -> RegexRuleset:
|
||||
"""将 SillyTavern 格式转换为内部格式"""
|
||||
rules = []
|
||||
for idx, st_rule in enumerate(st_rules):
|
||||
find_regex = st_rule.get('findRegex', '')
|
||||
pattern, flags = self._parse_st_regex(find_regex)
|
||||
|
||||
# 解析 placement(默认为 AI_OUTPUT)
|
||||
placement_data = st_rule.get('placement', [2])
|
||||
placement = [RegexPlacement(p) for p in placement_data]
|
||||
|
||||
rule = RegexRule(
|
||||
id=str(uuid4()),
|
||||
scriptName=st_rule.get('scriptName', f"{name} 规则 {idx + 1}"),
|
||||
findRegex=pattern,
|
||||
replaceString=st_rule.get('replaceString', ''),
|
||||
trimStrings=st_rule.get('trimStrings', []),
|
||||
placement=placement,
|
||||
substituteRegex=SubstituteMode(st_rule.get('substituteRegex', 0)),
|
||||
markdownOnly=st_rule.get('markdownOnly', False),
|
||||
promptOnly=st_rule.get('promptOnly', False),
|
||||
runOnEdit=st_rule.get('runOnEdit', True),
|
||||
minDepth=st_rule.get('minDepth', 0),
|
||||
maxDepth=st_rule.get('maxDepth'),
|
||||
scope=scope,
|
||||
characterName=name if scope == RegexScope.CHARACTER else None,
|
||||
presetName=name if scope == RegexScope.PRESET else None,
|
||||
disabled=st_rule.get('disabled', False),
|
||||
order=idx
|
||||
)
|
||||
rules.append(rule)
|
||||
|
||||
ruleset = RegexRuleset(
|
||||
id=str(uuid4()),
|
||||
name=f"{name} 规则集",
|
||||
description=f"从 SillyTavern 导入的规则",
|
||||
rules=rules,
|
||||
isSillyTavernFormat=True
|
||||
)
|
||||
|
||||
return ruleset
|
||||
|
||||
def _parse_st_regex(self, st_regex: str) -> tuple[str, str]:
|
||||
"""解析 SillyTavern 的正则表达式格式 /pattern/flags"""
|
||||
if st_regex.startswith('/') and st_regex.count('/') >= 2:
|
||||
parts = st_regex.split('/')
|
||||
pattern = '/'.join(parts[1:-1])
|
||||
flags = parts[-1] if len(parts) > 2 else ''
|
||||
return pattern, flags
|
||||
else:
|
||||
return st_regex, ''
|
||||
|
||||
def apply_rules_by_placement(
|
||||
self,
|
||||
text: str,
|
||||
placement: int,
|
||||
character_name: Optional[str] = None,
|
||||
preset_name: Optional[str] = None,
|
||||
message_depth: int = 0
|
||||
) -> str:
|
||||
"""
|
||||
根据 placement 应用正则规则
|
||||
|
||||
Args:
|
||||
text: 要处理的文本
|
||||
placement: 应用位置(0-5)
|
||||
character_name: 当前角色卡名称
|
||||
preset_name: 当前预设名称
|
||||
message_depth: 消息深度
|
||||
|
||||
Returns:
|
||||
处理后的文本
|
||||
"""
|
||||
rules = self.get_rules_for_context(character_name, preset_name)
|
||||
|
||||
result = text
|
||||
for rule in rules:
|
||||
# 检查此规则是否适用于当前 placement
|
||||
if placement not in [p.value for p in rule.placement]:
|
||||
continue
|
||||
|
||||
# 检查消息深度限制
|
||||
if message_depth < rule.minDepth:
|
||||
continue
|
||||
if rule.maxDepth is not None and message_depth > rule.maxDepth:
|
||||
continue
|
||||
|
||||
# 应用规则
|
||||
result = self._apply_single_rule(result, rule)
|
||||
|
||||
return result
|
||||
|
||||
def _apply_single_rule(self, text: str, rule: RegexRule) -> str:
|
||||
"""应用单条正则规则"""
|
||||
try:
|
||||
flags = 0
|
||||
if 'i' in rule.findRegex:
|
||||
flags |= re.IGNORECASE
|
||||
if 'm' in rule.findRegex:
|
||||
flags |= re.MULTILINE
|
||||
if 's' in rule.findRegex:
|
||||
flags |= re.DOTALL
|
||||
|
||||
pattern = rule.findRegex.replace('i', '').replace('m', '').replace('s', '')
|
||||
|
||||
if rule.substituteRegex == SubstituteMode.REPLACE_FIRST:
|
||||
result = re.sub(pattern, rule.replaceString, text, count=1, flags=flags)
|
||||
else:
|
||||
result = re.sub(pattern, rule.replaceString, text, flags=flags)
|
||||
|
||||
for trim_str in rule.trimStrings:
|
||||
result = result.replace(trim_str, '')
|
||||
|
||||
return result
|
||||
except re.error as e:
|
||||
logger.error(f"正则表达式错误 [{rule.scriptName}]: {e}")
|
||||
return text
|
||||
|
||||
def get_rules_for_context(
|
||||
self,
|
||||
character_name: Optional[str] = None,
|
||||
preset_name: Optional[str] = None
|
||||
) -> List[RegexRule]:
|
||||
"""
|
||||
根据上下文获取适用的规则列表
|
||||
|
||||
优先级:全局规则 + 角色规则 + 预设规则
|
||||
"""
|
||||
applicable_rules = []
|
||||
|
||||
# 1. 加载全局规则
|
||||
for ruleset in self.global_rulesets.values():
|
||||
for rule in ruleset.rules:
|
||||
if not rule.disabled:
|
||||
applicable_rules.append(rule)
|
||||
|
||||
# 2. 加载角色卡规则
|
||||
if character_name and character_name in self.character_rulesets:
|
||||
ruleset = self.character_rulesets[character_name]
|
||||
for rule in ruleset.rules:
|
||||
if not rule.disabled:
|
||||
applicable_rules.append(rule)
|
||||
|
||||
# 3. 加载预设规则
|
||||
if preset_name and preset_name in self.preset_rulesets:
|
||||
ruleset = self.preset_rulesets[preset_name]
|
||||
for rule in ruleset.rules:
|
||||
if not rule.disabled:
|
||||
applicable_rules.append(rule)
|
||||
|
||||
# 按 order 排序
|
||||
applicable_rules.sort(key=lambda r: r.order)
|
||||
|
||||
return applicable_rules
|
||||
|
||||
def save_ruleset(self, ruleset: RegexRuleset, scope: RegexScope, name: Optional[str] = None):
|
||||
"""保存规则集到文件"""
|
||||
if scope == RegexScope.GLOBAL:
|
||||
file_path = self.global_path / f"{ruleset.id}.json"
|
||||
elif scope == RegexScope.CHARACTER:
|
||||
file_path = self.characters_path / f"{name or 'unknown'}.json"
|
||||
elif scope == RegexScope.PRESET:
|
||||
file_path = self.presets_path / f"{name or 'unknown'}.json"
|
||||
else:
|
||||
raise ValueError(f"未知的作用域: {scope}")
|
||||
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(ruleset.dict(), f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.info(f"保存规则集到: {file_path}")
|
||||
|
||||
def delete_ruleset(self, scope: RegexScope, name: str):
|
||||
"""删除规则集"""
|
||||
if scope == RegexScope.CHARACTER:
|
||||
file_path = self.characters_path / f"{name}.json"
|
||||
elif scope == RegexScope.PRESET:
|
||||
file_path = self.presets_path / f"{name}.json"
|
||||
else:
|
||||
raise ValueError(f"不能删除全局规则集")
|
||||
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
logger.info(f"删除规则集: {file_path}")
|
||||
|
||||
|
||||
# 全局实例
|
||||
regex_service = RegexService()
|
||||
95
backend/services/system_settings_service.py
Normal file
95
backend/services/system_settings_service.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""
|
||||
系统设置服务
|
||||
|
||||
负责加载、保存和管理全局系统设置。
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from core.config import settings
|
||||
from models.system_settings import SystemSettings, DEFAULT_SYSTEM_SETTINGS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SystemSettingsService:
|
||||
"""
|
||||
系统设置服务
|
||||
|
||||
提供设置的加载、保存和访问功能
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.settings_file = settings.SYSTEM_SETTINGS_FILE
|
||||
self._settings: Optional[SystemSettings] = None
|
||||
self._load_settings()
|
||||
|
||||
def _load_settings(self):
|
||||
"""从文件加载系统设置"""
|
||||
if self.settings_file.exists():
|
||||
try:
|
||||
with open(self.settings_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
self._settings = SystemSettings(**data)
|
||||
logger.info(f"加载系统设置成功")
|
||||
except Exception as e:
|
||||
logger.error(f"加载系统设置失败: {e}")
|
||||
self._settings = DEFAULT_SYSTEM_SETTINGS.copy()
|
||||
else:
|
||||
logger.info("系统设置文件不存在,使用默认设置")
|
||||
self._settings = DEFAULT_SYSTEM_SETTINGS.copy()
|
||||
self._save_settings()
|
||||
|
||||
def _save_settings(self):
|
||||
"""保存系统设置到文件"""
|
||||
try:
|
||||
# 确保父目录存在
|
||||
self.settings_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 写入文件
|
||||
with open(self.settings_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(self._settings.dict(), f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.info(f"系统设置已保存到 {self.settings_file}")
|
||||
except Exception as e:
|
||||
logger.error(f"保存系统设置失败: {e}")
|
||||
|
||||
@property
|
||||
def settings(self) -> SystemSettings:
|
||||
"""获取当前系统设置"""
|
||||
return self._settings
|
||||
|
||||
def update_thinking_tags(self, prefix: str, suffix: str):
|
||||
"""更新思考标签配置"""
|
||||
self._settings.thinkingTagPrefix = prefix
|
||||
self._settings.thinkingTagSuffix = suffix
|
||||
self._settings.updatedAt = int(__import__('time').time())
|
||||
self._save_settings()
|
||||
logger.info(f"思考标签已更新: {prefix} ... {suffix}")
|
||||
|
||||
def update_current_preset(self, preset_name: Optional[str]):
|
||||
"""更新当前选中的预设名称"""
|
||||
self._settings.currentPresetName = preset_name
|
||||
self._settings.updatedAt = int(__import__('time').time())
|
||||
self._save_settings()
|
||||
logger.info(f"当前预设已更新: {preset_name}")
|
||||
|
||||
def get_thinking_tag_pattern(self) -> str:
|
||||
"""获取思考标签的正则表达式模式"""
|
||||
prefix = self._settings.thinkingTagPrefix
|
||||
suffix = self._settings.thinkingTagSuffix
|
||||
|
||||
# 转义特殊字符
|
||||
import re
|
||||
escaped_prefix = re.escape(prefix)
|
||||
escaped_suffix = re.escape(suffix)
|
||||
|
||||
# 返回匹配思考内容的正则模式
|
||||
return f"{escaped_prefix}[\\s\\S]*?{escaped_suffix}"
|
||||
|
||||
|
||||
# 全局实例
|
||||
system_settings_service = SystemSettingsService()
|
||||
161
backend/services/task_queue_manager.py
Normal file
161
backend/services/task_queue_manager.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
任务队列管理器
|
||||
管理并行任务(生图、动态表格维护等)的状态和生命周期
|
||||
"""
|
||||
import asyncio
|
||||
from typing import Dict, List, Optional
|
||||
from enum import Enum
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
"""任务状态枚举"""
|
||||
PENDING = "pending" # 等待中
|
||||
RUNNING = "running" # 进行中
|
||||
COMPLETED = "completed" # 已完成
|
||||
FAILED = "failed" # 失败
|
||||
CANCELLED = "cancelled" # 已取消
|
||||
|
||||
|
||||
class TaskType(Enum):
|
||||
"""任务类型枚举"""
|
||||
IMAGE_WORKFLOW = "image_workflow"
|
||||
DYNAMIC_TABLE = "dynamic_table"
|
||||
|
||||
|
||||
class TaskItem:
|
||||
"""任务项"""
|
||||
|
||||
def __init__(self, task_id: str, task_type: TaskType, chat_id: str):
|
||||
self.task_id = task_id
|
||||
self.task_type = task_type
|
||||
self.chat_id = chat_id
|
||||
self.status = TaskStatus.PENDING
|
||||
self.created_at = datetime.now()
|
||||
self.started_at = None
|
||||
self.completed_at = None
|
||||
self.error = None
|
||||
self.metadata = {} # 用于存储提示词、修改内容等
|
||||
|
||||
def to_dict(self):
|
||||
"""转换为字典格式(前端友好)"""
|
||||
return {
|
||||
"taskId": self.task_id,
|
||||
"taskType": self.task_type.value,
|
||||
"chatId": self.chat_id,
|
||||
"status": self.status.value,
|
||||
"createdAt": self.created_at.isoformat(),
|
||||
"startedAt": self.started_at.isoformat() if self.started_at else None,
|
||||
"completedAt": self.completed_at.isoformat() if self.completed_at else None,
|
||||
"error": self.error,
|
||||
"metadata": self.metadata
|
||||
}
|
||||
|
||||
|
||||
class TaskQueueManager:
|
||||
"""
|
||||
全局任务队列管理器
|
||||
|
||||
功能:
|
||||
- 管理所有并行任务的生命周期
|
||||
- 支持按聊天ID查询任务
|
||||
- 支持取消任务
|
||||
- 自动清理已完成的任务
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.tasks: Dict[str, TaskItem] = {}
|
||||
self.chat_tasks: Dict[str, List[str]] = {} # chat_id -> [task_ids]
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def add_task(self, task_id: str, task_type: TaskType, chat_id: str) -> TaskItem:
|
||||
"""添加任务到队列"""
|
||||
async with self._lock:
|
||||
task = TaskItem(task_id, task_type, chat_id)
|
||||
self.tasks[task_id] = task
|
||||
|
||||
if chat_id not in self.chat_tasks:
|
||||
self.chat_tasks[chat_id] = []
|
||||
self.chat_tasks[chat_id].append(task_id)
|
||||
|
||||
return task
|
||||
|
||||
async def start_task(self, task_id: str):
|
||||
"""标记任务开始执行"""
|
||||
async with self._lock:
|
||||
if task_id in self.tasks:
|
||||
self.tasks[task_id].status = TaskStatus.RUNNING
|
||||
self.tasks[task_id].started_at = datetime.now()
|
||||
|
||||
async def complete_task(self, task_id: str, metadata: dict = None):
|
||||
"""标记任务完成"""
|
||||
async with self._lock:
|
||||
if task_id in self.tasks:
|
||||
self.tasks[task_id].status = TaskStatus.COMPLETED
|
||||
self.tasks[task_id].completed_at = datetime.now()
|
||||
if metadata:
|
||||
self.tasks[task_id].metadata.update(metadata)
|
||||
|
||||
async def fail_task(self, task_id: str, error: str):
|
||||
"""标记任务失败"""
|
||||
async with self._lock:
|
||||
if task_id in self.tasks:
|
||||
self.tasks[task_id].status = TaskStatus.FAILED
|
||||
self.tasks[task_id].completed_at = datetime.now()
|
||||
self.tasks[task_id].error = error
|
||||
|
||||
async def cancel_task(self, task_id: str) -> bool:
|
||||
"""
|
||||
取消任务
|
||||
|
||||
Returns:
|
||||
bool: 是否成功取消
|
||||
"""
|
||||
async with self._lock:
|
||||
if task_id in self.tasks:
|
||||
task = self.tasks[task_id]
|
||||
if task.status in [TaskStatus.PENDING, TaskStatus.RUNNING]:
|
||||
task.status = TaskStatus.CANCELLED
|
||||
task.completed_at = datetime.now()
|
||||
return True
|
||||
return False
|
||||
|
||||
async def get_chat_tasks(self, chat_id: str, include_completed: bool = False) -> List[dict]:
|
||||
"""
|
||||
获取某个聊天的所有任务
|
||||
|
||||
Args:
|
||||
chat_id: 聊天ID
|
||||
include_completed: 是否包含已完成的任务
|
||||
|
||||
Returns:
|
||||
List[dict]: 任务列表
|
||||
"""
|
||||
async with self._lock:
|
||||
task_ids = self.chat_tasks.get(chat_id, [])
|
||||
tasks = []
|
||||
for task_id in task_ids:
|
||||
if task_id in self.tasks:
|
||||
task = self.tasks[task_id]
|
||||
# 根据参数决定是否包含已完成的任务
|
||||
if include_completed or task.status in [TaskStatus.PENDING, TaskStatus.RUNNING]:
|
||||
tasks.append(task.to_dict())
|
||||
return tasks
|
||||
|
||||
async def cleanup_completed_tasks(self, chat_id: str):
|
||||
"""清理已完成的任务"""
|
||||
async with self._lock:
|
||||
if chat_id in self.chat_tasks:
|
||||
task_ids = self.chat_tasks[chat_id]
|
||||
completed_ids = [
|
||||
tid for tid in task_ids
|
||||
if tid in self.tasks and
|
||||
self.tasks[tid].status in [TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED]
|
||||
]
|
||||
for tid in completed_ids:
|
||||
del self.tasks[tid]
|
||||
self.chat_tasks[chat_id].remove(tid)
|
||||
|
||||
|
||||
# 全局实例
|
||||
task_queue_manager = TaskQueueManager()
|
||||
427
backend/services/token_usage_service.py
Normal file
427
backend/services/token_usage_service.py
Normal file
@@ -0,0 +1,427 @@
|
||||
"""
|
||||
Token 使用统计服务
|
||||
|
||||
负责记录、查询和分析 LLM 调用的 token 使用情况
|
||||
数据持久化到 data/token_usage 目录,按月份组织
|
||||
采用双层存储:
|
||||
1. JSONL 文件 - 详细记录(按月存储)
|
||||
2. 索引文件 - 快速聚合统计(按 API URL、日期等维度)
|
||||
"""
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional, Any
|
||||
from datetime import datetime
|
||||
from collections import defaultdict
|
||||
|
||||
try:
|
||||
from backend.models.internal import TokenUsageRecord, TokenUsageStatus
|
||||
from backend.core.config import settings
|
||||
except ImportError:
|
||||
from models.internal import TokenUsageRecord, TokenUsageStatus
|
||||
from core.config import settings
|
||||
|
||||
|
||||
class TokenUsageService:
|
||||
"""
|
||||
Token 使用统计服务
|
||||
|
||||
功能:
|
||||
- 记录每次 LLM 调用的 token 使用情况
|
||||
- 按月份、日期、角色、聊天、API URL 维度统计
|
||||
- 支持中断和失败标记
|
||||
- 数据持久化到文件系统(JSONL + 索引)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.token_usage_dir = settings.DATA_PATH / "token_usage"
|
||||
self.token_usage_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ✅ 索引文件目录 - 用于快速聚合查询
|
||||
self.index_dir = self.token_usage_dir / "indexes"
|
||||
self.index_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _get_month_file(self, year: int, month: int) -> Path:
|
||||
"""获取指定月份的统计文件路径"""
|
||||
month_dir = self.token_usage_dir / f"{year}"
|
||||
month_dir.mkdir(parents=True, exist_ok=True)
|
||||
return month_dir / f"{month:02d}.jsonl"
|
||||
|
||||
def _load_month_records(self, year: int, month: int) -> List[TokenUsageRecord]:
|
||||
"""加载指定月份的所有记录"""
|
||||
file_path = self._get_month_file(year, month)
|
||||
|
||||
if not file_path.exists():
|
||||
return []
|
||||
|
||||
records = []
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
if line.strip():
|
||||
data = json.loads(line)
|
||||
records.append(TokenUsageRecord(**data))
|
||||
except Exception as e:
|
||||
print(f"[TokenUsage] 加载记录失败: {e}")
|
||||
|
||||
return records
|
||||
|
||||
def _save_record(self, record: TokenUsageRecord):
|
||||
"""保存单条记录到对应的月份文件"""
|
||||
dt = datetime.fromtimestamp(record.timestamp)
|
||||
file_path = self._get_month_file(dt.year, dt.month)
|
||||
|
||||
try:
|
||||
with open(file_path, 'a', encoding='utf-8') as f:
|
||||
f.write(json.dumps(record.model_dump(), ensure_ascii=False) + '\n')
|
||||
|
||||
# ✅ 同时更新索引文件(用于快速查询)
|
||||
self._update_indexes(record)
|
||||
except Exception as e:
|
||||
print(f"[TokenUsage] 保存记录失败: {e}")
|
||||
|
||||
def _update_indexes(self, record: TokenUsageRecord):
|
||||
"""
|
||||
更新索引文件 - 实现高效的按维度聚合查询
|
||||
|
||||
索引结构:
|
||||
- indexes/api_urls.json - 按 API URL 聚合
|
||||
- indexes/daily/{year}-{month}.json - 按日聚合
|
||||
"""
|
||||
dt = datetime.fromtimestamp(record.timestamp)
|
||||
|
||||
# 1. 更新 API URL 索引
|
||||
if record.apiUrl:
|
||||
api_url_index = self.index_dir / "api_urls.json"
|
||||
self._update_api_url_index(api_url_index, record)
|
||||
|
||||
# 2. 更新每日索引
|
||||
daily_index = self.index_dir / "daily" / f"{dt.year}-{dt.month:02d}.json"
|
||||
daily_index.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._update_daily_index(daily_index, record)
|
||||
|
||||
def _update_api_url_index(self, index_file: Path, record: TokenUsageRecord):
|
||||
"""更新 API URL 索引文件"""
|
||||
index_data = {}
|
||||
|
||||
# 加载现有索引
|
||||
if index_file.exists():
|
||||
try:
|
||||
with open(index_file, 'r', encoding='utf-8') as f:
|
||||
index_data = json.load(f)
|
||||
except:
|
||||
index_data = {}
|
||||
|
||||
# 更新统计
|
||||
api_url = record.apiUrl
|
||||
if api_url not in index_data:
|
||||
index_data[api_url] = {
|
||||
"totalPromptTokens": 0,
|
||||
"totalCompletionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"count": 0,
|
||||
"firstUsed": record.timestamp,
|
||||
"lastUsed": record.timestamp
|
||||
}
|
||||
|
||||
stats = index_data[api_url]
|
||||
stats["totalPromptTokens"] += record.promptTokens
|
||||
stats["totalCompletionTokens"] += record.completionTokens
|
||||
stats["totalTokens"] += record.totalTokens
|
||||
stats["count"] += 1
|
||||
stats["lastUsed"] = max(stats["lastUsed"], record.timestamp)
|
||||
stats["firstUsed"] = min(stats["firstUsed"], record.timestamp)
|
||||
|
||||
# 保存索引
|
||||
with open(index_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(index_data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
def _update_daily_index(self, index_file: Path, record: TokenUsageRecord):
|
||||
"""更新每日索引文件"""
|
||||
dt = datetime.fromtimestamp(record.timestamp)
|
||||
day_key = f"{dt.year}-{dt.month:02d}-{dt.day:02d}"
|
||||
|
||||
index_data = {}
|
||||
|
||||
# 加载现有索引
|
||||
if index_file.exists():
|
||||
try:
|
||||
with open(index_file, 'r', encoding='utf-8') as f:
|
||||
index_data = json.load(f)
|
||||
except:
|
||||
index_data = {}
|
||||
|
||||
# 更新统计
|
||||
if day_key not in index_data:
|
||||
index_data[day_key] = {
|
||||
"promptTokens": 0,
|
||||
"completionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"count": 0
|
||||
}
|
||||
|
||||
stats = index_data[day_key]
|
||||
stats["promptTokens"] += record.promptTokens
|
||||
stats["completionTokens"] += record.completionTokens
|
||||
stats["totalTokens"] += record.totalTokens
|
||||
stats["count"] += 1
|
||||
|
||||
# 保存索引
|
||||
with open(index_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(index_data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
async def record_usage(
|
||||
self,
|
||||
chat_id: str,
|
||||
role_name: str,
|
||||
chat_name: str,
|
||||
prompt_tokens: int,
|
||||
completion_tokens: int,
|
||||
total_tokens: int,
|
||||
status: TokenUsageStatus = TokenUsageStatus.COMPLETED,
|
||||
message_id: Optional[str] = None,
|
||||
floor: Optional[int] = None,
|
||||
error_message: Optional[str] = None,
|
||||
duration: Optional[float] = None,
|
||||
model: Optional[str] = None,
|
||||
api_provider: Optional[str] = None,
|
||||
api_url: Optional[str] = None
|
||||
) -> TokenUsageRecord:
|
||||
"""
|
||||
记录一次 LLM 调用的 token 使用情况
|
||||
|
||||
Args:
|
||||
chat_id: 聊天ID
|
||||
role_name: 角色名称
|
||||
chat_name: 聊天名称
|
||||
prompt_tokens: 输入 token 数
|
||||
completion_tokens: 输出 token 数
|
||||
total_tokens: 总 token 数
|
||||
status: 请求状态
|
||||
message_id: 关联的消息ID
|
||||
floor: 楼层号
|
||||
error_message: 错误信息
|
||||
duration: 请求耗时
|
||||
model: 使用的模型
|
||||
api_provider: API 提供商
|
||||
api_url: API URL地址
|
||||
|
||||
Returns:
|
||||
TokenUsageRecord: 创建的记录
|
||||
"""
|
||||
record = TokenUsageRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
chatId=chat_id,
|
||||
roleName=role_name,
|
||||
chatName=chat_name,
|
||||
messageId=message_id,
|
||||
floor=floor,
|
||||
promptTokens=prompt_tokens,
|
||||
completionTokens=completion_tokens,
|
||||
totalTokens=total_tokens,
|
||||
status=status,
|
||||
errorMessage=error_message,
|
||||
duration=duration,
|
||||
model=model,
|
||||
apiProvider=api_provider,
|
||||
apiUrl=api_url
|
||||
)
|
||||
|
||||
self._save_record(record)
|
||||
return record
|
||||
|
||||
async def get_stats_by_month(
|
||||
self,
|
||||
year: int,
|
||||
month: int,
|
||||
role_name: Optional[str] = None,
|
||||
chat_name: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
获取指定月份的统计数据
|
||||
|
||||
Args:
|
||||
year: 年份
|
||||
month: 月份
|
||||
role_name: 角色名称(可选,用于过滤)
|
||||
chat_name: 聊天名称(可选,用于过滤)
|
||||
|
||||
Returns:
|
||||
统计数据字典
|
||||
"""
|
||||
records = self._load_month_records(year, month)
|
||||
|
||||
# 过滤
|
||||
if role_name:
|
||||
records = [r for r in records if r.roleName == role_name]
|
||||
if chat_name:
|
||||
records = [r for r in records if r.chatName == chat_name]
|
||||
|
||||
# 统计
|
||||
total_prompt = sum(r.promptTokens for r in records)
|
||||
total_completion = sum(r.completionTokens for r in records)
|
||||
total_tokens = sum(r.totalTokens for r in records)
|
||||
|
||||
completed_count = sum(1 for r in records if r.status == TokenUsageStatus.COMPLETED)
|
||||
interrupted_count = sum(1 for r in records if r.status == TokenUsageStatus.INTERRUPTED)
|
||||
failed_count = sum(1 for r in records if r.status == TokenUsageStatus.FAILED)
|
||||
|
||||
# 按日期分组
|
||||
daily_stats = defaultdict(lambda: {
|
||||
"promptTokens": 0,
|
||||
"completionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"count": 0
|
||||
})
|
||||
|
||||
for r in records:
|
||||
dt = datetime.fromtimestamp(r.timestamp)
|
||||
day_key = f"{dt.year}-{dt.month:02d}-{dt.day:02d}"
|
||||
daily_stats[day_key]["promptTokens"] += r.promptTokens
|
||||
daily_stats[day_key]["completionTokens"] += r.completionTokens
|
||||
daily_stats[day_key]["totalTokens"] += r.totalTokens
|
||||
daily_stats[day_key]["count"] += 1
|
||||
|
||||
# 按角色分组
|
||||
role_stats = defaultdict(lambda: {
|
||||
"promptTokens": 0,
|
||||
"completionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"count": 0
|
||||
})
|
||||
|
||||
for r in records:
|
||||
role_stats[r.roleName]["promptTokens"] += r.promptTokens
|
||||
role_stats[r.roleName]["completionTokens"] += r.completionTokens
|
||||
role_stats[r.roleName]["totalTokens"] += r.totalTokens
|
||||
role_stats[r.roleName]["count"] += 1
|
||||
|
||||
# 按聊天分组
|
||||
chat_stats = defaultdict(lambda: {
|
||||
"promptTokens": 0,
|
||||
"completionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"count": 0
|
||||
})
|
||||
|
||||
for r in records:
|
||||
chat_key = f"{r.roleName}/{r.chatName}"
|
||||
chat_stats[chat_key]["promptTokens"] += r.promptTokens
|
||||
chat_stats[chat_key]["completionTokens"] += r.completionTokens
|
||||
chat_stats[chat_key]["totalTokens"] += r.totalTokens
|
||||
chat_stats[chat_key]["count"] += 1
|
||||
|
||||
# ✅ 按 API URL 分组
|
||||
api_url_stats = defaultdict(lambda: {
|
||||
"promptTokens": 0,
|
||||
"completionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"count": 0
|
||||
})
|
||||
|
||||
for r in records:
|
||||
if r.apiUrl:
|
||||
api_url_stats[r.apiUrl]["promptTokens"] += r.promptTokens
|
||||
api_url_stats[r.apiUrl]["completionTokens"] += r.completionTokens
|
||||
api_url_stats[r.apiUrl]["totalTokens"] += r.totalTokens
|
||||
api_url_stats[r.apiUrl]["count"] += 1
|
||||
|
||||
return {
|
||||
"year": year,
|
||||
"month": month,
|
||||
"totalRecords": len(records),
|
||||
"totalPromptTokens": total_prompt,
|
||||
"totalCompletionTokens": total_completion,
|
||||
"totalTokens": total_tokens,
|
||||
"completedCount": completed_count,
|
||||
"interruptedCount": interrupted_count,
|
||||
"failedCount": failed_count,
|
||||
"dailyStats": dict(daily_stats),
|
||||
"roleStats": dict(role_stats),
|
||||
"chatStats": dict(chat_stats),
|
||||
"apiUrlStats": dict(api_url_stats), # ✅ 新增
|
||||
"records": [r.model_dump() for r in records[:100]] # 最近100条记录
|
||||
}
|
||||
|
||||
async def list_months(self) -> List[Dict[str, int]]:
|
||||
"""列出所有有数据的月份"""
|
||||
months = []
|
||||
|
||||
if not self.token_usage_dir.exists():
|
||||
return months
|
||||
|
||||
for year_dir in sorted(self.token_usage_dir.iterdir()):
|
||||
if year_dir.is_dir() and year_dir.name.isdigit():
|
||||
year = int(year_dir.name)
|
||||
for month_file in sorted(year_dir.glob("*.jsonl")):
|
||||
month = int(month_file.stem)
|
||||
months.append({"year": year, "month": month})
|
||||
|
||||
return months
|
||||
|
||||
async def get_api_url_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
✅ 获取按 API URL 分组的统计数据(从索引文件快速读取)
|
||||
|
||||
Returns:
|
||||
{api_url: {totalPromptTokens, totalCompletionTokens, totalTokens, count, firstUsed, lastUsed}}
|
||||
"""
|
||||
api_url_index = self.index_dir / "api_urls.json"
|
||||
|
||||
if not api_url_index.exists():
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(api_url_index, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f"[TokenUsage] 读取 API URL 索引失败: {e}")
|
||||
return {}
|
||||
|
||||
async def get_daily_stats(self, year: int, month: int) -> Dict[str, Any]:
|
||||
"""
|
||||
✅ 获取指定月份的每日统计数据(从索引文件快速读取)
|
||||
|
||||
Args:
|
||||
year: 年份
|
||||
month: 月份
|
||||
|
||||
Returns:
|
||||
{day_key: {promptTokens, completionTokens, totalTokens, count}}
|
||||
"""
|
||||
daily_index = self.index_dir / "daily" / f"{year}-{month:02d}.json"
|
||||
|
||||
if not daily_index.exists():
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(daily_index, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f"[TokenUsage] 读取每日索引失败: {e}")
|
||||
return {}
|
||||
|
||||
async def get_available_roles(self, year: int, month: int) -> List[str]:
|
||||
"""获取指定月份有数据的角色列表"""
|
||||
records = self._load_month_records(year, month)
|
||||
roles = set(r.roleName for r in records)
|
||||
return sorted(list(roles))
|
||||
|
||||
async def get_available_chats(
|
||||
self,
|
||||
year: int,
|
||||
month: int,
|
||||
role_name: Optional[str] = None
|
||||
) -> List[str]:
|
||||
"""获取指定月份有数据的聊天列表"""
|
||||
records = self._load_month_records(year, month)
|
||||
|
||||
if role_name:
|
||||
records = [r for r in records if r.roleName == role_name]
|
||||
|
||||
chats = set(f"{r.roleName}/{r.chatName}" for r in records)
|
||||
return sorted(list(chats))
|
||||
|
||||
|
||||
# 全局实例
|
||||
token_usage_service = TokenUsageService()
|
||||
Reference in New Issue
Block a user