新增预设工作台：编辑、版本树、结构检查、用例试跑放在同一页。
改预设要能一边写一边看装出来的 messages，还要多模型对照，所以把编辑与试跑拆成两个视图塞进一个页面；版本存完整快照、可分叉，用例全局共用一份、运行记录按预设存。顺带让读预设文件先剥 BOM，静态资源改成不缓存，免得改了脚本还要强刷。
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
import { collectMacroUsages, findUnclosedMacro } from "../directives/macros.js";
|
||||
import type { PresetPackage } from "../types/preset.js";
|
||||
import { CHAT_HISTORY_MARKER } from "./markers.js";
|
||||
|
||||
/**
|
||||
* 预设编辑器的结构检查（不调用模型）。
|
||||
* error：会导致装配或发送异常；warn：会让预设行为与预期不符，但仍可发送。
|
||||
*/
|
||||
export type AuthoringIssue = {
|
||||
level: "error" | "warn";
|
||||
entryId: string | null;
|
||||
entryName: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
const VALID_ROLES = new Set(["system", "user", "assistant"]);
|
||||
const FULLWIDTH_OPEN = "｛｛";
|
||||
|
||||
export function checkAuthoringPreset(preset: PresetPackage): AuthoringIssue[] {
|
||||
const issues: AuthoringIssue[] = [];
|
||||
const byId = new Map<string, PresetPackage["prompts"][number]>();
|
||||
const push = (
|
||||
level: AuthoringIssue["level"],
|
||||
entryId: string | null,
|
||||
entryName: string,
|
||||
message: string,
|
||||
) => issues.push({ level, entryId, entryName, message });
|
||||
|
||||
if (preset.prompts.length === 0) {
|
||||
push("error", null, "", "预设没有任何模块");
|
||||
return issues;
|
||||
}
|
||||
|
||||
for (const prompt of preset.prompts) {
|
||||
if (byId.has(prompt.id)) {
|
||||
push("error", prompt.id, prompt.name || prompt.id, "模块 id 重复");
|
||||
continue;
|
||||
}
|
||||
byId.set(prompt.id, prompt);
|
||||
}
|
||||
|
||||
const enabledOrder = preset.promptOrder.filter((o) => o.enabled);
|
||||
if (!enabledOrder.some((o) => o.promptId === CHAT_HISTORY_MARKER)) {
|
||||
push("warn", null, "", "没有启用「对话历史」位置，真实聊天记录不会进入请求");
|
||||
}
|
||||
|
||||
const inOrder = new Set(preset.promptOrder.map((o) => o.promptId));
|
||||
for (const prompt of preset.prompts) {
|
||||
if (!inOrder.has(prompt.id)) {
|
||||
push("warn", prompt.id, prompt.name, "未加入顺序，不会发送");
|
||||
}
|
||||
}
|
||||
|
||||
const seenOrder = new Set<string>();
|
||||
const seenOrderIndex = new Set<number>();
|
||||
const seenMarkers = new Set<string>();
|
||||
for (const item of preset.promptOrder) {
|
||||
if (seenOrder.has(item.promptId)) {
|
||||
push("error", item.promptId, item.promptId, "顺序中重复引用了同一模块");
|
||||
}
|
||||
seenOrder.add(item.promptId);
|
||||
if (seenOrderIndex.has(item.orderIndex)) {
|
||||
push("warn", item.promptId, item.promptId, `顺序号重复：${item.orderIndex}`);
|
||||
}
|
||||
seenOrderIndex.add(item.orderIndex);
|
||||
if (!byId.has(item.promptId)) {
|
||||
push("error", item.promptId, item.promptId, "顺序中引用了不存在的模块");
|
||||
}
|
||||
}
|
||||
|
||||
for (const item of enabledOrder) {
|
||||
const entry = byId.get(item.promptId);
|
||||
if (!entry) continue;
|
||||
const name = entry.name || entry.id;
|
||||
|
||||
if (entry.marker) {
|
||||
if (seenMarkers.has(entry.id)) {
|
||||
push("warn", entry.id, name, "位置在同一顺序中启用了多次，会重复填充");
|
||||
}
|
||||
seenMarkers.add(entry.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!VALID_ROLES.has(entry.role)) {
|
||||
push("error", entry.id, name, `角色非法：${String(entry.role)}`);
|
||||
}
|
||||
|
||||
const body = entry.content ?? "";
|
||||
if (!body.trim()) {
|
||||
push("warn", entry.id, name, "模块正文为空，将被跳过");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (body.includes(FULLWIDTH_OPEN)) {
|
||||
push(
|
||||
"warn",
|
||||
entry.id,
|
||||
name,
|
||||
"含全角 ｛｛，运行时不会展开，请改为半角 {{",
|
||||
);
|
||||
}
|
||||
|
||||
const unclosed = findUnclosedMacro(body);
|
||||
if (unclosed) {
|
||||
push("error", entry.id, name, `存在未闭合的 {{ 宏：${unclosed}`);
|
||||
}
|
||||
|
||||
for (const usage of collectMacroUsages(body)) {
|
||||
if (!usage.known) {
|
||||
push("warn", entry.id, name, `未识别宏 ${usage.name}，将原样发出`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { getPresetsDir } from "../config/user-data-dir.js";
|
||||
|
||||
/**
|
||||
* 每个预设在试跑视图里的「上次打开过啥」。
|
||||
*
|
||||
* 用例本身是全局的（`presets/tests/global`），但「这个预设上次用的是哪条用例」
|
||||
* 得按预设记，否则切换预设时选择会跳。存盘而不是塞浏览器缓存：换浏览器、
|
||||
* 重开页面都还是同一条。
|
||||
*
|
||||
* 文件：`<data>/presets/lab-state/<presetId>.json`
|
||||
*/
|
||||
export type PresetLabState = {
|
||||
presetId: string;
|
||||
/** 试跑视图的当前用例（`TestCase.id`） */
|
||||
selectedCaseId: string | null;
|
||||
/** 编辑视图右栏预览用的用例 */
|
||||
previewCaseId: string | null;
|
||||
updatedAt: string;
|
||||
};
|
||||
|
||||
const SAFE_ID = /^[\w\u4e00-\u9fff-]+$/;
|
||||
|
||||
function stateDir(): string {
|
||||
return path.join(getPresetsDir(), "lab-state");
|
||||
}
|
||||
|
||||
function stateFile(presetId: string): string {
|
||||
if (!SAFE_ID.test(presetId)) throw new Error(`非法预设 id：${presetId}`);
|
||||
return path.join(stateDir(), `${presetId}.json`);
|
||||
}
|
||||
|
||||
function emptyState(presetId: string): PresetLabState {
|
||||
return { presetId, selectedCaseId: null, previewCaseId: null, updatedAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
export function loadPresetLabState(presetId: string): PresetLabState {
|
||||
const file = stateFile(presetId);
|
||||
if (!existsSync(file)) return emptyState(presetId);
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(file, "utf8")) as Partial<PresetLabState>;
|
||||
return {
|
||||
presetId,
|
||||
selectedCaseId: typeof parsed.selectedCaseId === "string" ? parsed.selectedCaseId : null,
|
||||
previewCaseId: typeof parsed.previewCaseId === "string" ? parsed.previewCaseId : null,
|
||||
updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : new Date().toISOString(),
|
||||
};
|
||||
} catch {
|
||||
return emptyState(presetId);
|
||||
}
|
||||
}
|
||||
|
||||
/** 只写传进来的字段，没传的保持原样。 */
|
||||
export function savePresetLabState(
|
||||
presetId: string,
|
||||
patch: { selectedCaseId?: string | null; previewCaseId?: string | null },
|
||||
): PresetLabState {
|
||||
const current = loadPresetLabState(presetId);
|
||||
const next: PresetLabState = {
|
||||
...current,
|
||||
selectedCaseId:
|
||||
patch.selectedCaseId === undefined ? current.selectedCaseId : patch.selectedCaseId,
|
||||
previewCaseId:
|
||||
patch.previewCaseId === undefined ? current.previewCaseId : patch.previewCaseId,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
mkdirSync(stateDir(), { recursive: true });
|
||||
writeFileSync(stateFile(presetId), JSON.stringify(next, null, 2), "utf8");
|
||||
return next;
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { PresetPackage } from "../types/preset.js";
|
||||
|
||||
/**
|
||||
* 工作台新建一份原生预设。
|
||||
* 骨架给一条可编辑规则 + 完整的扩展 marker 顺序，方便直接在页面上
|
||||
* 调整位置；applyExtendedMarkers 调用时仍是幂等的。
|
||||
*/
|
||||
export function createNativePreset(input: {
|
||||
id?: string | null;
|
||||
name?: string | null;
|
||||
} = {}): PresetPackage {
|
||||
const id = input.id?.trim() || `preset-${randomUUID().slice(0, 8)}`;
|
||||
const name = input.name?.trim() || "未命名预设";
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
source: "native",
|
||||
prompts: [
|
||||
{
|
||||
id: "system-rules",
|
||||
name: "写作规则",
|
||||
enabled: true,
|
||||
role: "system",
|
||||
content:
|
||||
"你是中文小说叙事者。按用户输入续写故事，保持人物、设定与前文一致；只输出正文，不解释写作过程。",
|
||||
marker: false,
|
||||
sourceIdentifier: "system-rules",
|
||||
},
|
||||
{
|
||||
id: "charDescription",
|
||||
name: "角色描述",
|
||||
enabled: true,
|
||||
role: "system",
|
||||
content: "",
|
||||
marker: true,
|
||||
sourceIdentifier: "charDescription",
|
||||
},
|
||||
{
|
||||
id: "personaDescription",
|
||||
name: "用户人设",
|
||||
enabled: true,
|
||||
role: "system",
|
||||
content: "",
|
||||
marker: true,
|
||||
sourceIdentifier: "personaDescription",
|
||||
},
|
||||
{
|
||||
id: "worldBookBefore",
|
||||
name: "世界书（历史前）",
|
||||
enabled: true,
|
||||
role: "system",
|
||||
content: "",
|
||||
marker: true,
|
||||
sourceIdentifier: "worldBookBefore",
|
||||
},
|
||||
{
|
||||
id: "chatHistory",
|
||||
name: "对话历史",
|
||||
enabled: true,
|
||||
role: "system",
|
||||
content: "",
|
||||
marker: true,
|
||||
sourceIdentifier: "chatHistory",
|
||||
},
|
||||
{
|
||||
id: "worldBookAfter",
|
||||
name: "世界书（历史后）",
|
||||
enabled: true,
|
||||
role: "system",
|
||||
content: "",
|
||||
marker: true,
|
||||
sourceIdentifier: "worldBookAfter",
|
||||
},
|
||||
{
|
||||
id: "currentTurn",
|
||||
name: "本轮对话",
|
||||
enabled: true,
|
||||
role: "user",
|
||||
content: "",
|
||||
marker: true,
|
||||
sourceIdentifier: "currentTurn",
|
||||
},
|
||||
{
|
||||
id: "postTurn",
|
||||
name: "本轮之后",
|
||||
enabled: true,
|
||||
role: "system",
|
||||
content: "",
|
||||
marker: true,
|
||||
sourceIdentifier: "postTurn",
|
||||
},
|
||||
],
|
||||
promptOrder: [
|
||||
{ promptId: "system-rules", enabled: true, orderIndex: 0 },
|
||||
{ promptId: "charDescription", enabled: true, orderIndex: 1 },
|
||||
{ promptId: "personaDescription", enabled: true, orderIndex: 2 },
|
||||
{ promptId: "worldBookBefore", enabled: true, orderIndex: 3 },
|
||||
{ promptId: "chatHistory", enabled: true, orderIndex: 4 },
|
||||
{ promptId: "worldBookAfter", enabled: true, orderIndex: 5 },
|
||||
{ promptId: "currentTurn", enabled: true, orderIndex: 6 },
|
||||
{ promptId: "postTurn", enabled: true, orderIndex: 7 },
|
||||
],
|
||||
generation: {},
|
||||
unsupported: [],
|
||||
importedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
+7
-2
@@ -19,6 +19,11 @@ function presetPath(id: string): string {
|
||||
return path.join(getPresetsDir(), `${id}.json`);
|
||||
}
|
||||
|
||||
/** Windows 编辑器可能给 JSON 加 BOM；JSON.parse 不认，先剥掉。 */
|
||||
function parsePreset(raw: string): PresetPackage {
|
||||
return JSON.parse(raw.replace(/^\uFEFF/, "")) as PresetPackage;
|
||||
}
|
||||
|
||||
export function listPresets(): PresetPackage[] {
|
||||
ensureUserDataDirs();
|
||||
let files: string[];
|
||||
@@ -33,7 +38,7 @@ export function listPresets(): PresetPackage[] {
|
||||
try {
|
||||
const filePath = path.join(getPresetsDir(), file);
|
||||
const raw = readFileSync(filePath, "utf8");
|
||||
presets.push(withoutReasoningEffort(JSON.parse(raw) as PresetPackage, filePath));
|
||||
presets.push(withoutReasoningEffort(parsePreset(raw), filePath));
|
||||
} catch {
|
||||
/* skip corrupt files */
|
||||
}
|
||||
@@ -45,7 +50,7 @@ export function getPreset(id: string): PresetPackage | null {
|
||||
try {
|
||||
const filePath = presetPath(id);
|
||||
const raw = readFileSync(filePath, "utf8");
|
||||
return withoutReasoningEffort(JSON.parse(raw) as PresetPackage, filePath);
|
||||
return withoutReasoningEffort(parsePreset(raw), filePath);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { getPresetsDir } from "../config/user-data-dir.js";
|
||||
|
||||
/**
|
||||
* 测试用例：**全局，不和预设绑定**——一个用例只描述「要测的输入 + 上下文」，
|
||||
* 任何预设都能拿它去跑。存在 `<data>/presets/tests/global/` 下，每组一个 JSON 文件
|
||||
* （沿用 TestCaseGroup 形状，旧的按预设分目录的文件会在第一次访问时并进来）。
|
||||
*
|
||||
* 每个用例只包含变动项：角色名与描述（{{char}}）、世界书、用户输入、可选历史。
|
||||
*/
|
||||
export type TestCaseHistoryItem = { role: "user" | "assistant"; content: string };
|
||||
|
||||
export type TestCase = {
|
||||
id: string;
|
||||
name: string;
|
||||
charName: string;
|
||||
charDescription: string;
|
||||
worldBook: string;
|
||||
userInput: string;
|
||||
history: TestCaseHistoryItem[];
|
||||
};
|
||||
|
||||
export type TestCaseGroup = {
|
||||
id: string;
|
||||
/** 历史字段：现在恒为 GLOBAL_CASES_ID，留着只是为了旧文件能直接读。 */
|
||||
presetId: string;
|
||||
name: string;
|
||||
updatedAt: string;
|
||||
cases: TestCase[];
|
||||
};
|
||||
|
||||
export const GLOBAL_CASES_ID = "global";
|
||||
|
||||
const SAFE_ID = /^[\w\u4e00-\u9fff-]+$/;
|
||||
|
||||
function assertSafe(id: string, label: string): void {
|
||||
if (!SAFE_ID.test(id)) throw new Error(`非法${label}：${id}`);
|
||||
}
|
||||
|
||||
function casesRoot(): string {
|
||||
return path.join(getPresetsDir(), "tests");
|
||||
}
|
||||
|
||||
function globalDir(): string {
|
||||
return path.join(casesRoot(), GLOBAL_CASES_ID);
|
||||
}
|
||||
|
||||
function groupFile(groupId: string): string {
|
||||
assertSafe(groupId, "用例 id");
|
||||
return path.join(globalDir(), `${groupId}.json`);
|
||||
}
|
||||
|
||||
/**
|
||||
* 老版本把用例按预设分目录存（`tests/<presetId>/*.json`）。全局目录第一次被用到时，
|
||||
* 把那些文件并进来（id 冲突就跳过），原文件不删，方便回退。
|
||||
*/
|
||||
function ensureGlobalDir(): string {
|
||||
const dir = globalDir();
|
||||
if (existsSync(dir)) return dir;
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const root = casesRoot();
|
||||
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || entry.name === GLOBAL_CASES_ID) continue;
|
||||
const legacyDir = path.join(root, entry.name);
|
||||
for (const file of readdirSync(legacyDir).filter((name) => name.endsWith(".json"))) {
|
||||
const target = path.join(dir, file);
|
||||
if (existsSync(target)) continue;
|
||||
try {
|
||||
const group = JSON.parse(readFileSync(path.join(legacyDir, file), "utf8")) as TestCaseGroup;
|
||||
writeFileSync(
|
||||
target,
|
||||
JSON.stringify({ ...group, presetId: GLOBAL_CASES_ID }, null, 2),
|
||||
"utf8",
|
||||
);
|
||||
} catch {
|
||||
/* 坏文件跳过 */
|
||||
}
|
||||
}
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
export function listTestGroups(): TestCaseGroup[] {
|
||||
const dir = ensureGlobalDir();
|
||||
const groups: TestCaseGroup[] = [];
|
||||
for (const file of readdirSync(dir).filter((name) => name.endsWith(".json"))) {
|
||||
try {
|
||||
groups.push(JSON.parse(readFileSync(path.join(dir, file), "utf8")) as TestCaseGroup);
|
||||
} catch {
|
||||
/* skip corrupt */
|
||||
}
|
||||
}
|
||||
return groups.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
||||
}
|
||||
|
||||
export function getTestGroup(groupId: string): TestCaseGroup | null {
|
||||
try {
|
||||
return JSON.parse(readFileSync(groupFile(groupId), "utf8")) as TestCaseGroup;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** 新建或覆盖一组。cases 中缺 id 的会分配 id。 */
|
||||
export function saveTestGroup(input: {
|
||||
id?: string;
|
||||
name: string;
|
||||
cases: Array<Partial<TestCase>>;
|
||||
}): TestCaseGroup {
|
||||
const id = input.id?.trim() || randomUUID().slice(0, 8);
|
||||
const text = (value: unknown): string => (typeof value === "string" ? value : "");
|
||||
const cases: TestCase[] = input.cases.map((c) => ({
|
||||
id: text(c.id).trim() || randomUUID().slice(0, 8),
|
||||
name: text(c.name).trim() || "未命名用例",
|
||||
charName: text(c.charName).trim(),
|
||||
charDescription: text(c.charDescription),
|
||||
worldBook: text(c.worldBook),
|
||||
userInput: text(c.userInput),
|
||||
history: (Array.isArray(c.history) ? c.history : []).filter(
|
||||
(h) => (h.role === "user" || h.role === "assistant") && typeof h.content === "string",
|
||||
),
|
||||
}));
|
||||
const group: TestCaseGroup = {
|
||||
id,
|
||||
presetId: GLOBAL_CASES_ID,
|
||||
name: input.name.trim() || "未命名测试组",
|
||||
updatedAt: new Date().toISOString(),
|
||||
cases,
|
||||
};
|
||||
ensureGlobalDir();
|
||||
writeFileSync(groupFile(id), JSON.stringify(group, null, 2), "utf8");
|
||||
return group;
|
||||
}
|
||||
|
||||
export function deleteTestGroup(groupId: string): void {
|
||||
try {
|
||||
unlinkSync(groupFile(groupId));
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
|
||||
/* ---------- 用例（工作台的展示单位） ----------
|
||||
* 盘上仍是一组一个文件，但界面上不出现「组」：每份用例都是一条独立记录，
|
||||
* 自己存、自己运行。新建用例会为它开一个只装一份用例的组文件。
|
||||
*/
|
||||
|
||||
export type TestCaseRef = { groupId: string; case: TestCase };
|
||||
|
||||
/** 平铺所有用例（按组更新时间排序，组内保持原顺序）。 */
|
||||
export function listTestCases(): TestCaseRef[] {
|
||||
const out: TestCaseRef[] = [];
|
||||
for (const group of listTestGroups()) {
|
||||
for (const item of group.cases) out.push({ groupId: group.id, case: item });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存单份用例：给了 groupId 就写回那个组（组内其他用例不动），
|
||||
* 没给就新建一个只装它的组。
|
||||
*/
|
||||
export function saveTestCase(input: {
|
||||
groupId?: string;
|
||||
case: Partial<TestCase>;
|
||||
}): TestCaseRef {
|
||||
const groupId = input.groupId?.trim();
|
||||
if (!groupId) {
|
||||
const name = typeof input.case.name === "string" ? input.case.name.trim() : "";
|
||||
const group = saveTestGroup({
|
||||
name: name || "未命名用例",
|
||||
cases: [input.case],
|
||||
});
|
||||
return { groupId: group.id, case: group.cases[0]! };
|
||||
}
|
||||
|
||||
const group = getTestGroup(groupId);
|
||||
if (!group) throw new Error(`用例不存在：${groupId}`);
|
||||
const caseId = typeof input.case.id === "string" ? input.case.id.trim() : "";
|
||||
const index = group.cases.findIndex((item) => item.id === caseId);
|
||||
const cases =
|
||||
index >= 0
|
||||
? group.cases.map((item, i) => (i === index ? { ...item, ...input.case } : item))
|
||||
: [...group.cases, input.case];
|
||||
// 只剩一份用例时，组名跟着用例名走，盘上读起来不至于两个名字打架。
|
||||
const single = cases.length === 1;
|
||||
const caseName = typeof input.case.name === "string" ? input.case.name.trim() : "";
|
||||
const saved = saveTestGroup({
|
||||
id: groupId,
|
||||
name: single && caseName ? caseName : group.name,
|
||||
cases,
|
||||
});
|
||||
const wanted = index >= 0 ? caseId : saved.cases[saved.cases.length - 1]!.id;
|
||||
return {
|
||||
groupId,
|
||||
case: saved.cases.find((item) => item.id === wanted) ?? saved.cases[0]!,
|
||||
};
|
||||
}
|
||||
|
||||
/** 删除单份用例；组里没别的人就把组文件一起删掉。 */
|
||||
export function deleteTestCase(groupId: string, caseId: string): void {
|
||||
const group = getTestGroup(groupId);
|
||||
if (!group) return;
|
||||
const cases = group.cases.filter((item) => item.id !== caseId);
|
||||
if (cases.length === 0) deleteTestGroup(groupId);
|
||||
else saveTestGroup({ id: groupId, name: group.name, cases });
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { getApiProfile, listApiProfiles, profileToLlmConfig } from "../config/api-profiles.js";
|
||||
import { OpenAiCompatibleProvider, type ChatMessage, type TokenUsage } from "../llm/client.js";
|
||||
import { assemblePlayWorkerMessages, shapeFramedMessages } from "./play-frame.js";
|
||||
import { resolveSquashForPreset } from "../llm/preset-wrapper.js";
|
||||
import { checkAuthoringPreset, type AuthoringIssue } from "./authoring-check.js";
|
||||
import type { PresetPackage } from "../types/preset.js";
|
||||
import type { MessageSquashMode } from "../types/preset.js";
|
||||
import type { TestCase } from "./test-cases.js";
|
||||
import type { WorldInfoPack } from "./world-info-pack.js";
|
||||
|
||||
/**
|
||||
* 按真实发送路径组装一个用例的 messages（不调用模型）。
|
||||
* 与线上一致：预设 → assemblePlayWorkerMessages → 消息整形。
|
||||
* 变动的只有 char 描述、世界书、用户输入、历史。
|
||||
*/
|
||||
export function buildCaseMessages(preset: PresetPackage, testCase: TestCase): ChatMessage[] {
|
||||
return shapeFramedMessages(
|
||||
assembleCaseMessages(preset, testCase),
|
||||
resolveSquashForPreset(preset, "follow-preset"),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 工作台右侧预览：同一组装结果，多告诉页面两件事——
|
||||
* 本次用的整形模式、整形前的条数。左轨是真实顺序，整形后条数变少要能看出来。
|
||||
*/
|
||||
export function buildCasePreview(
|
||||
preset: PresetPackage,
|
||||
testCase: TestCase,
|
||||
): { messages: ChatMessage[]; squash: MessageSquashMode; assembledCount: number } {
|
||||
const framed = assembleCaseMessages(preset, testCase);
|
||||
const squash = resolveSquashForPreset(preset, "follow-preset");
|
||||
return {
|
||||
messages: shapeFramedMessages(framed, squash),
|
||||
squash,
|
||||
assembledCount: framed.length,
|
||||
};
|
||||
}
|
||||
|
||||
/** 装配那一步本身（右栏与试跑共用，整形在调用方决定）。 */
|
||||
function assembleCaseMessages(
|
||||
preset: PresetPackage,
|
||||
testCase: TestCase,
|
||||
): ChatMessage[] {
|
||||
const asText = (value: unknown): string => (typeof value === "string" ? value : "");
|
||||
const historyInput = (Array.isArray(testCase.history) ? testCase.history : []).filter(
|
||||
(item) => item && typeof item === "object",
|
||||
);
|
||||
const history: ChatMessage[] = historyInput.map((h) => ({
|
||||
role: h.role === "assistant" ? "assistant" : "user",
|
||||
content: typeof h.content === "string" ? h.content : "",
|
||||
}));
|
||||
const worldBook = asText(testCase.worldBook);
|
||||
const charDescription = asText(testCase.charDescription);
|
||||
const pack: WorldInfoPack = {
|
||||
worldBookBefore: worldBook.trim()
|
||||
? [{ id: "test-world", name: "世界书", content: worldBook }]
|
||||
: [],
|
||||
history,
|
||||
worldBookAfter: [],
|
||||
turn: asText(testCase.userInput),
|
||||
postTurn: [],
|
||||
card: charDescription.trim() ? { description: charDescription } : undefined,
|
||||
};
|
||||
const framed = assemblePlayWorkerMessages({
|
||||
systemPrompt: "",
|
||||
preset,
|
||||
pack,
|
||||
persona: null,
|
||||
charName: asText(testCase.charName) || null,
|
||||
});
|
||||
return framed;
|
||||
}
|
||||
|
||||
export type RunStatus = "running" | "done" | "error";
|
||||
|
||||
export type CaseRunResult = {
|
||||
profileId: string;
|
||||
profileName: string;
|
||||
model: string;
|
||||
caseId: string;
|
||||
caseName?: string;
|
||||
status: RunStatus;
|
||||
content?: string;
|
||||
error?: string;
|
||||
usage?: TokenUsage;
|
||||
durationMs?: number;
|
||||
/** 手动标记：null 表示未标记 */
|
||||
verdict: "success" | "failure" | null;
|
||||
startedAt: string;
|
||||
finishedAt?: string;
|
||||
};
|
||||
|
||||
export type TestRun = {
|
||||
id: string;
|
||||
presetId: string;
|
||||
presetVersionId: string;
|
||||
groupId: string;
|
||||
createdAt: string;
|
||||
/** 本次运行时的消息快照（每个用例一份），便于事后核对发送内容 */
|
||||
requests: Array<{ caseId: string; messages: ChatMessage[] }>;
|
||||
results: CaseRunResult[];
|
||||
issues: AuthoringIssue[];
|
||||
};
|
||||
|
||||
export const TEST_RUN_TIMEOUT_MS = 180_000;
|
||||
const DEFAULT_CONCURRENCY = 4;
|
||||
|
||||
/** 小并发池：多模型 × 多用例时不要一次全打出去。 */
|
||||
async function runPool(jobs: Array<() => Promise<void>>, limit: number): Promise<void> {
|
||||
const queue = [...jobs];
|
||||
const width = Math.max(1, Math.min(limit, queue.length));
|
||||
const workers = Array.from({ length: width }, async () => {
|
||||
while (queue.length > 0) {
|
||||
const job = queue.shift();
|
||||
if (job) await job();
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
}
|
||||
|
||||
/**
|
||||
* 对一组用例、多个 API 配置并行运行。
|
||||
* 同一用例的同一配置只发一次；各配置之间并行。
|
||||
*/
|
||||
export async function runTestGroup(input: {
|
||||
preset: PresetPackage;
|
||||
presetVersionId: string;
|
||||
presetId: string;
|
||||
groupId: string;
|
||||
cases: TestCase[];
|
||||
profileIds: string[];
|
||||
timeoutMs?: number;
|
||||
concurrency?: number;
|
||||
}): Promise<TestRun> {
|
||||
const profileIds = [...new Set(input.profileIds.map((x) => x.trim()).filter(Boolean))];
|
||||
if (profileIds.length === 0) throw new Error("请至少选择一个 API 配置");
|
||||
if (input.cases.length === 0) throw new Error("测试组中没有用例");
|
||||
|
||||
const profiles = profileIds.map((id) => {
|
||||
const profile = getApiProfile(id);
|
||||
if (!profile?.apiKey?.trim()) throw new Error(`API 配置不可用：${id}`);
|
||||
return profile;
|
||||
});
|
||||
|
||||
const requests = input.cases.map((c) => ({ caseId: c.id, messages: buildCaseMessages(input.preset, c) }));
|
||||
const run: TestRun = {
|
||||
id: randomUUID().slice(0, 8),
|
||||
presetId: input.presetId,
|
||||
presetVersionId: input.presetVersionId,
|
||||
groupId: input.groupId,
|
||||
createdAt: new Date().toISOString(),
|
||||
requests,
|
||||
results: [],
|
||||
issues: checkAuthoringPreset(input.preset),
|
||||
};
|
||||
|
||||
const timeoutMs = Math.max(1_000, input.timeoutMs ?? TEST_RUN_TIMEOUT_MS);
|
||||
const jobs: Array<() => Promise<void>> = [];
|
||||
for (const profile of profiles) {
|
||||
for (const req of requests) {
|
||||
const testCase = input.cases.find((c) => c.id === req.caseId);
|
||||
const slot: CaseRunResult = {
|
||||
profileId: profile.id,
|
||||
profileName: profile.name,
|
||||
model: profile.model,
|
||||
caseId: req.caseId,
|
||||
caseName: testCase?.name,
|
||||
status: "running",
|
||||
verdict: null,
|
||||
startedAt: new Date().toISOString(),
|
||||
};
|
||||
run.results.push(slot);
|
||||
jobs.push(async () => {
|
||||
const started = Date.now();
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const llm = new OpenAiCompatibleProvider(profileToLlmConfig(profile));
|
||||
const result = await llm.complete(req.messages, {
|
||||
generation: input.preset.generation,
|
||||
caller: `preset-test:${profile.id}`,
|
||||
signal: controller.signal,
|
||||
});
|
||||
slot.content = result.content ?? "";
|
||||
if (result.model) slot.model = result.model;
|
||||
if (result.usage) slot.usage = result.usage;
|
||||
slot.status = "done";
|
||||
} catch (err) {
|
||||
slot.status = "error";
|
||||
if (controller.signal.aborted) {
|
||||
slot.error = `超时：${Math.round(timeoutMs / 1000)} 秒内没有完成`;
|
||||
} else {
|
||||
slot.error = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
slot.durationMs = Date.now() - started;
|
||||
slot.finishedAt = new Date().toISOString();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
await runPool(jobs, input.concurrency ?? DEFAULT_CONCURRENCY);
|
||||
return run;
|
||||
}
|
||||
|
||||
export function listRunnableProfiles(): Array<{ id: string; name: string; model: string; hasKey: boolean }> {
|
||||
return listApiProfiles().map((p) => ({
|
||||
id: p.id,
|
||||
name: p.name,
|
||||
model: p.model,
|
||||
hasKey: Boolean(p.apiKey?.trim()),
|
||||
}));
|
||||
}
|
||||
|
||||
/** 手动标记结果：只保存标签，不做自动评分。 */
|
||||
export function setVerdict(run: TestRun, profileId: string, caseId: string, verdict: "success" | "failure" | null): TestRun {
|
||||
const slot = run.results.find((r) => r.profileId === profileId && r.caseId === caseId);
|
||||
if (!slot) throw new Error("结果不存在");
|
||||
slot.verdict = verdict;
|
||||
return run;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { getPresetsDir } from "../config/user-data-dir.js";
|
||||
import type { TestRun } from "./test-run.js";
|
||||
|
||||
/** 运行记录：presets/runs/<presetId>/<runId>.json，保存预设版本 id 以便追溯。 */
|
||||
const SAFE_ID = /^[\w\u4e00-\u9fff-]+$/;
|
||||
|
||||
function runDir(presetId: string): string {
|
||||
if (!SAFE_ID.test(presetId)) throw new Error(`非法预设 id：${presetId}`);
|
||||
return path.join(getPresetsDir(), "runs", presetId);
|
||||
}
|
||||
|
||||
function runFile(presetId: string, runId: string): string {
|
||||
if (!SAFE_ID.test(runId)) throw new Error(`非法运行 id：${runId}`);
|
||||
return path.join(runDir(presetId), `${runId}.json`);
|
||||
}
|
||||
|
||||
export function saveTestRun(run: TestRun): void {
|
||||
mkdirSync(runDir(run.presetId), { recursive: true });
|
||||
writeFileSync(runFile(run.presetId, run.id), JSON.stringify(run, null, 2), "utf8");
|
||||
}
|
||||
|
||||
export function getTestRun(presetId: string, runId: string): TestRun | null {
|
||||
try {
|
||||
return JSON.parse(readFileSync(runFile(presetId, runId), "utf8")) as TestRun;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function listTestRuns(presetId: string): TestRun[] {
|
||||
const dir = runDir(presetId);
|
||||
if (!existsSync(dir)) return [];
|
||||
const out: TestRun[] = [];
|
||||
for (const file of readdirSync(dir).filter((f) => f.endsWith(".json"))) {
|
||||
try {
|
||||
out.push(JSON.parse(readFileSync(path.join(dir, file), "utf8")) as TestRun);
|
||||
} catch {
|
||||
/* skip */
|
||||
}
|
||||
}
|
||||
return out.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
||||
}
|
||||
|
||||
/** 修改某个结果的手动标记并落盘。 */
|
||||
export function updateVerdict(
|
||||
presetId: string,
|
||||
runId: string,
|
||||
profileId: string,
|
||||
caseId: string,
|
||||
verdict: "success" | "failure" | null,
|
||||
): TestRun {
|
||||
const run = getTestRun(presetId, runId);
|
||||
if (!run) throw new Error("运行记录不存在");
|
||||
const slot = run.results.find((r) => r.profileId === profileId && r.caseId === caseId);
|
||||
if (!slot) throw new Error("结果不存在");
|
||||
slot.verdict = verdict;
|
||||
saveTestRun(run);
|
||||
return run;
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { getPresetsDir } from "../config/user-data-dir.js";
|
||||
import type { PresetPackage } from "../types/preset.js";
|
||||
|
||||
/**
|
||||
* 预设版本树：每个预设一个 JSON 文件，节点是完整快照。
|
||||
* - 保存：在指定父节点（缺省为当前节点）下追加子节点；
|
||||
* - 分叉：父节点选历史中的任意一个，形成新分支，原分支不动；
|
||||
* - 追溯：沿 parentId 往上走即为修改历史。
|
||||
* 不依赖 git，不做远程同步。
|
||||
*/
|
||||
export type PresetVersionNode = {
|
||||
id: string;
|
||||
parentId: string | null;
|
||||
createdAt: string;
|
||||
/** 被「保存至当前版本」覆盖过的时间；没改过就不写 */
|
||||
updatedAt?: string;
|
||||
note: string;
|
||||
preset: PresetPackage;
|
||||
};
|
||||
|
||||
export type PresetVersionTree = {
|
||||
presetId: string;
|
||||
currentId: string | null;
|
||||
nodes: PresetVersionNode[];
|
||||
};
|
||||
|
||||
const SAFE_ID = /^[\w\u4e00-\u9fff-]+$/;
|
||||
|
||||
function versionsDir(): string {
|
||||
return path.join(getPresetsDir(), "versions");
|
||||
}
|
||||
|
||||
function treeFile(presetId: string): string {
|
||||
if (!SAFE_ID.test(presetId)) throw new Error(`非法预设 id：${presetId}`);
|
||||
return path.join(versionsDir(), `${presetId}.json`);
|
||||
}
|
||||
|
||||
function writeTree(tree: PresetVersionTree): void {
|
||||
mkdirSync(versionsDir(), { recursive: true });
|
||||
writeFileSync(treeFile(tree.presetId), JSON.stringify(tree, null, 2), "utf8");
|
||||
}
|
||||
|
||||
export function loadVersionTree(presetId: string): PresetVersionTree {
|
||||
const file = treeFile(presetId);
|
||||
if (!existsSync(file)) return { presetId, currentId: null, nodes: [] };
|
||||
return JSON.parse(readFileSync(file, "utf8")) as PresetVersionTree;
|
||||
}
|
||||
|
||||
export function getVersionNode(
|
||||
presetId: string,
|
||||
nodeId: string,
|
||||
): PresetVersionNode | null {
|
||||
return loadVersionTree(presetId).nodes.find((n) => n.id === nodeId) ?? null;
|
||||
}
|
||||
|
||||
/** 从根到指定节点的路径（含两端），即该版本的修改历史。 */
|
||||
export function versionLineage(
|
||||
tree: PresetVersionTree,
|
||||
nodeId: string,
|
||||
): PresetVersionNode[] {
|
||||
const byId = new Map(tree.nodes.map((n) => [n.id, n]));
|
||||
const out: PresetVersionNode[] = [];
|
||||
let cursor = byId.get(nodeId) ?? null;
|
||||
const guard = new Set<string>();
|
||||
while (cursor && !guard.has(cursor.id)) {
|
||||
guard.add(cursor.id);
|
||||
out.unshift(cursor);
|
||||
cursor = cursor.parentId ? byId.get(cursor.parentId) ?? null : null;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* 提交一个版本。
|
||||
* parentId 缺省为当前节点；显式传入历史节点即为分叉。
|
||||
* 树为空时 parentId 必须为 null（建根）；已有根时不允许再建第二个根。
|
||||
*/
|
||||
export function commitVersion(input: {
|
||||
presetId: string;
|
||||
preset: PresetPackage;
|
||||
note?: string;
|
||||
parentId?: string | null;
|
||||
}): PresetVersionTree {
|
||||
const tree = loadVersionTree(input.presetId);
|
||||
const parentId =
|
||||
input.parentId !== undefined ? input.parentId : tree.currentId;
|
||||
|
||||
if (parentId === null) {
|
||||
if (tree.nodes.length > 0) throw new Error("已有版本，不能再新建根版本");
|
||||
} else if (!tree.nodes.some((n) => n.id === parentId)) {
|
||||
throw new Error(`父版本不存在：${parentId}`);
|
||||
}
|
||||
|
||||
let id = randomUUID().slice(0, 8);
|
||||
while (tree.nodes.some((n) => n.id === id)) id = randomUUID().slice(0, 8);
|
||||
const node: PresetVersionNode = {
|
||||
id,
|
||||
parentId,
|
||||
createdAt: new Date().toISOString(),
|
||||
note: (input.note ?? "").trim(),
|
||||
preset: structuredClone(input.preset),
|
||||
};
|
||||
tree.nodes.push(node);
|
||||
tree.currentId = node.id;
|
||||
writeTree(tree);
|
||||
return tree;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存到当前版本（amend）：把快照直接换掉 `currentId` 那个节点，不产生新节点。
|
||||
* 还没有任何版本时退化成建根。
|
||||
*/
|
||||
export function amendCurrentVersion(input: {
|
||||
presetId: string;
|
||||
preset: PresetPackage;
|
||||
note?: string;
|
||||
}): PresetVersionTree {
|
||||
const tree = loadVersionTree(input.presetId);
|
||||
if (!tree.currentId) return commitVersion({ ...input, parentId: null });
|
||||
const node = tree.nodes.find((item) => item.id === tree.currentId);
|
||||
if (!node) throw new Error(`当前版本不存在：${tree.currentId}`);
|
||||
node.preset = structuredClone(input.preset);
|
||||
const note = (input.note ?? "").trim();
|
||||
if (note) node.note = note;
|
||||
node.updatedAt = new Date().toISOString();
|
||||
writeTree(tree);
|
||||
return tree;
|
||||
}
|
||||
|
||||
/** 切换当前编辑位置（不产生新节点）。 */
|
||||
export function checkoutVersion(presetId: string, nodeId: string): PresetVersionTree {
|
||||
const tree = loadVersionTree(presetId);
|
||||
if (!tree.nodes.some((n) => n.id === nodeId)) {
|
||||
throw new Error(`版本不存在：${nodeId}`);
|
||||
}
|
||||
tree.currentId = nodeId;
|
||||
writeTree(tree);
|
||||
return tree;
|
||||
}
|
||||
|
||||
export function deleteVersionTree(presetId: string): void {
|
||||
try {
|
||||
rmSync(treeFile(presetId), { force: true });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user