新增预设工作台：编辑、版本树、结构检查、用例试跑放在同一页。
改预设要能一边写一边看装出来的 messages，还要多模型对照，所以把编辑与试跑拆成两个视图塞进一个页面；版本存完整快照、可分叉，用例全局共用一份、运行记录按预设存。顺带让读预设文件先剥 BOM，静态资源改成不缓存，免得改了脚本还要强刷。
This commit is contained in:
@@ -0,0 +1,224 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { getApiProfile, listApiProfiles, profileToLlmConfig } from "../config/api-profiles.js";
|
||||
import { OpenAiCompatibleProvider, type ChatMessage, type TokenUsage } from "../llm/client.js";
|
||||
import { assemblePlayWorkerMessages, shapeFramedMessages } from "./play-frame.js";
|
||||
import { resolveSquashForPreset } from "../llm/preset-wrapper.js";
|
||||
import { checkAuthoringPreset, type AuthoringIssue } from "./authoring-check.js";
|
||||
import type { PresetPackage } from "../types/preset.js";
|
||||
import type { MessageSquashMode } from "../types/preset.js";
|
||||
import type { TestCase } from "./test-cases.js";
|
||||
import type { WorldInfoPack } from "./world-info-pack.js";
|
||||
|
||||
/**
|
||||
* 按真实发送路径组装一个用例的 messages（不调用模型）。
|
||||
* 与线上一致：预设 → assemblePlayWorkerMessages → 消息整形。
|
||||
* 变动的只有 char 描述、世界书、用户输入、历史。
|
||||
*/
|
||||
export function buildCaseMessages(preset: PresetPackage, testCase: TestCase): ChatMessage[] {
|
||||
return shapeFramedMessages(
|
||||
assembleCaseMessages(preset, testCase),
|
||||
resolveSquashForPreset(preset, "follow-preset"),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 工作台右侧预览：同一组装结果，多告诉页面两件事——
|
||||
* 本次用的整形模式、整形前的条数。左轨是真实顺序，整形后条数变少要能看出来。
|
||||
*/
|
||||
export function buildCasePreview(
|
||||
preset: PresetPackage,
|
||||
testCase: TestCase,
|
||||
): { messages: ChatMessage[]; squash: MessageSquashMode; assembledCount: number } {
|
||||
const framed = assembleCaseMessages(preset, testCase);
|
||||
const squash = resolveSquashForPreset(preset, "follow-preset");
|
||||
return {
|
||||
messages: shapeFramedMessages(framed, squash),
|
||||
squash,
|
||||
assembledCount: framed.length,
|
||||
};
|
||||
}
|
||||
|
||||
/** 装配那一步本身（右栏与试跑共用，整形在调用方决定）。 */
|
||||
function assembleCaseMessages(
|
||||
preset: PresetPackage,
|
||||
testCase: TestCase,
|
||||
): ChatMessage[] {
|
||||
const asText = (value: unknown): string => (typeof value === "string" ? value : "");
|
||||
const historyInput = (Array.isArray(testCase.history) ? testCase.history : []).filter(
|
||||
(item) => item && typeof item === "object",
|
||||
);
|
||||
const history: ChatMessage[] = historyInput.map((h) => ({
|
||||
role: h.role === "assistant" ? "assistant" : "user",
|
||||
content: typeof h.content === "string" ? h.content : "",
|
||||
}));
|
||||
const worldBook = asText(testCase.worldBook);
|
||||
const charDescription = asText(testCase.charDescription);
|
||||
const pack: WorldInfoPack = {
|
||||
worldBookBefore: worldBook.trim()
|
||||
? [{ id: "test-world", name: "世界书", content: worldBook }]
|
||||
: [],
|
||||
history,
|
||||
worldBookAfter: [],
|
||||
turn: asText(testCase.userInput),
|
||||
postTurn: [],
|
||||
card: charDescription.trim() ? { description: charDescription } : undefined,
|
||||
};
|
||||
const framed = assemblePlayWorkerMessages({
|
||||
systemPrompt: "",
|
||||
preset,
|
||||
pack,
|
||||
persona: null,
|
||||
charName: asText(testCase.charName) || null,
|
||||
});
|
||||
return framed;
|
||||
}
|
||||
|
||||
export type RunStatus = "running" | "done" | "error";
|
||||
|
||||
export type CaseRunResult = {
|
||||
profileId: string;
|
||||
profileName: string;
|
||||
model: string;
|
||||
caseId: string;
|
||||
caseName?: string;
|
||||
status: RunStatus;
|
||||
content?: string;
|
||||
error?: string;
|
||||
usage?: TokenUsage;
|
||||
durationMs?: number;
|
||||
/** 手动标记：null 表示未标记 */
|
||||
verdict: "success" | "failure" | null;
|
||||
startedAt: string;
|
||||
finishedAt?: string;
|
||||
};
|
||||
|
||||
export type TestRun = {
|
||||
id: string;
|
||||
presetId: string;
|
||||
presetVersionId: string;
|
||||
groupId: string;
|
||||
createdAt: string;
|
||||
/** 本次运行时的消息快照（每个用例一份），便于事后核对发送内容 */
|
||||
requests: Array<{ caseId: string; messages: ChatMessage[] }>;
|
||||
results: CaseRunResult[];
|
||||
issues: AuthoringIssue[];
|
||||
};
|
||||
|
||||
export const TEST_RUN_TIMEOUT_MS = 180_000;
|
||||
const DEFAULT_CONCURRENCY = 4;
|
||||
|
||||
/** 小并发池：多模型 × 多用例时不要一次全打出去。 */
|
||||
async function runPool(jobs: Array<() => Promise<void>>, limit: number): Promise<void> {
|
||||
const queue = [...jobs];
|
||||
const width = Math.max(1, Math.min(limit, queue.length));
|
||||
const workers = Array.from({ length: width }, async () => {
|
||||
while (queue.length > 0) {
|
||||
const job = queue.shift();
|
||||
if (job) await job();
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
}
|
||||
|
||||
/**
|
||||
* 对一组用例、多个 API 配置并行运行。
|
||||
* 同一用例的同一配置只发一次；各配置之间并行。
|
||||
*/
|
||||
export async function runTestGroup(input: {
|
||||
preset: PresetPackage;
|
||||
presetVersionId: string;
|
||||
presetId: string;
|
||||
groupId: string;
|
||||
cases: TestCase[];
|
||||
profileIds: string[];
|
||||
timeoutMs?: number;
|
||||
concurrency?: number;
|
||||
}): Promise<TestRun> {
|
||||
const profileIds = [...new Set(input.profileIds.map((x) => x.trim()).filter(Boolean))];
|
||||
if (profileIds.length === 0) throw new Error("请至少选择一个 API 配置");
|
||||
if (input.cases.length === 0) throw new Error("测试组中没有用例");
|
||||
|
||||
const profiles = profileIds.map((id) => {
|
||||
const profile = getApiProfile(id);
|
||||
if (!profile?.apiKey?.trim()) throw new Error(`API 配置不可用：${id}`);
|
||||
return profile;
|
||||
});
|
||||
|
||||
const requests = input.cases.map((c) => ({ caseId: c.id, messages: buildCaseMessages(input.preset, c) }));
|
||||
const run: TestRun = {
|
||||
id: randomUUID().slice(0, 8),
|
||||
presetId: input.presetId,
|
||||
presetVersionId: input.presetVersionId,
|
||||
groupId: input.groupId,
|
||||
createdAt: new Date().toISOString(),
|
||||
requests,
|
||||
results: [],
|
||||
issues: checkAuthoringPreset(input.preset),
|
||||
};
|
||||
|
||||
const timeoutMs = Math.max(1_000, input.timeoutMs ?? TEST_RUN_TIMEOUT_MS);
|
||||
const jobs: Array<() => Promise<void>> = [];
|
||||
for (const profile of profiles) {
|
||||
for (const req of requests) {
|
||||
const testCase = input.cases.find((c) => c.id === req.caseId);
|
||||
const slot: CaseRunResult = {
|
||||
profileId: profile.id,
|
||||
profileName: profile.name,
|
||||
model: profile.model,
|
||||
caseId: req.caseId,
|
||||
caseName: testCase?.name,
|
||||
status: "running",
|
||||
verdict: null,
|
||||
startedAt: new Date().toISOString(),
|
||||
};
|
||||
run.results.push(slot);
|
||||
jobs.push(async () => {
|
||||
const started = Date.now();
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const llm = new OpenAiCompatibleProvider(profileToLlmConfig(profile));
|
||||
const result = await llm.complete(req.messages, {
|
||||
generation: input.preset.generation,
|
||||
caller: `preset-test:${profile.id}`,
|
||||
signal: controller.signal,
|
||||
});
|
||||
slot.content = result.content ?? "";
|
||||
if (result.model) slot.model = result.model;
|
||||
if (result.usage) slot.usage = result.usage;
|
||||
slot.status = "done";
|
||||
} catch (err) {
|
||||
slot.status = "error";
|
||||
if (controller.signal.aborted) {
|
||||
slot.error = `超时：${Math.round(timeoutMs / 1000)} 秒内没有完成`;
|
||||
} else {
|
||||
slot.error = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
slot.durationMs = Date.now() - started;
|
||||
slot.finishedAt = new Date().toISOString();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
await runPool(jobs, input.concurrency ?? DEFAULT_CONCURRENCY);
|
||||
return run;
|
||||
}
|
||||
|
||||
export function listRunnableProfiles(): Array<{ id: string; name: string; model: string; hasKey: boolean }> {
|
||||
return listApiProfiles().map((p) => ({
|
||||
id: p.id,
|
||||
name: p.name,
|
||||
model: p.model,
|
||||
hasKey: Boolean(p.apiKey?.trim()),
|
||||
}));
|
||||
}
|
||||
|
||||
/** 手动标记结果：只保存标签，不做自动评分。 */
|
||||
export function setVerdict(run: TestRun, profileId: string, caseId: string, verdict: "success" | "failure" | null): TestRun {
|
||||
const slot = run.results.find((r) => r.profileId === profileId && r.caseId === caseId);
|
||||
if (!slot) throw new Error("结果不存在");
|
||||
slot.verdict = verdict;
|
||||
return run;
|
||||
}
|
||||
Reference in New Issue
Block a user