import { randomUUID } from "node:crypto"; import { getApiProfile, listApiProfiles, profileToLlmConfig } from "../config/api-profiles.js"; import { OpenAiCompatibleProvider, type ChatMessage, type TokenUsage } from "../llm/client.js"; import { assemblePlayWorkerMessages, shapeFramedMessages } from "./play-frame.js"; import { resolveSquashForPreset } from "../llm/preset-wrapper.js"; import { checkAuthoringPreset, type AuthoringIssue } from "./authoring-check.js"; import type { PresetPackage } from "../types/preset.js"; import type { MessageSquashMode } from "../types/preset.js"; import type { TestCase } from "./test-cases.js"; import type { WorldInfoPack } from "./world-info-pack.js"; /** * 按真实发送路径组装一个用例的 messages(不调用模型)。 * 与线上一致:预设 → assemblePlayWorkerMessages → 消息整形。 * 变动的只有 char 描述、世界书、用户输入、历史。 */ export function buildCaseMessages(preset: PresetPackage, testCase: TestCase): ChatMessage[] { return shapeFramedMessages( assembleCaseMessages(preset, testCase), resolveSquashForPreset(preset, "follow-preset"), ); } /** * 工作台右侧预览:同一组装结果,多告诉页面两件事—— * 本次用的整形模式、整形前的条数。左轨是真实顺序,整形后条数变少要能看出来。 */ export function buildCasePreview( preset: PresetPackage, testCase: TestCase, ): { messages: ChatMessage[]; squash: MessageSquashMode; assembledCount: number } { const framed = assembleCaseMessages(preset, testCase); const squash = resolveSquashForPreset(preset, "follow-preset"); return { messages: shapeFramedMessages(framed, squash), squash, assembledCount: framed.length, }; } /** 装配那一步本身(右栏与试跑共用,整形在调用方决定)。 */ function assembleCaseMessages( preset: PresetPackage, testCase: TestCase, ): ChatMessage[] { const asText = (value: unknown): string => (typeof value === "string" ? value : ""); const historyInput = (Array.isArray(testCase.history) ? testCase.history : []).filter( (item) => item && typeof item === "object", ); const history: ChatMessage[] = historyInput.map((h) => ({ role: h.role === "assistant" ? "assistant" : "user", content: typeof h.content === "string" ? h.content : "", })); const worldBook = asText(testCase.worldBook); const charDescription = asText(testCase.charDescription); const pack: WorldInfoPack = { worldBookBefore: worldBook.trim() ? [{ id: "test-world", name: "世界书", content: worldBook }] : [], history, worldBookAfter: [], turn: asText(testCase.userInput), postTurn: [], card: charDescription.trim() ? { description: charDescription } : undefined, }; const framed = assemblePlayWorkerMessages({ systemPrompt: "", preset, pack, persona: null, charName: asText(testCase.charName) || null, }); return framed; } export type RunStatus = "running" | "done" | "error"; export type CaseRunResult = { profileId: string; profileName: string; model: string; caseId: string; caseName?: string; status: RunStatus; content?: string; error?: string; usage?: TokenUsage; durationMs?: number; /** 手动标记:null 表示未标记 */ verdict: "success" | "failure" | null; startedAt: string; finishedAt?: string; }; export type TestRun = { id: string; presetId: string; presetVersionId: string; groupId: string; createdAt: string; /** 本次运行时的消息快照(每个用例一份),便于事后核对发送内容 */ requests: Array<{ caseId: string; messages: ChatMessage[] }>; results: CaseRunResult[]; issues: AuthoringIssue[]; }; export const TEST_RUN_TIMEOUT_MS = 180_000; const DEFAULT_CONCURRENCY = 4; /** 小并发池:多模型 × 多用例时不要一次全打出去。 */ async function runPool(jobs: Array<() => Promise>, limit: number): Promise { const queue = [...jobs]; const width = Math.max(1, Math.min(limit, queue.length)); const workers = Array.from({ length: width }, async () => { while (queue.length > 0) { const job = queue.shift(); if (job) await job(); } }); await Promise.all(workers); } /** * 对一组用例、多个 API 配置并行运行。 * 同一用例的同一配置只发一次;各配置之间并行。 */ export async function runTestGroup(input: { preset: PresetPackage; presetVersionId: string; presetId: string; groupId: string; cases: TestCase[]; profileIds: string[]; timeoutMs?: number; concurrency?: number; }): Promise { const profileIds = [...new Set(input.profileIds.map((x) => x.trim()).filter(Boolean))]; if (profileIds.length === 0) throw new Error("请至少选择一个 API 配置"); if (input.cases.length === 0) throw new Error("测试组中没有用例"); const profiles = profileIds.map((id) => { const profile = getApiProfile(id); if (!profile?.apiKey?.trim()) throw new Error(`API 配置不可用:${id}`); return profile; }); const requests = input.cases.map((c) => ({ caseId: c.id, messages: buildCaseMessages(input.preset, c) })); const run: TestRun = { id: randomUUID().slice(0, 8), presetId: input.presetId, presetVersionId: input.presetVersionId, groupId: input.groupId, createdAt: new Date().toISOString(), requests, results: [], issues: checkAuthoringPreset(input.preset), }; const timeoutMs = Math.max(1_000, input.timeoutMs ?? TEST_RUN_TIMEOUT_MS); const jobs: Array<() => Promise> = []; for (const profile of profiles) { for (const req of requests) { const testCase = input.cases.find((c) => c.id === req.caseId); const slot: CaseRunResult = { profileId: profile.id, profileName: profile.name, model: profile.model, caseId: req.caseId, caseName: testCase?.name, status: "running", verdict: null, startedAt: new Date().toISOString(), }; run.results.push(slot); jobs.push(async () => { const started = Date.now(); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { const llm = new OpenAiCompatibleProvider(profileToLlmConfig(profile)); const result = await llm.complete(req.messages, { generation: input.preset.generation, caller: `preset-test:${profile.id}`, signal: controller.signal, }); slot.content = result.content ?? ""; if (result.model) slot.model = result.model; if (result.usage) slot.usage = result.usage; slot.status = "done"; } catch (err) { slot.status = "error"; if (controller.signal.aborted) { slot.error = `超时:${Math.round(timeoutMs / 1000)} 秒内没有完成`; } else { slot.error = err instanceof Error ? err.message : String(err); } } finally { clearTimeout(timer); slot.durationMs = Date.now() - started; slot.finishedAt = new Date().toISOString(); } }); } } await runPool(jobs, input.concurrency ?? DEFAULT_CONCURRENCY); return run; } export function listRunnableProfiles(): Array<{ id: string; name: string; model: string; hasKey: boolean }> { return listApiProfiles().map((p) => ({ id: p.id, name: p.name, model: p.model, hasKey: Boolean(p.apiKey?.trim()), })); } /** 手动标记结果:只保存标签,不做自动评分。 */ export function setVerdict(run: TestRun, profileId: string, caseId: string, verdict: "success" | "failure" | null): TestRun { const slot = run.results.find((r) => r.profileId === profileId && r.caseId === caseId); if (!slot) throw new Error("结果不存在"); slot.verdict = verdict; return run; }