168 lines
8.1 KiB
JavaScript
168 lines
8.1 KiB
JavaScript
// A/B：同一场景、同一模型、同参数，把多个预设各跑 N 轮，输出对比报告（走本仓库直聊的真实调用）。
|
||
// 用法: node ab.mjs --preset <a.json> --preset <b.json> [--runs 2] [--profile deepseek-flash] [--port 23388]
|
||
import { spawn } from "node:child_process";
|
||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||
import os from "node:os";
|
||
import path from "node:path";
|
||
import { fileURLToPath } from "node:url";
|
||
import { waitFor } from "./cdp.mjs";
|
||
|
||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||
const REPO = path.resolve(HERE, "../..");
|
||
const OUT = path.join(HERE, "captures");
|
||
|
||
function readArgs(argv) {
|
||
const out = { presets: [], runs: 2, profile: "deepseek-flash", port: 23388, dataRoot: path.join(HERE, ".ab-data"), timeoutMs: 240000 };
|
||
for (let i = 0; i < argv.length; i++) {
|
||
const k = argv[i], v = argv[i + 1];
|
||
if (k === "--preset") { out.presets.push(path.resolve(String(v))); i++; }
|
||
else if (k === "--runs") { out.runs = Number(v); i++; }
|
||
else if (k === "--profile") { out.profile = String(v); i++; }
|
||
else if (k === "--port") { out.port = Number(v); i++; }
|
||
else if (k === "--timeout") { out.timeoutMs = Number(v) * 1000; i++; }
|
||
}
|
||
if (out.presets.length < 2) throw new Error("至少要两个 --preset");
|
||
return out;
|
||
}
|
||
|
||
// 同一场景，两边共用
|
||
const SCENE = {
|
||
charName: "阿箬",
|
||
charDescription: "三十上下的客栈老板娘，眉眼有倦色，说话慢，手上总在做事。",
|
||
charPersonality: "沉静、寡言、看人极准，不轻易信人。",
|
||
scenario: "雨夜，城郊旧客栈。他是第一次独自押镖的年轻镖师，也是今夜唯一的客人。",
|
||
exampleDialogue: "",
|
||
text: "把刀搁在桌上，说：「上一壶热的。」",
|
||
};
|
||
|
||
async function postJson(url, body, timeoutMs) {
|
||
const ctrl = new AbortController();
|
||
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
||
try {
|
||
const res = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), signal: ctrl.signal });
|
||
const text = await res.text();
|
||
let data = null;
|
||
try { data = JSON.parse(text); } catch (e) { data = { raw: text.slice(0, 1200) }; }
|
||
if (!res.ok) throw new Error("HTTP " + res.status + " " + JSON.stringify(data).slice(0, 600));
|
||
return data;
|
||
} finally {
|
||
clearTimeout(timer);
|
||
}
|
||
}
|
||
|
||
function pickProfile(name) {
|
||
const file = path.join(os.homedir(), ".writing-agent", "profiles.json");
|
||
const data = JSON.parse(readFileSync(file, "utf8"));
|
||
const hit = (data.profiles || []).find((p) => p.name === name) || (data.profiles || []).find((p) => p.apiKey);
|
||
if (!hit) throw new Error("找不到可用 profile: " + name);
|
||
return hit;
|
||
}
|
||
|
||
async function main() {
|
||
const args = readArgs(process.argv.slice(2));
|
||
mkdirSync(OUT, { recursive: true });
|
||
const profile = pickProfile(args.profile);
|
||
const dataRoot = args.dataRoot;
|
||
rmSync(dataRoot, { recursive: true, force: true });
|
||
mkdirSync(dataRoot, { recursive: true });
|
||
|
||
const now = new Date().toISOString();
|
||
writeFileSync(path.join(dataRoot, "profiles.json"), JSON.stringify({
|
||
version: 1,
|
||
profiles: [{ id: "ab", name: "ab", baseUrl: profile.baseUrl, apiKey: profile.apiKey, model: profile.model, createdAt: now, updatedAt: now }],
|
||
groups: [],
|
||
}, null, 2), "utf8");
|
||
writeFileSync(path.join(dataRoot, "settings.json"), JSON.stringify({
|
||
version: 1, activeProfileId: "ab", activePresetId: null, messageSquash: "follow-preset", presetToolPolicy: "adapt",
|
||
}, null, 2), "utf8");
|
||
|
||
const env = { ...process.env, PORT: String(args.port), WRITING_AGENT_DATA_DIR: dataRoot };
|
||
const server = spawn(process.execPath, [path.join(REPO, "node_modules/tsx/dist/cli.mjs"), path.join(REPO, "src/server/web-server.ts")], { cwd: REPO, env, stdio: ["ignore", "pipe", "pipe"] });
|
||
const logs = [];
|
||
server.stdout.on("data", (d) => logs.push(String(d)));
|
||
server.stderr.on("data", (d) => logs.push(String(d)));
|
||
const base = "http://127.0.0.1:" + args.port;
|
||
|
||
const rows = [];
|
||
try {
|
||
await waitFor(async () => (await fetch(base + "/api/health").catch(() => null)) !== null, { timeoutMs: 60000, label: "服务起来" });
|
||
console.log("[ab] 模型: " + profile.model + " (" + profile.baseUrl + ") 跑 " + args.runs + " 轮/预设");
|
||
|
||
for (const file of args.presets) {
|
||
const name = path.basename(file, ".json");
|
||
const raw = JSON.parse(readFileSync(file, "utf8"));
|
||
const imported = await postJson(base + "/api/presets/import", { raw, name, activate: true }, 60000);
|
||
const presetId = imported.preset.id;
|
||
const promptChars = (raw.prompts || []).reduce((n, p) => n + String(p.content || "").length, 0);
|
||
console.log("[ab] " + name + ": 已导入 id=" + presetId + "，预设正文 " + promptChars + " 字");
|
||
|
||
for (let i = 1; i <= args.runs; i++) {
|
||
const t0 = Date.now();
|
||
let reply = "", err = "", usage = null;
|
||
try {
|
||
const result = await postJson(base + "/api/direct-chat/ephemeral/complete", {
|
||
text: SCENE.text, presetId, complete: true, turns: [],
|
||
windows: [{ id: "w1", profileId: "ab" }],
|
||
systemPrompt: "", loreBefore: "", seedHistory: "", loreAfter: "", postTurn: "",
|
||
charName: SCENE.charName, exampleDialogue: SCENE.exampleDialogue,
|
||
charDescription: SCENE.charDescription, charPersonality: SCENE.charPersonality,
|
||
scenario: SCENE.scenario,
|
||
}, args.timeoutMs);
|
||
const first = result.multi && result.multi.results && result.multi.results[0];
|
||
if (!first) throw new Error("没有 results: " + JSON.stringify(result).slice(0, 400));
|
||
if (first.error) { err = String(first.error); } else {
|
||
reply = String(first.reply || "");
|
||
usage = first.usage || result.usage || null;
|
||
}
|
||
} catch (e) {
|
||
err = e.message;
|
||
}
|
||
const ms = Date.now() - t0;
|
||
rows.push({ preset: name, run: i, ms, chars: reply.length, reply, err, usage, promptChars });
|
||
console.log("[ab] " + name + " 第" + i + "轮: " + ms + "ms " + reply.length + " 字" + (err ? " 失败: " + err.slice(0, 120) : ""));
|
||
}
|
||
}
|
||
} finally {
|
||
try { server.kill(); } catch (e) {}
|
||
}
|
||
|
||
// 从 stats 里补 token
|
||
let stats = [];
|
||
try {
|
||
const day = new Date().toISOString().slice(0, 10);
|
||
const file = path.join(dataRoot, "stats", day + ".jsonl");
|
||
stats = readFileSync(file, "utf8").trim().split("\n").filter(Boolean).map((l) => JSON.parse(l));
|
||
} catch (e) {}
|
||
|
||
const lines = ["# A/B 报告 " + new Date().toLocaleString("zh-CN"), ""];
|
||
lines.push("模型: `" + profile.model + "` ｜ 预设正文合计字数见下表 ｜ 每预设 " + args.runs + " 轮");
|
||
lines.push("");
|
||
lines.push("场景: " + SCENE.scenario);
|
||
lines.push("角色: " + SCENE.charName + " — " + SCENE.charDescription);
|
||
lines.push("用户输入: " + SCENE.text);
|
||
lines.push("");
|
||
lines.push("| 预设 | 预设正文字数 | 轮次 | 耗时(ms) | 输出字数 | prompt tokens | completion tokens | 结果 |");
|
||
lines.push("| --- | --- | --- | --- | --- | --- | --- | --- |");
|
||
rows.forEach((r, idx) => {
|
||
const s = stats[idx] || {};
|
||
lines.push("| " + r.preset + " | " + r.promptChars + " | " + r.run + " | " + r.ms + " | " + r.chars + " | " + (s.promptTokens ?? "—") + " | " + (s.completionTokens ?? "—") + " | " + (r.err ? "❌ " + r.err.slice(0, 60) : "✅") + " |");
|
||
});
|
||
for (const name of new Set(rows.map((r) => r.preset))) {
|
||
lines.push("");
|
||
lines.push("## " + name);
|
||
for (const r of rows.filter((x) => x.preset === name)) {
|
||
lines.push("");
|
||
lines.push("### 第 " + r.run + " 轮（" + r.ms + "ms，" + r.chars + " 字）");
|
||
lines.push("");
|
||
lines.push(r.err ? "失败: " + r.err : "```\n" + r.reply + "\n```");
|
||
}
|
||
}
|
||
const report = path.join(OUT, "ab-" + new Date().toISOString().replace(/[:.]/g, "-") + ".md");
|
||
writeFileSync(report, lines.join("\n") + "\n", "utf8");
|
||
console.log("\n" + lines.slice(5, 12 + rows.length).join("\n"));
|
||
console.log("\n[ab] 报告: " + report);
|
||
if (logs.some((l) => l.includes("失败"))) console.log("[ab] 服务端日志尾部:\n" + logs.join("").slice(-1500));
|
||
}
|
||
|
||
await main();
|