228 lines
9.7 KiB
JavaScript
228 lines
9.7 KiB
JavaScript
// 「100% 对齐」的可测量化：同一个预设，两边各发一轮，逐字段/逐条消息比对，输出对齐度与差异明细。
|
||
// 用法: node conform.mjs --preset "C:/path/x.json" [--st-port 8124] [--wa-port 23377] [--st-capture 8801] [--wa-capture 8802]
|
||
// 退出码: 0 = 完全对齐, 1 = 有差异
|
||
import { spawn } from "node:child_process";
|
||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||
import path from "node:path";
|
||
import { fileURLToPath } from "node:url";
|
||
import { waitFor } from "./cdp.mjs";
|
||
|
||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||
|
||
function readArgs(argv) {
|
||
const out = {
|
||
preset: "",
|
||
message: "你好",
|
||
stPort: 8124,
|
||
waPort: 23377,
|
||
stCapture: 8801,
|
||
waCapture: 8802,
|
||
out: path.join(HERE, "captures"),
|
||
};
|
||
for (let i = 0; i < argv.length; i++) {
|
||
const k = argv[i];
|
||
const v = argv[i + 1];
|
||
if (k === "--preset") { out.preset = path.resolve(String(v)); i++; }
|
||
else if (k === "--message") { out.message = String(v); i++; }
|
||
else if (k === "--st-port") { out.stPort = Number(v); i++; }
|
||
else if (k === "--wa-port") { out.waPort = Number(v); i++; }
|
||
else if (k === "--st-capture") { out.stCapture = Number(v); i++; }
|
||
else if (k === "--wa-capture") { out.waCapture = Number(v); i++; }
|
||
}
|
||
if (!out.preset) throw new Error("缺少 --preset");
|
||
return out;
|
||
}
|
||
|
||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||
|
||
function runScript(script, args) {
|
||
return new Promise((resolve) => {
|
||
const child = spawn(process.execPath, [path.join(HERE, script), ...args], { cwd: HERE, stdio: ["ignore", "pipe", "pipe"] });
|
||
let out = "";
|
||
const pipe = (c) => { const t = String(c); out += t; };
|
||
child.stdout.on("data", pipe);
|
||
child.stderr.on("data", pipe);
|
||
child.on("close", (code) => resolve({ code, out }));
|
||
});
|
||
}
|
||
|
||
function lastJsonLine(out) {
|
||
const lines = out.split(/\r?\n/).filter((l) => l.trim().startsWith("{") && l.includes("\"capture\""));
|
||
if (!lines.length) return null;
|
||
try { return JSON.parse(lines[lines.length - 1]); } catch (e) { return null; }
|
||
}
|
||
|
||
// ---- 比对 ----
|
||
const norm = (s) => String(s ?? "").replace(/\r/g, "");
|
||
function ratio(a, b) {
|
||
a = norm(a); b = norm(b);
|
||
if (a === b) return 1;
|
||
if (!a || !b) return 0;
|
||
const max = Math.max(a.length, b.length);
|
||
let same = 0;
|
||
const n = Math.min(a.length, b.length);
|
||
for (let i = 0; i < n; i++) if (a[i] === b[i]) same++;
|
||
return same / max;
|
||
}
|
||
const mark = (ok) => (ok ? "✅" : "❌");
|
||
|
||
function loadBody(file) {
|
||
const outer = JSON.parse(readFileSync(file, "utf8"));
|
||
return JSON.parse(outer.body);
|
||
}
|
||
|
||
const REQUEST_KEYS = [
|
||
"model", "temperature", "top_p", "max_tokens", "presence_penalty",
|
||
"frequency_penalty", "stop", "stream", "stream_options", "tools", "tool_choice",
|
||
"response_format", "reasoning_effort", "top_k", "min_p", "repetition_penalty",
|
||
];
|
||
|
||
function compareRequest(st, wa) {
|
||
const rows = [];
|
||
const keys = new Set([...Object.keys(st), ...Object.keys(wa)]);
|
||
for (const k of REQUEST_KEYS) if (!(k in st) && !(k in wa)) continue;
|
||
for (const k of keys) {
|
||
if (k === "messages") continue;
|
||
if (!REQUEST_KEYS.includes(k)) {
|
||
rows.push({ key: k, st: "<有>", wa: "<有>", ok: false, note: "未纳入比对的字段" });
|
||
continue;
|
||
}
|
||
const a = JSON.stringify(st[k]);
|
||
const b = JSON.stringify(wa[k]);
|
||
rows.push({ key: k, st: a === undefined ? "(无)" : a, wa: b === undefined ? "(无)" : b, ok: a === b });
|
||
}
|
||
return rows;
|
||
}
|
||
|
||
function compareMessages(stMsgs, waMsgs) {
|
||
const rows = [];
|
||
const n = Math.max(stMsgs.length, waMsgs.length);
|
||
for (let i = 0; i < n; i++) {
|
||
const a = stMsgs[i];
|
||
const b = waMsgs[i];
|
||
if (!a) { rows.push({ i, ok: false, note: "只在本仓库有", wa: b }); continue; }
|
||
if (!b) { rows.push({ i, ok: false, note: "只在酒馆有", st: a }); continue; }
|
||
const roleOk = a.role === b.role;
|
||
const nameOk = (a.name ?? null) === (b.name ?? null);
|
||
const contentOk = norm(a.content) === norm(b.content);
|
||
rows.push({
|
||
i,
|
||
ok: roleOk && nameOk && contentOk,
|
||
roleOk,
|
||
nameOk,
|
||
contentOk,
|
||
similarity: ratio(a.content, b.content),
|
||
st: a,
|
||
wa: b,
|
||
});
|
||
}
|
||
return rows;
|
||
}
|
||
|
||
function firstDiff(rows) {
|
||
const hit = rows.find((r) => !r.ok);
|
||
if (!hit) return "无";
|
||
if (hit.note) return "#" + hit.i + " " + hit.note;
|
||
const bits = [];
|
||
if (!hit.roleOk) bits.push("role " + hit.st.role + " vs " + hit.wa.role);
|
||
if (!hit.nameOk) bits.push("name " + (hit.st.name ?? "无") + " vs " + (hit.wa.name ?? "无"));
|
||
if (!hit.contentOk) bits.push("正文相似度 " + (hit.similarity * 100).toFixed(1) + "%");
|
||
return "#" + hit.i + " " + bits.join(", ");
|
||
}
|
||
|
||
async function main() {
|
||
const args = readArgs(process.argv.slice(2));
|
||
mkdirSync(args.out, { recursive: true });
|
||
const presetName = path.basename(args.preset, ".json");
|
||
|
||
const caps = [
|
||
spawn(process.execPath, [path.join(HERE, "capture-server.mjs"), "--label", "st", "--port", String(args.stCapture), "--out", args.out], { cwd: HERE, stdio: "ignore" }),
|
||
spawn(process.execPath, [path.join(HERE, "capture-server.mjs"), "--label", "wa", "--port", String(args.waCapture), "--out", args.out], { cwd: HERE, stdio: "ignore" }),
|
||
];
|
||
try {
|
||
for (const port of [args.stCapture, args.waCapture]) {
|
||
await waitFor(async () => (await fetch("http://127.0.0.1:" + port + "/v1/models").catch(() => null)) !== null, { timeoutMs: 15000, label: "假上游 " + port });
|
||
}
|
||
|
||
console.log("[conform] 酒馆侧发一轮…");
|
||
const st = await runScript("drive-st.mjs", ["--preset", args.preset, "--message", args.message, "--capture-port", String(args.stCapture), "--st-port", String(args.stPort), "--out", args.out]);
|
||
if (st.code !== 0) throw new Error("酒馆侧失败:\n" + st.out.slice(-2000));
|
||
const stMeta = lastJsonLine(st.out);
|
||
if (!stMeta) throw new Error("酒馆侧没拿到抓包文件:\n" + st.out.slice(-2000));
|
||
|
||
console.log("[conform] 本仓库侧发一轮…");
|
||
const wa = await runScript("drive-wa.mjs", ["--preset", args.preset, "--message", args.message, "--capture-port", String(args.waCapture), "--port", String(args.waPort), "--out", args.out]);
|
||
if (wa.code !== 0) throw new Error("本仓库侧失败:\n" + wa.out.slice(-2000));
|
||
const waMeta = lastJsonLine(wa.out);
|
||
if (!waMeta) throw new Error("本仓库侧没拿到抓包文件:\n" + wa.out.slice(-2000));
|
||
|
||
const stBody = loadBody(stMeta.capture);
|
||
const waBody = loadBody(waMeta.capture);
|
||
const reqRows = compareRequest(stBody, waBody);
|
||
const msgRows = compareMessages(stBody.messages, waBody.messages);
|
||
|
||
const msgOk = msgRows.filter((r) => r.ok).length;
|
||
const reqOk = reqRows.filter((r) => r.ok).length;
|
||
const total = msgRows.length + reqRows.length;
|
||
const okTotal = msgOk + reqOk;
|
||
const pct = ((okTotal / total) * 100).toFixed(1);
|
||
|
||
const lines = [];
|
||
lines.push("# 对齐报告: " + presetName);
|
||
lines.push("");
|
||
lines.push("| 项目 | 值 |");
|
||
lines.push("| --- | --- |");
|
||
lines.push("| 酒馆抓包 | `" + path.basename(stMeta.capture) + "` |");
|
||
lines.push("| 本仓库抓包 | `" + path.basename(waMeta.capture) + "` |");
|
||
lines.push("| 请求级字段 | " + reqOk + "/" + reqRows.length + " 对齐 |");
|
||
lines.push("| 消息条数 | " + stBody.messages.length + " vs " + waBody.messages.length + " |");
|
||
lines.push("| 消息逐条 | " + msgOk + "/" + msgRows.length + " 对齐 |");
|
||
lines.push("| **总对齐度** | **" + pct + "%** |");
|
||
lines.push("| 首个差异 | " + firstDiff(msgRows) + " |");
|
||
lines.push("");
|
||
lines.push("## 请求级字段");
|
||
lines.push("");
|
||
lines.push("| 对齐 | 字段 | 酒馆 | 本仓库 |");
|
||
lines.push("| --- | --- | --- | --- |");
|
||
for (const r of reqRows) {
|
||
lines.push("| " + mark(r.ok) + " | `" + r.key + "` | " + String(r.st).slice(0, 80) + " | " + String(r.wa).slice(0, 80) + " |");
|
||
}
|
||
lines.push("");
|
||
lines.push("## 消息逐条");
|
||
lines.push("");
|
||
lines.push("| # | 对齐 | 酒馆 | 本仓库 |");
|
||
lines.push("| --- | --- | --- | --- |");
|
||
for (const r of msgRows) {
|
||
const one = (m) => m ? "[" + m.role + (m.name ? "/" + m.name : "") + "] " + norm(m.content).length + " 字: " + JSON.stringify(norm(m.content).slice(0, 40)) : "—";
|
||
const why = r.ok ? "" : " **" + (r.note ? r.note : (!r.contentOk ? "正文差 " + ((1 - r.similarity) * 100).toFixed(1) + "%" : !r.roleOk ? "role 不同" : "name 不同")) + "**";
|
||
lines.push("| " + r.i + " | " + mark(r.ok) + " | " + one(r.st) + " | " + one(r.wa) + why + " |");
|
||
}
|
||
lines.push("");
|
||
lines.push("## 差异明细");
|
||
lines.push("");
|
||
const bad = msgRows.filter((r) => !r.ok);
|
||
if (!bad.length && reqRows.every((r) => r.ok)) lines.push("完全对齐。");
|
||
for (const r of bad) {
|
||
if (r.note) { lines.push("- #" + r.i + " " + r.note); continue; }
|
||
lines.push("- #" + r.i + (r.roleOk ? "" : " role " + r.st.role + " vs " + r.wa.role) + (r.nameOk ? "" : " name " + (r.st.name ?? "无") + " vs " + (r.wa.name ?? "无")));
|
||
if (!r.contentOk) {
|
||
lines.push(" - 酒馆: " + JSON.stringify(norm(r.st.content).slice(0, 160)));
|
||
lines.push(" - 我们: " + JSON.stringify(norm(r.wa.content).slice(0, 160)));
|
||
}
|
||
}
|
||
|
||
const report = path.join(args.out, "conform-" + presetName.replace(/[^\w.\-]+/g, "_") + ".md");
|
||
writeFileSync(report, lines.join("\n") + "\n", "utf8");
|
||
|
||
console.log("\n" + lines.slice(0, 12).join("\n"));
|
||
console.log("\n[conform] 对齐度 " + pct + "%（消息 " + msgOk + "/" + msgRows.length + "，字段 " + reqOk + "/" + reqRows.length + "）");
|
||
console.log("[conform] 报告: " + report);
|
||
process.exitCode = okTotal === total ? 0 : 1;
|
||
} finally {
|
||
for (const c of caps) { try { c.kill(); } catch (e) {} }
|
||
await sleep(300);
|
||
}
|
||
}
|
||
|
||
await main();
|