7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
1
2
3
4
5
6
|
async function fetchJSON(url, options) {
const res = await fetch(url, options);
if (!res.ok) throw new Error(await res.text());
return await res.json();
}
|
2059d959
tangwang
feat(eval): 多评估集统...
|
7
8
9
|
let _datasets = [];
let _currentDatasetId = "";
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
10
11
12
13
14
|
function fmtNumber(value, digits = 3) {
if (value == null || Number.isNaN(Number(value))) return "-";
return Number(value).toFixed(digits);
}
|
30b490e1
tangwang
添加ERR评估指标
|
15
16
|
function metricColumns(metrics) {
const defs = [
|
465f90e1
tangwang
添加LTR数据收集
|
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
{
title: "Primary",
keys: [
"Primary_Metric_Score",
"NDCG@20",
"NDCG@50",
"ERR@10",
"Strong_Precision@10",
"Strong_Precision@20",
"Useful_Precision@50",
"Avg_Grade@10",
"Gain_Recall@20",
],
},
|
30b490e1
tangwang
添加ERR评估指标
|
31
32
|
{ title: "NDCG", keys: ["NDCG@5", "NDCG@10", "NDCG@20", "NDCG@50"] },
{ title: "ERR", keys: ["ERR@5", "ERR@10", "ERR@20", "ERR@50"] },
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
33
|
{
|
30b490e1
tangwang
添加ERR评估指标
|
34
35
36
37
38
39
40
41
|
title: "Top slot",
keys: [
"Exact_Precision@5",
"Exact_Precision@10",
"Strong_Precision@5",
"Strong_Precision@10",
"Strong_Precision@20",
],
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
42
43
|
},
{
|
30b490e1
tangwang
添加ERR评估指标
|
44
45
46
47
48
49
50
51
52
|
title: "Recall",
keys: [
"Useful_Precision@10",
"Useful_Precision@20",
"Useful_Precision@50",
"Gain_Recall@10",
"Gain_Recall@20",
"Gain_Recall@50",
],
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
53
54
|
},
{
|
30b490e1
tangwang
添加ERR评估指标
|
55
56
57
58
59
60
61
62
63
64
|
title: "First good",
keys: [
"Exact_Success@5",
"Exact_Success@10",
"Strong_Success@5",
"Strong_Success@10",
"MRR_Exact@10",
"MRR_Strong@10",
"Avg_Grade@10",
],
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
65
66
67
|
},
];
const seen = new Set();
|
30b490e1
tangwang
添加ERR评估指标
|
68
69
70
|
const columns = defs
.map((col) => {
const rows = col.keys
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
71
72
73
74
75
|
.filter((key) => metrics && Object.prototype.hasOwnProperty.call(metrics, key))
.map((key) => {
seen.add(key);
return [key, metrics[key]];
});
|
30b490e1
tangwang
添加ERR评估指标
|
76
|
return { title: col.title, rows };
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
77
|
})
|
30b490e1
tangwang
添加ERR评估指标
|
78
79
80
81
82
83
84
|
.filter((col) => col.rows.length);
const rest = Object.keys(metrics || {})
.filter((key) => !seen.has(key))
.sort()
.map((key) => [key, metrics[key]]);
if (rest.length) columns.push({ title: "Other", rows: rest });
return columns;
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
85
86
87
88
89
90
|
}
function renderMetrics(metrics, metricContext) {
const root = document.getElementById("metrics");
root.innerHTML = "";
const ctx = document.getElementById("metricContext");
|
30b490e1
tangwang
添加ERR评估指标
|
91
92
93
94
|
const parts = [];
if (metricContext && metricContext.primary_metric) {
parts.push(`Primary: ${metricContext.primary_metric}`);
}
|
465f90e1
tangwang
添加LTR数据收集
|
95
96
97
98
99
100
|
if (metricContext && Array.isArray(metricContext.primary_metrics) && metricContext.primary_metrics.length) {
parts.push(`Primary metrics: ${metricContext.primary_metrics.join(", ")}`);
}
if (metricContext && metricContext.primary_metric_formula) {
parts.push(metricContext.primary_metric_formula);
}
|
30b490e1
tangwang
添加ERR评估指标
|
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
if (metricContext && metricContext.gain_scheme) {
parts.push(
`NDCG gains: ${Object.entries(metricContext.gain_scheme)
.map(([label, gain]) => `${label}=${gain}`)
.join(", ")}`
);
}
if (metricContext && metricContext.stop_prob_scheme) {
parts.push(
`ERR P(stop): ${Object.entries(metricContext.stop_prob_scheme)
.map(([label, p]) => `${label}=${p}`)
.join(", ")}`
);
}
ctx.textContent = parts.length ? `${parts.join(". ")}.` : "";
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
116
|
|
30b490e1
tangwang
添加ERR评估指标
|
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
const bar = document.createElement("div");
bar.className = "metrics-columns";
metricColumns(metrics || {}).forEach((col) => {
const column = document.createElement("div");
column.className = "metric-column";
const h = document.createElement("h4");
h.className = "metric-column-title";
h.textContent = col.title;
column.appendChild(h);
col.rows.forEach(([key, value]) => {
const row = document.createElement("div");
row.className = "metric-row";
row.innerHTML = `<span class="metric-row-name">${key}:</span> <span class="metric-row-value">${fmtNumber(value)}</span>`;
column.appendChild(row);
|
c81b0fc1
tangwang
scripts/evaluatio...
|
131
|
});
|
30b490e1
tangwang
添加ERR评估指标
|
132
|
bar.appendChild(column);
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
133
|
});
|
30b490e1
tangwang
添加ERR评估指标
|
134
|
root.appendChild(bar);
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
}
function labelBadgeClass(label) {
if (!label || label === "Unknown") return "badge-unknown";
return "label-" + String(label).toLowerCase().replace(/\s+/g, "-");
}
function renderResults(results, rootId = "results", showRank = true) {
const mount = document.getElementById(rootId);
mount.innerHTML = "";
(results || []).forEach((item) => {
const label = item.label || "Unknown";
const box = document.createElement("div");
box.className = "result";
box.innerHTML = `
<div><span class="badge ${labelBadgeClass(label)}">${label}</span><div class="muted" style="margin-top:8px">${showRank ? `#${item.rank || "-"}` : (item.rerank_score != null ? `rerank=${item.rerank_score.toFixed ? item.rerank_score.toFixed(4) : item.rerank_score}` : "not recalled")}</div></div>
<img class="thumb" src="${item.image_url || ""}" alt="" />
<div>
<div class="title">${item.title || ""}</div>
${item.title_zh ? `<div class="title-zh">${item.title_zh}</div>` : ""}
<div class="options">
<div>${(item.option_values || [])[0] || ""}</div>
<div>${(item.option_values || [])[1] || ""}</div>
<div>${(item.option_values || [])[2] || ""}</div>
</div>
</div>`;
mount.appendChild(box);
});
if (!(results || []).length) {
mount.innerHTML = '<div class="muted">None.</div>';
}
}
function renderTips(data) {
const root = document.getElementById("tips");
const tips = [...(data.tips || [])];
const stats = data.label_stats || {};
tips.unshift(
`Cached labels: ${stats.total || 0}. Recalled hits: ${stats.recalled_hits || 0}. Missed judged useful results: ${stats.missing_relevant_count || 0} (Exact ${stats.missing_exact_count || 0}, High ${stats.missing_high_count || 0}, Low ${stats.missing_low_count || 0}).`
);
root.innerHTML = tips.map((text) => `<div class="tip">${text}</div>`).join("");
}
async function loadQueries() {
|
2059d959
tangwang
feat(eval): 多评估集统...
|
179
180
|
if (!_currentDatasetId) return;
const data = await fetchJSON("/api/datasets/" + encodeURIComponent(_currentDatasetId) + "/queries");
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
181
182
|
const root = document.getElementById("queryList");
root.innerHTML = "";
|
2059d959
tangwang
feat(eval): 多评估集统...
|
183
184
185
186
187
188
189
190
|
const dataset = data.dataset || {};
document.getElementById("queriesMeta").innerHTML = `Loaded from <code>${dataset.query_file || ""}</code>`;
document.getElementById("datasetMeta").textContent =
`${dataset.display_name || dataset.dataset_id || ""} · ${dataset.query_count || 0} queries`;
document.getElementById("pageSubtitle").textContent =
`Current dataset: ${dataset.display_name || dataset.dataset_id || ""}. Single-query evaluation and batch evaluation share the same service on port 6010.`;
document.getElementById("batchButton").textContent =
`Batch Evaluation: ${dataset.display_name || dataset.dataset_id || ""}`;
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
191
192
193
194
195
196
197
198
199
200
201
202
|
data.queries.forEach((query) => {
const btn = document.createElement("button");
btn.className = "query-item";
btn.textContent = query;
btn.onclick = () => {
document.getElementById("queryInput").value = query;
runSingle();
};
root.appendChild(btn);
});
}
|
2059d959
tangwang
feat(eval): 多评估集统...
|
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
|
async function loadDatasets() {
const data = await fetchJSON("/api/datasets");
_datasets = data.datasets || [];
if (!_currentDatasetId) _currentDatasetId = data.current_dataset_id || (_datasets[0] && _datasets[0].dataset_id) || "";
const select = document.getElementById("datasetSelect");
select.innerHTML = "";
_datasets.forEach((dataset) => {
const opt = document.createElement("option");
opt.value = dataset.dataset_id;
opt.textContent = `${dataset.display_name || dataset.dataset_id} (${dataset.query_count || 0})`;
if (dataset.dataset_id === _currentDatasetId) opt.selected = true;
select.appendChild(opt);
});
select.onchange = async (ev) => {
_currentDatasetId = ev.target.value;
await loadQueries();
await loadHistory();
};
}
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
223
224
|
function historySummaryHtml(meta) {
const m = meta && meta.aggregate_metrics;
|
d73ca84a
tangwang
refine eval case ...
|
225
|
const nq = (meta && meta.query_count) || (meta && meta.queries && meta.queries.length) || (meta && meta.per_query && meta.per_query.length) || null;
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
226
227
|
const parts = [];
if (nq != null) parts.push(`<span>Queries</span> ${nq}`);
|
465f90e1
tangwang
添加LTR数据收集
|
228
229
|
if (m && m["Primary_Metric_Score"] != null) parts.push(`<span>Primary</span> ${fmtNumber(m["Primary_Metric_Score"])}`);
if (m && m["NDCG@20"] != null) parts.push(`<span>NDCG@20</span> ${fmtNumber(m["NDCG@20"])}`);
|
30b490e1
tangwang
添加ERR评估指标
|
230
|
if (m && m["ERR@10"] != null) parts.push(`<span>ERR@10</span> ${fmtNumber(m["ERR@10"])}`);
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
231
|
if (m && m["Strong_Precision@10"] != null) parts.push(`<span>Strong@10</span> ${fmtNumber(m["Strong_Precision@10"])}`);
|
465f90e1
tangwang
添加LTR数据收集
|
232
|
if (m && m["Gain_Recall@20"] != null) parts.push(`<span>Gain Recall@20</span> ${fmtNumber(m["Gain_Recall@20"])}`);
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
233
234
235
236
237
|
if (!parts.length) return "";
return `<div class="hstats">${parts.join(" · ")}</div>`;
}
async function loadHistory() {
|
2059d959
tangwang
feat(eval): 多评估集统...
|
238
239
|
if (!_currentDatasetId) return;
const data = await fetchJSON("/api/history?dataset_id=" + encodeURIComponent(_currentDatasetId));
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
|
const root = document.getElementById("history");
root.classList.remove("muted");
const items = data.history || [];
if (!items.length) {
root.innerHTML = '<span class="muted">No history yet.</span>';
return;
}
root.innerHTML = `<div class="history-list"></div>`;
const list = root.querySelector(".history-list");
items.forEach((item) => {
const btn = document.createElement("button");
btn.type = "button";
btn.className = "history-item";
btn.setAttribute("aria-label", `Open report ${item.batch_id}`);
const sum = historySummaryHtml(item.metadata);
|
2059d959
tangwang
feat(eval): 多评估集统...
|
255
256
|
const dataset = (item.metadata && item.metadata.dataset) || {};
const datasetName = dataset.display_name || dataset.dataset_id || item.dataset_id || "";
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
257
|
btn.innerHTML = `<div class="hid">${item.batch_id}</div>
|
2059d959
tangwang
feat(eval): 多评估集统...
|
258
|
<div class="hmeta">${item.created_at} · tenant ${item.tenant_id}${datasetName ? ` · ${datasetName}` : ""}</div>${sum}`;
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
|
btn.onclick = () => openBatchReport(item.batch_id);
list.appendChild(btn);
});
}
let _lastReportPath = "";
function closeReportModal() {
const el = document.getElementById("reportModal");
el.classList.remove("is-open");
el.setAttribute("aria-hidden", "true");
document.getElementById("reportModalBody").innerHTML = "";
document.getElementById("reportModalMeta").textContent = "";
}
async function openBatchReport(batchId) {
const el = document.getElementById("reportModal");
const body = document.getElementById("reportModalBody");
const metaEl = document.getElementById("reportModalMeta");
const titleEl = document.getElementById("reportModalTitle");
el.classList.add("is-open");
el.setAttribute("aria-hidden", "false");
titleEl.textContent = batchId;
metaEl.textContent = "";
body.className = "report-modal-body batch-report-md report-modal-loading";
body.textContent = "Loading report…";
try {
const rep = await fetchJSON("/api/history/" + encodeURIComponent(batchId) + "/report");
_lastReportPath = rep.report_markdown_path || "";
|
2059d959
tangwang
feat(eval): 多评估集统...
|
288
289
290
291
|
const dataset = rep.dataset || {};
metaEl.textContent = [dataset.display_name || dataset.dataset_id || "", rep.report_markdown_path || ""]
.filter(Boolean)
.join(" · ");
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
|
const raw = marked.parse(rep.markdown || "", { gfm: true });
const safe = DOMPurify.sanitize(raw, { USE_PROFILES: { html: true } });
body.className = "report-modal-body batch-report-md";
body.innerHTML = safe;
} catch (e) {
body.className = "report-modal-body report-modal-error";
body.textContent = e && e.message ? e.message : String(e);
}
}
document.getElementById("reportModal").addEventListener("click", (ev) => {
if (ev.target && ev.target.getAttribute("data-close-report") === "1") closeReportModal();
});
document.addEventListener("keydown", (ev) => {
if (ev.key === "Escape") closeReportModal();
});
document.getElementById("reportCopyPath").addEventListener("click", async () => {
if (!_lastReportPath) return;
try {
await navigator.clipboard.writeText(_lastReportPath);
} catch (_) {}
});
async function runSingle() {
const query = document.getElementById("queryInput").value.trim();
if (!query) return;
|
2059d959
tangwang
feat(eval): 多评估集统...
|
320
|
document.getElementById("status").textContent = `Evaluating "${query}" on ${_currentDatasetId}...`;
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
321
322
323
|
const data = await fetchJSON("/api/search-eval", {
method: "POST",
headers: { "Content-Type": "application/json" },
|
2059d959
tangwang
feat(eval): 多评估集统...
|
324
|
body: JSON.stringify({ query, dataset_id: _currentDatasetId, top_k: 100, auto_annotate: false }),
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
325
326
327
328
329
330
331
332
333
334
|
});
document.getElementById("status").textContent = `Done. total=${data.total}`;
renderMetrics(data.metrics, data.metric_context);
renderResults(data.results, "results", true);
renderResults(data.missing_relevant, "missingRelevant", false);
renderTips(data);
loadHistory();
}
async function runBatch() {
|
2059d959
tangwang
feat(eval): 多评估集统...
|
335
|
document.getElementById("status").textContent = `Running batch evaluation for ${_currentDatasetId}...`;
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
336
337
338
|
const data = await fetchJSON("/api/batch-eval", {
method: "POST",
headers: { "Content-Type": "application/json" },
|
2059d959
tangwang
feat(eval): 多评估集统...
|
339
|
body: JSON.stringify({ dataset_id: _currentDatasetId, top_k: 100, auto_annotate: false }),
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
340
341
342
343
344
|
});
document.getElementById("status").textContent = `Batch done. report=${data.batch_id}`;
renderMetrics(data.aggregate_metrics, data.metric_context);
renderResults([], "results", true);
renderResults([], "missingRelevant", false);
|
2059d959
tangwang
feat(eval): 多评估集统...
|
345
346
|
document.getElementById("tips").innerHTML =
'<div class="tip">Batch evaluation uses cached labels only unless force refresh is requested via CLI/API.</div>';
|
7ddd4cb3
tangwang
评估体系从三等级->四等级 Exa...
|
347
348
349
|
loadHistory();
}
|
2059d959
tangwang
feat(eval): 多评估集统...
|
350
|
loadDatasets().then(() => loadQueries()).then(() => loadHistory());
|