Blame view

scripts/evaluation/eval_framework/static/eval_web.js 13.3 KB
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
1
2
3
4
5
6
  async function fetchJSON(url, options) {
    const res = await fetch(url, options);
    if (!res.ok) throw new Error(await res.text());
    return await res.json();
  }
  
2059d959   tangwang   feat(eval): 多评估集统...
7
8
9
  let _datasets = [];
  let _currentDatasetId = "";
  
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
10
11
12
13
14
  function fmtNumber(value, digits = 3) {
    if (value == null || Number.isNaN(Number(value))) return "-";
    return Number(value).toFixed(digits);
  }
  
30b490e1   tangwang   添加ERR评估指标
15
16
  function metricColumns(metrics) {
    const defs = [
465f90e1   tangwang   添加LTR数据收集
17
18
19
20
21
22
23
24
25
26
27
28
29
30
      {
        title: "Primary",
        keys: [
          "Primary_Metric_Score",
          "NDCG@20",
          "NDCG@50",
          "ERR@10",
          "Strong_Precision@10",
          "Strong_Precision@20",
          "Useful_Precision@50",
          "Avg_Grade@10",
          "Gain_Recall@20",
        ],
      },
30b490e1   tangwang   添加ERR评估指标
31
32
      { title: "NDCG", keys: ["NDCG@5", "NDCG@10", "NDCG@20", "NDCG@50"] },
      { title: "ERR", keys: ["ERR@5", "ERR@10", "ERR@20", "ERR@50"] },
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
33
      {
30b490e1   tangwang   添加ERR评估指标
34
35
36
37
38
39
40
41
        title: "Top slot",
        keys: [
          "Exact_Precision@5",
          "Exact_Precision@10",
          "Strong_Precision@5",
          "Strong_Precision@10",
          "Strong_Precision@20",
        ],
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
42
43
      },
      {
30b490e1   tangwang   添加ERR评估指标
44
45
46
47
48
49
50
51
52
        title: "Recall",
        keys: [
          "Useful_Precision@10",
          "Useful_Precision@20",
          "Useful_Precision@50",
          "Gain_Recall@10",
          "Gain_Recall@20",
          "Gain_Recall@50",
        ],
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
53
54
      },
      {
30b490e1   tangwang   添加ERR评估指标
55
56
57
58
59
60
61
62
63
64
        title: "First good",
        keys: [
          "Exact_Success@5",
          "Exact_Success@10",
          "Strong_Success@5",
          "Strong_Success@10",
          "MRR_Exact@10",
          "MRR_Strong@10",
          "Avg_Grade@10",
        ],
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
65
66
67
      },
    ];
    const seen = new Set();
30b490e1   tangwang   添加ERR评估指标
68
69
70
    const columns = defs
      .map((col) => {
        const rows = col.keys
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
71
72
73
74
75
          .filter((key) => metrics && Object.prototype.hasOwnProperty.call(metrics, key))
          .map((key) => {
            seen.add(key);
            return [key, metrics[key]];
          });
30b490e1   tangwang   添加ERR评估指标
76
        return { title: col.title, rows };
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
77
      })
30b490e1   tangwang   添加ERR评估指标
78
79
80
81
82
83
84
      .filter((col) => col.rows.length);
    const rest = Object.keys(metrics || {})
      .filter((key) => !seen.has(key))
      .sort()
      .map((key) => [key, metrics[key]]);
    if (rest.length) columns.push({ title: "Other", rows: rest });
    return columns;
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
85
86
87
88
89
90
  }
  
  function renderMetrics(metrics, metricContext) {
    const root = document.getElementById("metrics");
    root.innerHTML = "";
    const ctx = document.getElementById("metricContext");
30b490e1   tangwang   添加ERR评估指标
91
92
93
94
    const parts = [];
    if (metricContext && metricContext.primary_metric) {
      parts.push(`Primary: ${metricContext.primary_metric}`);
    }
465f90e1   tangwang   添加LTR数据收集
95
96
97
98
99
100
    if (metricContext && Array.isArray(metricContext.primary_metrics) && metricContext.primary_metrics.length) {
      parts.push(`Primary metrics: ${metricContext.primary_metrics.join(", ")}`);
    }
    if (metricContext && metricContext.primary_metric_formula) {
      parts.push(metricContext.primary_metric_formula);
    }
30b490e1   tangwang   添加ERR评估指标
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
    if (metricContext && metricContext.gain_scheme) {
      parts.push(
        `NDCG gains: ${Object.entries(metricContext.gain_scheme)
          .map(([label, gain]) => `${label}=${gain}`)
          .join(", ")}`
      );
    }
    if (metricContext && metricContext.stop_prob_scheme) {
      parts.push(
        `ERR P(stop): ${Object.entries(metricContext.stop_prob_scheme)
          .map(([label, p]) => `${label}=${p}`)
          .join(", ")}`
      );
    }
    ctx.textContent = parts.length ? `${parts.join(". ")}.` : "";
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
116
  
30b490e1   tangwang   添加ERR评估指标
117
118
119
120
121
122
123
124
125
126
127
128
129
130
    const bar = document.createElement("div");
    bar.className = "metrics-columns";
    metricColumns(metrics || {}).forEach((col) => {
      const column = document.createElement("div");
      column.className = "metric-column";
      const h = document.createElement("h4");
      h.className = "metric-column-title";
      h.textContent = col.title;
      column.appendChild(h);
      col.rows.forEach(([key, value]) => {
        const row = document.createElement("div");
        row.className = "metric-row";
        row.innerHTML = `<span class="metric-row-name">${key}:</span> <span class="metric-row-value">${fmtNumber(value)}</span>`;
        column.appendChild(row);
c81b0fc1   tangwang   scripts/evaluatio...
131
      });
30b490e1   tangwang   添加ERR评估指标
132
      bar.appendChild(column);
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
133
    });
30b490e1   tangwang   添加ERR评估指标
134
    root.appendChild(bar);
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
  }
  
  function labelBadgeClass(label) {
    if (!label || label === "Unknown") return "badge-unknown";
    return "label-" + String(label).toLowerCase().replace(/\s+/g, "-");
  }
  
  function renderResults(results, rootId = "results", showRank = true) {
    const mount = document.getElementById(rootId);
    mount.innerHTML = "";
    (results || []).forEach((item) => {
      const label = item.label || "Unknown";
      const box = document.createElement("div");
      box.className = "result";
      box.innerHTML = `
        <div><span class="badge ${labelBadgeClass(label)}">${label}</span><div class="muted" style="margin-top:8px">${showRank ? `#${item.rank || "-"}` : (item.rerank_score != null ? `rerank=${item.rerank_score.toFixed ? item.rerank_score.toFixed(4) : item.rerank_score}` : "not recalled")}</div></div>
        <img class="thumb" src="${item.image_url || ""}" alt="" />
        <div>
          <div class="title">${item.title || ""}</div>
          ${item.title_zh ? `<div class="title-zh">${item.title_zh}</div>` : ""}
          <div class="options">
            <div>${(item.option_values || [])[0] || ""}</div>
            <div>${(item.option_values || [])[1] || ""}</div>
            <div>${(item.option_values || [])[2] || ""}</div>
          </div>
        </div>`;
      mount.appendChild(box);
    });
    if (!(results || []).length) {
      mount.innerHTML = '<div class="muted">None.</div>';
    }
  }
  
  function renderTips(data) {
    const root = document.getElementById("tips");
    const tips = [...(data.tips || [])];
    const stats = data.label_stats || {};
    tips.unshift(
      `Cached labels: ${stats.total || 0}. Recalled hits: ${stats.recalled_hits || 0}. Missed judged useful results: ${stats.missing_relevant_count || 0} (Exact ${stats.missing_exact_count || 0}, High ${stats.missing_high_count || 0}, Low ${stats.missing_low_count || 0}).`
    );
    root.innerHTML = tips.map((text) => `<div class="tip">${text}</div>`).join("");
  }
  
  async function loadQueries() {
2059d959   tangwang   feat(eval): 多评估集统...
179
180
    if (!_currentDatasetId) return;
    const data = await fetchJSON("/api/datasets/" + encodeURIComponent(_currentDatasetId) + "/queries");
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
181
182
    const root = document.getElementById("queryList");
    root.innerHTML = "";
2059d959   tangwang   feat(eval): 多评估集统...
183
184
185
186
187
188
189
190
    const dataset = data.dataset || {};
    document.getElementById("queriesMeta").innerHTML = `Loaded from <code>${dataset.query_file || ""}</code>`;
    document.getElementById("datasetMeta").textContent =
      `${dataset.display_name || dataset.dataset_id || ""} · ${dataset.query_count || 0} queries`;
    document.getElementById("pageSubtitle").textContent =
      `Current dataset: ${dataset.display_name || dataset.dataset_id || ""}. Single-query evaluation and batch evaluation share the same service on port 6010.`;
    document.getElementById("batchButton").textContent =
      `Batch Evaluation: ${dataset.display_name || dataset.dataset_id || ""}`;
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
191
192
193
194
195
196
197
198
199
200
201
202
    data.queries.forEach((query) => {
      const btn = document.createElement("button");
      btn.className = "query-item";
      btn.textContent = query;
      btn.onclick = () => {
        document.getElementById("queryInput").value = query;
        runSingle();
      };
      root.appendChild(btn);
    });
  }
  
2059d959   tangwang   feat(eval): 多评估集统...
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
  async function loadDatasets() {
    const data = await fetchJSON("/api/datasets");
    _datasets = data.datasets || [];
    if (!_currentDatasetId) _currentDatasetId = data.current_dataset_id || (_datasets[0] && _datasets[0].dataset_id) || "";
    const select = document.getElementById("datasetSelect");
    select.innerHTML = "";
    _datasets.forEach((dataset) => {
      const opt = document.createElement("option");
      opt.value = dataset.dataset_id;
      opt.textContent = `${dataset.display_name || dataset.dataset_id} (${dataset.query_count || 0})`;
      if (dataset.dataset_id === _currentDatasetId) opt.selected = true;
      select.appendChild(opt);
    });
    select.onchange = async (ev) => {
      _currentDatasetId = ev.target.value;
      await loadQueries();
      await loadHistory();
    };
  }
  
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
223
224
  function historySummaryHtml(meta) {
    const m = meta && meta.aggregate_metrics;
d73ca84a   tangwang   refine eval case ...
225
    const nq = (meta && meta.query_count) || (meta && meta.queries && meta.queries.length) || (meta && meta.per_query && meta.per_query.length) || null;
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
226
227
    const parts = [];
    if (nq != null) parts.push(`<span>Queries</span> ${nq}`);
465f90e1   tangwang   添加LTR数据收集
228
229
    if (m && m["Primary_Metric_Score"] != null) parts.push(`<span>Primary</span> ${fmtNumber(m["Primary_Metric_Score"])}`);
    if (m && m["NDCG@20"] != null) parts.push(`<span>NDCG@20</span> ${fmtNumber(m["NDCG@20"])}`);
30b490e1   tangwang   添加ERR评估指标
230
    if (m && m["ERR@10"] != null) parts.push(`<span>ERR@10</span> ${fmtNumber(m["ERR@10"])}`);
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
231
    if (m && m["Strong_Precision@10"] != null) parts.push(`<span>Strong@10</span> ${fmtNumber(m["Strong_Precision@10"])}`);
465f90e1   tangwang   添加LTR数据收集
232
    if (m && m["Gain_Recall@20"] != null) parts.push(`<span>Gain Recall@20</span> ${fmtNumber(m["Gain_Recall@20"])}`);
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
233
234
235
236
237
    if (!parts.length) return "";
    return `<div class="hstats">${parts.join(" · ")}</div>`;
  }
  
  async function loadHistory() {
2059d959   tangwang   feat(eval): 多评估集统...
238
239
    if (!_currentDatasetId) return;
    const data = await fetchJSON("/api/history?dataset_id=" + encodeURIComponent(_currentDatasetId));
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
    const root = document.getElementById("history");
    root.classList.remove("muted");
    const items = data.history || [];
    if (!items.length) {
      root.innerHTML = '<span class="muted">No history yet.</span>';
      return;
    }
    root.innerHTML = `<div class="history-list"></div>`;
    const list = root.querySelector(".history-list");
    items.forEach((item) => {
      const btn = document.createElement("button");
      btn.type = "button";
      btn.className = "history-item";
      btn.setAttribute("aria-label", `Open report ${item.batch_id}`);
      const sum = historySummaryHtml(item.metadata);
2059d959   tangwang   feat(eval): 多评估集统...
255
256
      const dataset = (item.metadata && item.metadata.dataset) || {};
      const datasetName = dataset.display_name || dataset.dataset_id || item.dataset_id || "";
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
257
      btn.innerHTML = `<div class="hid">${item.batch_id}</div>
2059d959   tangwang   feat(eval): 多评估集统...
258
        <div class="hmeta">${item.created_at} · tenant ${item.tenant_id}${datasetName ? ` · ${datasetName}` : ""}</div>${sum}`;
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
      btn.onclick = () => openBatchReport(item.batch_id);
      list.appendChild(btn);
    });
  }
  
  let _lastReportPath = "";
  
  function closeReportModal() {
    const el = document.getElementById("reportModal");
    el.classList.remove("is-open");
    el.setAttribute("aria-hidden", "true");
    document.getElementById("reportModalBody").innerHTML = "";
    document.getElementById("reportModalMeta").textContent = "";
  }
  
  async function openBatchReport(batchId) {
    const el = document.getElementById("reportModal");
    const body = document.getElementById("reportModalBody");
    const metaEl = document.getElementById("reportModalMeta");
    const titleEl = document.getElementById("reportModalTitle");
    el.classList.add("is-open");
    el.setAttribute("aria-hidden", "false");
    titleEl.textContent = batchId;
    metaEl.textContent = "";
    body.className = "report-modal-body batch-report-md report-modal-loading";
    body.textContent = "Loading report…";
    try {
      const rep = await fetchJSON("/api/history/" + encodeURIComponent(batchId) + "/report");
      _lastReportPath = rep.report_markdown_path || "";
2059d959   tangwang   feat(eval): 多评估集统...
288
289
290
291
      const dataset = rep.dataset || {};
      metaEl.textContent = [dataset.display_name || dataset.dataset_id || "", rep.report_markdown_path || ""]
        .filter(Boolean)
        .join(" · ");
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
      const raw = marked.parse(rep.markdown || "", { gfm: true });
      const safe = DOMPurify.sanitize(raw, { USE_PROFILES: { html: true } });
      body.className = "report-modal-body batch-report-md";
      body.innerHTML = safe;
    } catch (e) {
      body.className = "report-modal-body report-modal-error";
      body.textContent = e && e.message ? e.message : String(e);
    }
  }
  
  document.getElementById("reportModal").addEventListener("click", (ev) => {
    if (ev.target && ev.target.getAttribute("data-close-report") === "1") closeReportModal();
  });
  
  document.addEventListener("keydown", (ev) => {
    if (ev.key === "Escape") closeReportModal();
  });
  
  document.getElementById("reportCopyPath").addEventListener("click", async () => {
    if (!_lastReportPath) return;
    try {
      await navigator.clipboard.writeText(_lastReportPath);
    } catch (_) {}
  });
  
  async function runSingle() {
    const query = document.getElementById("queryInput").value.trim();
    if (!query) return;
2059d959   tangwang   feat(eval): 多评估集统...
320
    document.getElementById("status").textContent = `Evaluating "${query}" on ${_currentDatasetId}...`;
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
321
322
323
    const data = await fetchJSON("/api/search-eval", {
      method: "POST",
      headers: { "Content-Type": "application/json" },
2059d959   tangwang   feat(eval): 多评估集统...
324
      body: JSON.stringify({ query, dataset_id: _currentDatasetId, top_k: 100, auto_annotate: false }),
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
325
326
327
328
329
330
331
332
333
334
    });
    document.getElementById("status").textContent = `Done. total=${data.total}`;
    renderMetrics(data.metrics, data.metric_context);
    renderResults(data.results, "results", true);
    renderResults(data.missing_relevant, "missingRelevant", false);
    renderTips(data);
    loadHistory();
  }
  
  async function runBatch() {
2059d959   tangwang   feat(eval): 多评估集统...
335
    document.getElementById("status").textContent = `Running batch evaluation for ${_currentDatasetId}...`;
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
336
337
338
    const data = await fetchJSON("/api/batch-eval", {
      method: "POST",
      headers: { "Content-Type": "application/json" },
2059d959   tangwang   feat(eval): 多评估集统...
339
      body: JSON.stringify({ dataset_id: _currentDatasetId, top_k: 100, auto_annotate: false }),
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
340
341
342
343
344
    });
    document.getElementById("status").textContent = `Batch done. report=${data.batch_id}`;
    renderMetrics(data.aggregate_metrics, data.metric_context);
    renderResults([], "results", true);
    renderResults([], "missingRelevant", false);
2059d959   tangwang   feat(eval): 多评估集统...
345
346
    document.getElementById("tips").innerHTML =
      '<div class="tip">Batch evaluation uses cached labels only unless force refresh is requested via CLI/API.</div>';
7ddd4cb3   tangwang   评估体系从三等级->四等级 Exa...
347
348
349
    loadHistory();
  }
  
2059d959   tangwang   feat(eval): 多评估集统...
350
  loadDatasets().then(() => loadQueries()).then(() => loadHistory());