eval_web.js 10.3 KB
async function fetchJSON(url, options) {
  const res = await fetch(url, options);
  if (!res.ok) throw new Error(await res.text());
  return await res.json();
}

function fmtNumber(value, digits = 3) {
  if (value == null || Number.isNaN(Number(value))) return "-";
  return Number(value).toFixed(digits);
}

function metricSections(metrics) {
  const groups = [
    {
      title: "Primary Ranking",
      keys: ["NDCG@5", "NDCG@10", "NDCG@20", "NDCG@50"],
      description: "Graded ranking quality across the four relevance tiers.",
    },
    {
      title: "Top Slot Quality",
      keys: ["Exact_Precision@5", "Exact_Precision@10", "Strong_Precision@5", "Strong_Precision@10", "Strong_Precision@20"],
      description: "How much of the visible top rank is exact or strong business relevance.",
    },
    {
      title: "Recall Coverage",
      keys: ["Useful_Precision@10", "Useful_Precision@20", "Useful_Precision@50", "Gain_Recall@10", "Gain_Recall@20", "Gain_Recall@50"],
      description: "How much judged relevance is captured in the returned list.",
    },
    {
      title: "First Good Result",
      keys: ["Exact_Success@5", "Exact_Success@10", "Strong_Success@5", "Strong_Success@10", "MRR_Exact@10", "MRR_Strong@10", "Avg_Grade@10"],
      description: "Whether users see a good result early and how good the top page feels overall.",
    },
  ];
  const seen = new Set();
  return groups
    .map((group) => {
      const items = group.keys
        .filter((key) => metrics && Object.prototype.hasOwnProperty.call(metrics, key))
        .map((key) => {
          seen.add(key);
          return [key, metrics[key]];
        });
      return { ...group, items };
    })
    .filter((group) => group.items.length)
    .concat(
      (() => {
        const rest = Object.entries(metrics || {}).filter(([key]) => !seen.has(key));
        return rest.length
          ? [{ title: "Other Metrics", description: "", items: rest }]
          : [];
      })()
    );
}

function renderMetrics(metrics, metricContext) {
  const root = document.getElementById("metrics");
  root.innerHTML = "";
  const ctx = document.getElementById("metricContext");
  const gainScheme = metricContext && metricContext.gain_scheme;
  const primary = metricContext && metricContext.primary_metric;
  ctx.textContent = primary
    ? `Primary metric: ${primary}. Gain scheme: ${Object.entries(gainScheme || {}).map(([label, gain]) => `${label}=${gain}`).join(", ")}.`
    : "";

  metricSections(metrics || {}).forEach((section) => {
    const wrap = document.createElement("section");
    wrap.className = "metric-section";
    wrap.innerHTML = `
      <div class="metric-section-head">
        <h3>${section.title}</h3>
        ${section.description ? `<p>${section.description}</p>` : ""}
      </div>
      <div class="grid metric-grid"></div>
    `;
    const grid = wrap.querySelector(".metric-grid");
    section.items.forEach(([key, value]) => {
      const card = document.createElement("div");
      card.className = "metric";
      card.innerHTML = `<div class="label">${key}</div><div class="value">${fmtNumber(value)}</div>`;
      grid.appendChild(card);
    });
    root.appendChild(wrap);
  });
}

function labelBadgeClass(label) {
  if (!label || label === "Unknown") return "badge-unknown";
  return "label-" + String(label).toLowerCase().replace(/\s+/g, "-");
}

function renderResults(results, rootId = "results", showRank = true) {
  const mount = document.getElementById(rootId);
  mount.innerHTML = "";
  (results || []).forEach((item) => {
    const label = item.label || "Unknown";
    const box = document.createElement("div");
    box.className = "result";
    box.innerHTML = `
      <div><span class="badge ${labelBadgeClass(label)}">${label}</span><div class="muted" style="margin-top:8px">${showRank ? `#${item.rank || "-"}` : (item.rerank_score != null ? `rerank=${item.rerank_score.toFixed ? item.rerank_score.toFixed(4) : item.rerank_score}` : "not recalled")}</div></div>
      <img class="thumb" src="${item.image_url || ""}" alt="" />
      <div>
        <div class="title">${item.title || ""}</div>
        ${item.title_zh ? `<div class="title-zh">${item.title_zh}</div>` : ""}
        <div class="options">
          <div>${(item.option_values || [])[0] || ""}</div>
          <div>${(item.option_values || [])[1] || ""}</div>
          <div>${(item.option_values || [])[2] || ""}</div>
        </div>
      </div>`;
    mount.appendChild(box);
  });
  if (!(results || []).length) {
    mount.innerHTML = '<div class="muted">None.</div>';
  }
}

function renderTips(data) {
  const root = document.getElementById("tips");
  const tips = [...(data.tips || [])];
  const stats = data.label_stats || {};
  tips.unshift(
    `Cached labels: ${stats.total || 0}. Recalled hits: ${stats.recalled_hits || 0}. Missed judged useful results: ${stats.missing_relevant_count || 0} (Exact ${stats.missing_exact_count || 0}, High ${stats.missing_high_count || 0}, Low ${stats.missing_low_count || 0}).`
  );
  root.innerHTML = tips.map((text) => `<div class="tip">${text}</div>`).join("");
}

async function loadQueries() {
  const data = await fetchJSON("/api/queries");
  const root = document.getElementById("queryList");
  root.innerHTML = "";
  data.queries.forEach((query) => {
    const btn = document.createElement("button");
    btn.className = "query-item";
    btn.textContent = query;
    btn.onclick = () => {
      document.getElementById("queryInput").value = query;
      runSingle();
    };
    root.appendChild(btn);
  });
}

function historySummaryHtml(meta) {
  const m = meta && meta.aggregate_metrics;
  const nq = (meta && meta.queries && meta.queries.length) || (meta && meta.per_query && meta.per_query.length) || null;
  const parts = [];
  if (nq != null) parts.push(`<span>Queries</span> ${nq}`);
  if (m && m["NDCG@10"] != null) parts.push(`<span>NDCG@10</span> ${fmtNumber(m["NDCG@10"])}`);
  if (m && m["Strong_Precision@10"] != null) parts.push(`<span>Strong@10</span> ${fmtNumber(m["Strong_Precision@10"])}`);
  if (m && m["Gain_Recall@50"] != null) parts.push(`<span>Gain Recall@50</span> ${fmtNumber(m["Gain_Recall@50"])}`);
  if (!parts.length) return "";
  return `<div class="hstats">${parts.join(" · ")}</div>`;
}

async function loadHistory() {
  const data = await fetchJSON("/api/history");
  const root = document.getElementById("history");
  root.classList.remove("muted");
  const items = data.history || [];
  if (!items.length) {
    root.innerHTML = '<span class="muted">No history yet.</span>';
    return;
  }
  root.innerHTML = `<div class="history-list"></div>`;
  const list = root.querySelector(".history-list");
  items.forEach((item) => {
    const btn = document.createElement("button");
    btn.type = "button";
    btn.className = "history-item";
    btn.setAttribute("aria-label", `Open report ${item.batch_id}`);
    const sum = historySummaryHtml(item.metadata);
    btn.innerHTML = `<div class="hid">${item.batch_id}</div>
      <div class="hmeta">${item.created_at} · tenant ${item.tenant_id}</div>${sum}`;
    btn.onclick = () => openBatchReport(item.batch_id);
    list.appendChild(btn);
  });
}

let _lastReportPath = "";

function closeReportModal() {
  const el = document.getElementById("reportModal");
  el.classList.remove("is-open");
  el.setAttribute("aria-hidden", "true");
  document.getElementById("reportModalBody").innerHTML = "";
  document.getElementById("reportModalMeta").textContent = "";
}

async function openBatchReport(batchId) {
  const el = document.getElementById("reportModal");
  const body = document.getElementById("reportModalBody");
  const metaEl = document.getElementById("reportModalMeta");
  const titleEl = document.getElementById("reportModalTitle");
  el.classList.add("is-open");
  el.setAttribute("aria-hidden", "false");
  titleEl.textContent = batchId;
  metaEl.textContent = "";
  body.className = "report-modal-body batch-report-md report-modal-loading";
  body.textContent = "Loading report…";
  try {
    const rep = await fetchJSON("/api/history/" + encodeURIComponent(batchId) + "/report");
    _lastReportPath = rep.report_markdown_path || "";
    metaEl.textContent = rep.report_markdown_path || "";
    const raw = marked.parse(rep.markdown || "", { gfm: true });
    const safe = DOMPurify.sanitize(raw, { USE_PROFILES: { html: true } });
    body.className = "report-modal-body batch-report-md";
    body.innerHTML = safe;
  } catch (e) {
    body.className = "report-modal-body report-modal-error";
    body.textContent = e && e.message ? e.message : String(e);
  }
}

document.getElementById("reportModal").addEventListener("click", (ev) => {
  if (ev.target && ev.target.getAttribute("data-close-report") === "1") closeReportModal();
});

document.addEventListener("keydown", (ev) => {
  if (ev.key === "Escape") closeReportModal();
});

document.getElementById("reportCopyPath").addEventListener("click", async () => {
  if (!_lastReportPath) return;
  try {
    await navigator.clipboard.writeText(_lastReportPath);
  } catch (_) {}
});

async function runSingle() {
  const query = document.getElementById("queryInput").value.trim();
  if (!query) return;
  document.getElementById("status").textContent = `Evaluating "${query}"...`;
  const data = await fetchJSON("/api/search-eval", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ query, top_k: 100, auto_annotate: false }),
  });
  document.getElementById("status").textContent = `Done. total=${data.total}`;
  renderMetrics(data.metrics, data.metric_context);
  renderResults(data.results, "results", true);
  renderResults(data.missing_relevant, "missingRelevant", false);
  renderTips(data);
  loadHistory();
}

async function runBatch() {
  document.getElementById("status").textContent = "Running batch evaluation...";
  const data = await fetchJSON("/api/batch-eval", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ top_k: 100, auto_annotate: false }),
  });
  document.getElementById("status").textContent = `Batch done. report=${data.batch_id}`;
  renderMetrics(data.aggregate_metrics, data.metric_context);
  renderResults([], "results", true);
  renderResults([], "missingRelevant", false);
  document.getElementById("tips").innerHTML = '<div class="tip">Batch evaluation uses cached labels only unless force refresh is requested via CLI/API.</div>';
  loadHistory();
}

loadQueries();
loadHistory();