Blame view

scripts/evaluation/start_eval.sh 1.85 KB
881d338b   tangwang   评估框架
1
2
3
4
5
6
7
8
  #!/usr/bin/env bash
  # Search evaluation quick entrypoints. Run from any cwd; resolves repo root.
  set -euo pipefail
  
  ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
  cd "$ROOT"
  PY="${ROOT}/.venv/bin/python"
  TENANT_ID="${TENANT_ID:-163}"
2059d959   tangwang   feat(eval): 多评估集统...
9
  DATASET_ID="${REPO_EVAL_DATASET_ID:-core_queries}"
881d338b   tangwang   评估框架
10
  QUERIES="${REPO_EVAL_QUERIES:-scripts/evaluation/queries/queries.txt}"
881d338b   tangwang   评估框架
11
12
  
  usage() {
f8e7cb97   tangwang   evalution framework
13
    echo "Usage: $0 batch|batch-rebuild|serve"
a345b01f   tangwang   eval framework
14
    echo "  batch          — batch eval: live search every query, LLM only for missing labels (top_k=50)"
d172c259   tangwang   eval框架
15
    echo "  batch-rebuild  — deep rebuild: build --force-refresh-labels (search recall pool + full-corpus rerank + batched LLM; expensive)"
7b8d9e1a   tangwang   评估框架的启动脚本
16
    echo "  serve          — eval UI (default http://0.0.0.0:\${EVAL_WEB_PORT:-6010}/; also: ./scripts/start_eval_web.sh)"
2059d959   tangwang   feat(eval): 多评估集统...
17
    echo "Env: TENANT_ID (default 163), REPO_EVAL_DATASET_ID (default core_queries), REPO_EVAL_QUERIES, EVAL_WEB_HOST, EVAL_WEB_PORT (default 6010)"
881d338b   tangwang   评估框架
18
19
20
21
  }
  
  case "${1:-}" in
    batch)
286e9b4f   tangwang   evalution
22
23
      exec "$PY" scripts/evaluation/build_annotation_set.py batch \
        --tenant-id "$TENANT_ID" \
2059d959   tangwang   feat(eval): 多评估集统...
24
        --dataset-id "$DATASET_ID" \
286e9b4f   tangwang   evalution
25
26
        --queries-file "$QUERIES" \
        --top-k 50 \
a345b01f   tangwang   eval framework
27
        --language en
f8e7cb97   tangwang   evalution framework
28
29
      ;;
    batch-rebuild)
286e9b4f   tangwang   evalution
30
31
      exec "$PY" scripts/evaluation/build_annotation_set.py build \
        --tenant-id "$TENANT_ID" \
2059d959   tangwang   feat(eval): 多评估集统...
32
        --dataset-id "$DATASET_ID" \
286e9b4f   tangwang   evalution
33
34
35
36
37
38
        --queries-file "$QUERIES" \
        --search-depth 500 \
        --rerank-depth 10000 \
        --reset-artifacts \
        --force-refresh-rerank \
        --force-refresh-labels \
a345b01f   tangwang   eval framework
39
        --language en
881d338b   tangwang   评估框架
40
41
      ;;
    serve)
7b8d9e1a   tangwang   评估框架的启动脚本
42
43
      EVAL_WEB_PORT="${EVAL_WEB_PORT:-6010}"
      EVAL_WEB_HOST="${EVAL_WEB_HOST:-0.0.0.0}"
881d338b   tangwang   评估框架
44
45
      exec "$PY" scripts/evaluation/serve_eval_web.py serve \
        --tenant-id "$TENANT_ID" \
2059d959   tangwang   feat(eval): 多评估集统...
46
        --dataset-id "$DATASET_ID" \
881d338b   tangwang   评估框架
47
        --queries-file "$QUERIES" \
7b8d9e1a   tangwang   评估框架的启动脚本
48
49
        --host "$EVAL_WEB_HOST" \
        --port "$EVAL_WEB_PORT"
881d338b   tangwang   评估框架
50
51
52
53
54
55
      ;;
    *)
      usage
      exit 1
      ;;
  esac