metrics.py
2.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
"""IR metrics for labeled result lists."""
from __future__ import annotations
from typing import Dict, Sequence
from .constants import RELEVANCE_EXACT, RELEVANCE_IRRELEVANT, RELEVANCE_HIGH, RELEVANCE_LOW, RELEVANCE_NON_IRRELEVANT
def precision_at_k(labels: Sequence[str], k: int, relevant: Sequence[str]) -> float:
if k <= 0:
return 0.0
sliced = list(labels[:k])
if not sliced:
return 0.0
rel = set(relevant)
hits = sum(1 for label in sliced if label in rel)
return hits / float(min(k, len(sliced)))
def average_precision(labels: Sequence[str], relevant: Sequence[str]) -> float:
rel = set(relevant)
hit_count = 0
precision_sum = 0.0
for idx, label in enumerate(labels, start=1):
if label not in rel:
continue
hit_count += 1
precision_sum += hit_count / idx
if hit_count == 0:
return 0.0
return precision_sum / hit_count
def compute_query_metrics(labels: Sequence[str]) -> Dict[str, float]:
"""P@k / MAP_3: Exact Match only. P@k_2_3 / MAP_2_3: any non-irrelevant tier (legacy metric names)."""
metrics: Dict[str, float] = {}
non_irrel = list(RELEVANCE_NON_IRRELEVANT)
for k in (5, 10, 20, 50):
metrics[f"P@{k}"] = round(precision_at_k(labels, k, [RELEVANCE_EXACT]), 6)
metrics[f"P@{k}_2_3"] = round(precision_at_k(labels, k, non_irrel), 6)
metrics["MAP_3"] = round(average_precision(labels, [RELEVANCE_EXACT]), 6)
metrics["MAP_2_3"] = round(average_precision(labels, non_irrel), 6)
return metrics
def aggregate_metrics(metric_items: Sequence[Dict[str, float]]) -> Dict[str, float]:
if not metric_items:
return {}
keys = sorted(metric_items[0].keys())
return {
key: round(sum(float(item.get(key, 0.0)) for item in metric_items) / len(metric_items), 6)
for key in keys
}
def label_distribution(labels: Sequence[str]) -> Dict[str, int]:
return {
RELEVANCE_EXACT: sum(1 for label in labels if label == RELEVANCE_EXACT),
RELEVANCE_HIGH: sum(1 for label in labels if label == RELEVANCE_HIGH),
RELEVANCE_LOW: sum(1 for label in labels if label == RELEVANCE_LOW),
RELEVANCE_IRRELEVANT: sum(1 for label in labels if label == RELEVANCE_IRRELEVANT),
}