sku筛选匹配规则优化，按 token/短语序列匹配，fixbadcase

tangwang
1 parent b712a831
Showing 3 changed files with 146 additions and 4 deletions Show diff stats
config/config.yaml
search/sku_intent_selector.py
tests/test_sku_intent_selector.py
@@ -114,7 +114,7 @@ query_config:
   # 查询解析阶段：翻译与 query 向量并发执行，共用同一等待预算（毫秒）。
   # 检测语言已在租户 index_languages 内：较短；不在索引语言内：较长（翻译对召回更关键）。
   translation_embedding_wait_budget_ms_source_in_index: 500 # 80
-  translation_embedding_wait_budget_ms_source_not_in_index: 500 #200
+  translation_embedding_wait_budget_ms_source_not_in_index: 700 #200
  
   style_intent:
     enabled: true
@@ -380,7 +380,7 @@ services:
       max_docs: 1000
       normalize: true
     # 服务内后端（reranker 进程启动时读取）
-    backend: "bge"  # bge | qwen3_vllm | qwen3_transformers | dashscope_rerank
+    backend: "qwen3_vllm"  # bge | qwen3_vllm | qwen3_transformers | dashscope_rerank
     backends:
       bge:
         model_name: "BAAI/bge-reranker-v2-m3"
@@ -8,7 +8,7 @@ from dataclasses import dataclass, field
 from typing import Any, Callable, Dict, List, Optional, Tuple
  
 from query.style_intent import StyleIntentProfile, StyleIntentRegistry
-from query.tokenization import normalize_query_text
+from query.tokenization import normalize_query_text, simple_tokenize_query
  
  
 @dataclass(frozen=True)
@@ -35,6 +35,7 @@ class SkuSelectionDecision:
 class _SelectionContext:
     attribute_terms_by_intent: Dict[str, Tuple[str, ...]]
     normalized_text_cache: Dict[str, str] = field(default_factory=dict)
+    tokenized_text_cache: Dict[str, Tuple[str, ...]] = field(default_factory=dict)
     text_match_cache: Dict[Tuple[str, str], bool] = field(default_factory=dict)
  
  
@@ -194,10 +195,60 @@ class StyleSkuSelector:
             return cached
  
         attribute_terms = selection_context.attribute_terms_by_intent.get(intent_type, ())
-        matched = any(term in normalized_value for term in attribute_terms if term)
+        value_tokens = self._tokenize_cached(selection_context, normalized_value)
+        matched = any(
+            self._matches_term_tokens(
+                term=term,
+                value_tokens=value_tokens,
+                selection_context=selection_context,
+                normalized_value=normalized_value,
+            )
+            for term in attribute_terms
+            if term
+        )
         selection_context.text_match_cache[cache_key] = matched
         return matched
  
+    @staticmethod
+    def _tokenize_cached(selection_context: _SelectionContext, value: str) -> Tuple[str, ...]:
+        normalized_value = normalize_query_text(value)
+        if not normalized_value:
+            return ()
+        cached = selection_context.tokenized_text_cache.get(normalized_value)
+        if cached is not None:
+            return cached
+        tokens = tuple(normalize_query_text(token) for token in simple_tokenize_query(normalized_value) if token)
+        selection_context.tokenized_text_cache[normalized_value] = tokens
+        return tokens
+
+    def _matches_term_tokens(
+        self,
+        *,
+        term: str,
+        value_tokens: Tuple[str, ...],
+        selection_context: _SelectionContext,
+        normalized_value: str,
+    ) -> bool:
+        normalized_term = normalize_query_text(term)
+        if not normalized_term:
+            return False
+        if normalized_term == normalized_value:
+            return True
+
+        term_tokens = self._tokenize_cached(selection_context, normalized_term)
+        if not term_tokens or not value_tokens:
+            return normalized_term in normalized_value
+
+        term_length = len(term_tokens)
+        value_length = len(value_tokens)
+        if term_length > value_length:
+            return False
+
+        for start in range(value_length - term_length + 1):
+            if value_tokens[start:start + term_length] == term_tokens:
+                return True
+        return False
+
     def _find_first_text_match(
         self,
         skus: List[Dict[str, Any]],
@@ -104,3 +104,94 @@ def test_style_sku_selector_returns_no_match_without_attribute_contains():
  
     assert decisions["spu-1"].selected_sku_id is None
     assert decisions["spu-1"].matched_stage == "no_match"
+
+
+def test_is_text_match_uses_token_boundaries_for_sizes():
+    registry = StyleIntentRegistry.from_query_config(
+        QueryConfig(
+            style_intent_terms={
+                "size": [{"en_terms": ["l"], "zh_terms": ["大码"], "attribute_terms": ["l"]}],
+            },
+            style_intent_dimension_aliases={"size": ["size", "尺码"]},
+        )
+    )
+    selector = StyleSkuSelector(registry)
+    style_profile = StyleIntentProfile(
+        intents=(
+            DetectedStyleIntent(
+                intent_type="size",
+                canonical_value="l",
+                matched_term="l",
+                matched_query_text="l",
+                attribute_terms=("l",),
+                dimension_aliases=("size", "尺码"),
+            ),
+        ),
+    )
+    selection_context = selector._build_selection_context(style_profile)
+
+    assert selector._is_text_match("size", selection_context, normalized_value="l")
+    assert not selector._is_text_match("size", selection_context, normalized_value="xl")
+    assert not selector._is_text_match("size", selection_context, normalized_value="xxl")
+
+
+def test_is_text_match_handles_punctuation_and_descriptive_attribute_values():
+    registry = StyleIntentRegistry.from_query_config(
+        QueryConfig(
+            style_intent_terms={
+                "color": [{"en_terms": ["blue"], "zh_terms": ["蓝色"], "attribute_terms": ["blue"]}],
+                "style": [{"en_terms": ["off-white"], "zh_terms": ["米白"], "attribute_terms": ["off-white"]}],
+                "accessory": [{"en_terms": ["headscarf"], "zh_terms": ["头巾"], "attribute_terms": ["headscarf"]}],
+                "size": [{"en_terms": ["2xl"], "zh_terms": ["2xl"], "attribute_terms": ["2xl"]}],
+            },
+            style_intent_dimension_aliases={
+                "color": ["color", "颜色"],
+                "style": ["style", "风格"],
+                "accessory": ["accessory", "配饰"],
+                "size": ["size", "尺码"],
+            },
+        )
+    )
+    selector = StyleSkuSelector(registry)
+    style_profile = StyleIntentProfile(
+        intents=(
+            DetectedStyleIntent(
+                intent_type="color",
+                canonical_value="blue",
+                matched_term="blue",
+                matched_query_text="blue",
+                attribute_terms=("blue",),
+                dimension_aliases=("color", "颜色"),
+            ),
+            DetectedStyleIntent(
+                intent_type="style",
+                canonical_value="off-white",
+                matched_term="off-white",
+                matched_query_text="off-white",
+                attribute_terms=("off-white",),
+                dimension_aliases=("style", "风格"),
+            ),
+            DetectedStyleIntent(
+                intent_type="accessory",
+                canonical_value="headscarf",
+                matched_term="headscarf",
+                matched_query_text="headscarf",
+                attribute_terms=("headscarf",),
+                dimension_aliases=("accessory", "配饰"),
+            ),
+            DetectedStyleIntent(
+                intent_type="size",
+                canonical_value="2xl",
+                matched_term="2xl",
+                matched_query_text="2xl",
+                attribute_terms=("2xl",),
+                dimension_aliases=("size", "尺码"),
+            ),
+        ),
+    )
+    selection_context = selector._build_selection_context(style_profile)
+
+    assert selector._is_text_match("color", selection_context, normalized_value="gray blue")
+    assert selector._is_text_match("style", selection_context, normalized_value="off-white/lined")
+    assert selector._is_text_match("accessory", selection_context, normalized_value="army green + headscarf")
+    assert selector._is_text_match("size", selection_context, normalized_value="2xl recommended 65-70kg")