检索效果优化

tangwang
1 parent a47416ec
Showing 6 changed files with 133 additions and 7 deletions Show diff stats
config/config.yaml
query/query_parser.py
search/es_query_builder.py
search/searcher.py
tests/test_es_query_builder.py
tests/test_query_parser_mixed_language.py
@@ -18,6 +18,7 @@ field_boosts:
   title: 3.0
   brief: 1.5
   description: 1.0
+  qanchors: 1.5
   vendor: 1.5
   category_path: 1.5
   category_name_text: 1.5
@@ -135,8 +135,8 @@ class QueryParser:
             cfg = get_translation_config()
             logger.info(
                 "Initializing translator client at QueryParser construction (service_url=%s, default_model=%s)...",
-                cfg.service_url,
-                cfg.default_model,
+                cfg.get("service_url"),
+                cfg.get("default_model"),
             )
             self._translator = create_translation_client()
         self._translation_executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="query-translation")
@@ -207,6 +207,55 @@ class QueryParser:
             return [x[0] for x in tok_result] if tok_result else []
         return self._simple_tokenize(query)
  
+    @staticmethod
+    def _contains_cjk(text: str) -> bool:
+        """Whether query contains any CJK ideograph."""
+        return bool(re.search(r"[\u4e00-\u9fff]", text or ""))
+
+    @staticmethod
+    def _extract_latin_tokens(text: str) -> List[str]:
+        """Extract latin word tokens from query text."""
+        return re.findall(r"[A-Za-z]+(?:-[A-Za-z]+)*", text or "")
+
+    def _infer_supplemental_search_langs(
+        self,
+        query_text: str,
+        detected_lang: str,
+        index_langs: List[str],
+    ) -> List[str]:
+        """
+        Infer extra languages to search when the query mixes scripts.
+
+        Rules:
+        - If any Chinese characters appear, include `zh` when available.
+        - If the query contains meaningful latin tokens, include `en` when available.
+          "Meaningful" means either:
+          1) at least 2 latin tokens with length >= 4, or
+          2) at least 1 latin token with length >= 4 and latin chars occupy >= 20% of non-space chars.
+        """
+        supplemental: List[str] = []
+        normalized_index_langs = {str(lang or "").strip().lower() for lang in index_langs}
+        normalized_detected = str(detected_lang or "").strip().lower()
+        query_text = str(query_text or "")
+
+        if "zh" in normalized_index_langs and self._contains_cjk(query_text) and normalized_detected != "zh":
+            supplemental.append("zh")
+
+        latin_tokens = self._extract_latin_tokens(query_text)
+        significant_latin_tokens = [tok for tok in latin_tokens if len(tok) >= 4]
+        latin_chars = sum(len(tok) for tok in latin_tokens)
+        non_space_chars = len(re.sub(r"\s+", "", query_text))
+        latin_ratio = (latin_chars / non_space_chars) if non_space_chars > 0 else 0.0
+        has_meaningful_english = (
+            len(significant_latin_tokens) >= 2 or
+            (len(significant_latin_tokens) >= 1 and latin_ratio >= 0.2)
+        )
+
+        if "en" in normalized_index_langs and has_meaningful_english and normalized_detected != "en":
+            supplemental.append("en")
+
+        return supplemental
+
     def parse(
         self,
         query: str,
@@ -482,7 +531,17 @@ class QueryParser:
         for lang, translated_text in (translations or {}).items():
             if translated_text and str(translated_text).strip():
                 query_text_by_lang[str(lang).strip().lower()] = str(translated_text)
-        
+
+        supplemental_search_langs = self._infer_supplemental_search_langs(
+            query_text=query_text,
+            detected_lang=detected_lang,
+            index_langs=index_langs,
+        )
+        for lang in supplemental_search_langs:
+            if lang not in query_text_by_lang and query_text:
+                # Use the original mixed-script query as a robust fallback probe for that language field set.
+                query_text_by_lang[lang] = query_text
+
         source_in_index_languages = detected_lang in index_langs
         ordered_search_langs: List[str] = []
         seen_order = set()
@@ -501,6 +560,7 @@ class QueryParser:
         if context:
             context.store_intermediate_result("search_langs", ordered_search_langs)
             context.store_intermediate_result("query_text_by_lang", query_text_by_lang)
+            context.store_intermediate_result("supplemental_search_langs", supplemental_search_langs)
  
         # Build result
         result = ParsedQuery(
@@ -58,7 +58,7 @@ class ESQueryBuilder:
         self.match_fields = match_fields
         self.field_boosts = field_boosts or {}
         self.multilingual_fields = multilingual_fields or [
-            "title", "brief", "description", "vendor", "category_path", "category_name_text"
+            "title", "brief", "description", "qanchors", "vendor", "category_path", "category_name_text"
         ]
         self.shared_fields = shared_fields or ["tags", "option1_values", "option2_values", "option3_values"]
         self.core_multilingual_fields = core_multilingual_fields or ["title", "brief", "vendor", "category_name_text"]
@@ -276,7 +276,7 @@ class ESQueryBuilder:
                 "k": knn_k,
                 "num_candidates": knn_num_candidates,
                 "boost": knn_boost,
-                "name": "knn_query",
+                "_name": "knn_query",
             }
             # Top-level knn does not inherit query.bool.filter automatically.
             # Apply conjunctive + range filters here so vector recall respects hard filters.
@@ -670,9 +670,11 @@ class Searcher:
                     if rerank_debug:
                         debug_entry["doc_id"] = rerank_debug.get("doc_id")
                         # 与 rerank_client 中字段保持一致，便于前端直接使用
-                        debug_entry["es_score_norm"] = rerank_debug.get("es_score_norm")
                         debug_entry["rerank_score"] = rerank_debug.get("rerank_score")
+                        debug_entry["text_score"] = rerank_debug.get("text_score")
+                        debug_entry["knn_score"] = rerank_debug.get("knn_score")
                         debug_entry["fused_score"] = rerank_debug.get("fused_score")
+                        debug_entry["matched_queries"] = rerank_debug.get("matched_queries")
  
                     per_result_debug.append(debug_entry)
  
@@ -719,6 +721,9 @@ class Searcher:
                     "rewritten_query": context.query_analysis.rewritten_query,
                     "detected_language": context.query_analysis.detected_language,
                     "translations": context.query_analysis.translations,
+                    "query_text_by_lang": context.get_intermediate_result("query_text_by_lang", {}),
+                    "search_langs": context.get_intermediate_result("search_langs", []),
+                    "supplemental_search_langs": context.get_intermediate_result("supplemental_search_langs", []),
                     "has_vector": context.query_analysis.query_vector is not None,
                     "is_simple_query": context.query_analysis.is_simple_query,
                     "domain": context.query_analysis.domain
@@ -62,4 +62,4 @@ def test_knn_prefilter_not_added_without_filters():
  
     assert "knn" in q
     assert "filter" not in q["knn"]
-    assert q["knn"]["name"] == "knn_query"
+    assert q["knn"]["_name"] == "knn_query"
@@ -0,0 +1,60 @@
+from types import SimpleNamespace
+
+from config import FunctionScoreConfig, IndexConfig, QueryConfig, RerankConfig, SPUConfig, SearchConfig
+from query.query_parser import QueryParser
+
+
+class _DummyTranslator:
+    def translate(self, text, target_lang, source_lang, scene, model_name):
+        return f"{text}-{target_lang}"
+
+
+def _build_config() -> SearchConfig:
+    return SearchConfig(
+        es_index_name="test_products",
+        field_boosts={"title.en": 3.0, "title.zh": 3.0},
+        indexes=[IndexConfig(name="default", label="default", fields=["title.en", "title.zh"])],
+        query_config=QueryConfig(
+            enable_text_embedding=False,
+            enable_query_rewrite=False,
+            supported_languages=["en", "zh"],
+            default_language="zh",
+        ),
+        function_score=FunctionScoreConfig(),
+        rerank=RerankConfig(),
+        spu_config=SPUConfig(enabled=False),
+    )
+
+
+def test_parse_adds_en_fields_for_mixed_chinese_query_with_meaningful_english(monkeypatch):
+    parser = QueryParser(_build_config(), translator=_DummyTranslator())
+    monkeypatch.setattr(parser.language_detector, "detect", lambda text: "zh")
+    monkeypatch.setattr(
+        "query.query_parser.get_tenant_config_loader",
+        lambda: SimpleNamespace(get_tenant_config=lambda tenant_id: {"index_languages": ["zh", "en"]}),
+        raising=False,
+    )
+
+    result = parser.parse("法式 dress 连衣裙", tenant_id="162", generate_vector=False)
+
+    assert result.detected_language == "zh"
+    assert "en" in result.search_langs
+    assert result.query_text_by_lang["en"] == "法式 dress 连衣裙"
+    assert result.query_text_by_lang["zh"] == "法式 dress 连衣裙"
+
+
+def test_parse_adds_zh_fields_for_english_query_when_cjk_present(monkeypatch):
+    parser = QueryParser(_build_config(), translator=_DummyTranslator())
+    monkeypatch.setattr(parser.language_detector, "detect", lambda text: "en")
+    monkeypatch.setattr(
+        "query.query_parser.get_tenant_config_loader",
+        lambda: SimpleNamespace(get_tenant_config=lambda tenant_id: {"index_languages": ["en", "zh"]}),
+        raising=False,
+    )
+
+    result = parser.parse("red 连衣裙", tenant_id="0", generate_vector=False)
+
+    assert result.detected_language == "en"
+    assert "zh" in result.search_langs
+    assert result.query_text_by_lang["zh"] == "red 连衣裙"
+    assert result.query_text_by_lang["en"] == "red 连衣裙"