feat(search): 混合语种查询分析与跨语言字段召回

## 背景多语言索引下，用户查询常中英混写；需在解析阶段显式标记脚本类型，并在 BM25 子句中同时覆盖对应语言字段。 ## 方案 ### 1. Query 分析（query_parser.ParsedQuery） - 新增 `contains_chinese`：query 文本含 CJK（沿用 _contains_cjk）。 - 新增 `contains_english`：分词结果中存在「纯英文、len>=3」token（fullmatch 字母及可选连字符）。 - 写入 to_dict、请求 context 中间结果，便于调试与 API 透出。 ### 2. ES 文本召回（es_query_builder._build_advanced_text_query） - 对每个 search_lang 子句：若含英文且子句语言非 en（且租户 index_languages 含 en），合并 en 列字段；若含中文且子句语言非 zh（且含 zh），合并 zh 列字段。 - 合并进来的字段 boost 乘以 `mixed_script_merged_field_boost_scale`（默认 0.8，可在 ESQueryBuilder 构造参数调整）。 - fallback_original_query_* 分支同样应用上述逻辑。 ### 3. 实现整理 - 引入 `MatchFieldSpec = (field_path, boost)`：`_build_match_field_specs` 为唯一权重来源；`_merge_supplemental_lang_field_specs` / `_expand_match_field_specs_for_mixed_script` 在 tuple 上合并与缩放；最后 `_format_match_field_specs` 再格式化为 ES `path^boost`，避免先拼字符串再解析。 ## 测试 - tests/test_query_parser_mixed_language.py：脚本标记与 token 规则。 - tests/test_es_query_builder.py：合并字段、0.8 缩放、index_languages 限制。 Made-with: Cursor

feat(search): 混合语种查询分析与跨语言字段召回
## 背景多语言索引下，用户查询常中英混写；需在解析阶段显式标记脚本类型，并在 BM25 子句中同时覆盖对应语言字段。 ## 方案 ### 1. Query 分析（query_parser.ParsedQuery） - 新增 `contains_chinese`：query 文本含 CJK（沿用 _contains_cjk）。 - 新增 `contains_english`：分词结果中存在「纯英文、len>=3」token（fullmatch 字母及可选连字符）。 - 写入 to_dict、请求 context 中间结果，便于调试与 API 透出。 ### 2. ES 文本召回（es_query_builder._build_advanced_text_query） - 对每个 search_lang 子句：若含英文且子句语言非 en（且租户 index_languages 含 en），合并 en 列字段；若含中文且子句语言非 zh（且含 zh），合并 zh 列字段。 - 合并进来的字段 boost 乘以 `mixed_script_merged_field_boost_scale`（默认 0.8，可在 ESQueryBuilder 构造参数调整）。 - fallback_original_query_* 分支同样应用上述逻辑。 ### 3. 实现整理 - 引入 `MatchFieldSpec = (field_path, boost)`：`_build_match_field_specs` 为唯一权重来源；`_merge_supplemental_lang_field_specs` / `_expand_match_field_specs_for_mixed_script` 在 tuple 上合并与缩放；最后 `_format_match_field_specs` 再格式化为 ES `path^boost`，避免先拼字符串再解析。 ## 测试 - tests/test_query_parser_mixed_language.py：脚本标记与 token 规则。 - tests/test_es_query_builder.py：合并字段、0.8 缩放、index_languages 限制。 Made-with: Cursor
tangwang
1 parent 1556989b
Showing 4 changed files with 223 additions and 22 deletions Show diff stats
query/query_parser.py
search/es_query_builder.py
tests/test_es_query_builder.py
tests/test_query_parser_mixed_language.py
@@ -42,6 +42,8 @@ class ParsedQuery:
         search_langs: Optional[List[str]] = None,
         index_languages: Optional[List[str]] = None,
         source_in_index_languages: bool = True,
+        contains_chinese: bool = False,
+        contains_english: bool = False,
     ):
         self.original_query = original_query
         self.query_normalized = query_normalized
@@ -58,6 +60,8 @@ class ParsedQuery:
         self.search_langs = search_langs or []
         self.index_languages = index_languages or []
         self.source_in_index_languages = bool(source_in_index_languages)
+        self.contains_chinese = bool(contains_chinese)
+        self.contains_english = bool(contains_english)
     def to_dict(self) -> Dict[str, Any]:
         """Convert to dictionary representation."""
@@ -73,6 +77,8 @@ class ParsedQuery:
         result["search_langs"] = self.search_langs
         result["index_languages"] = self.index_languages
         result["source_in_index_languages"] = self.source_in_index_languages
+        result["contains_chinese"] = self.contains_chinese
+        result["contains_english"] = self.contains_english
         return result
@@ -217,6 +223,16 @@ class QueryParser:
         return bool(re.search(r"[\u4e00-\u9fff]", text or ""))
     @staticmethod
+    def _is_pure_english_word_token(token: str) -> bool:
+        """
+        A tokenizer token counts as English iff it is letters only (optional internal hyphens)
+        and length >= 3.
+        """
+        if not token or len(token) < 3:
+            return False
+        return bool(re.fullmatch(r"[A-Za-z]+(?:-[A-Za-z]+)*", token))
+
+    @staticmethod
     def _extract_latin_tokens(text: str) -> List[str]:
         """Extract latin word tokens from query text."""
         return re.findall(r"[A-Za-z]+(?:-[A-Za-z]+)*", text or "")
@@ -391,13 +407,18 @@ class QueryParser:
         keywords = self._extract_keywords(query_text)
         query_tokens = self._get_query_tokens(query_text)
         token_count = len(query_tokens)
+        contains_chinese = self._contains_cjk(query_text)
+        contains_english = any(self._is_pure_english_word_token(t) for t in query_tokens)
         log_debug(f"Query analysis | Keywords: {keywords} | Token count: {token_count} | "
-                 f"Query tokens: {query_tokens}")
+                 f"Query tokens: {query_tokens} | contains_chinese={contains_chinese} | "
+                 f"contains_english={contains_english}")
         if context:
             context.store_intermediate_result('keywords', keywords)
             context.store_intermediate_result('token_count', token_count)
             context.store_intermediate_result('query_tokens', query_tokens)
+            context.store_intermediate_result('contains_chinese', contains_chinese)
+            context.store_intermediate_result('contains_english', contains_english)
         # Stage 6: Text embedding (only for non-short queries) - async execution
         query_vector = None
@@ -578,6 +599,8 @@ class QueryParser:
             search_langs=ordered_search_langs,
             index_languages=index_langs,
             source_in_index_languages=source_in_index_languages,
+            contains_chinese=contains_chinese,
+            contains_english=contains_english,
         )
         if context and hasattr(context, 'logger'):
@@ -9,9 +9,13 @@ Simplified architecture:
 """
 from typing import Dict, Any, List, Optional, Union, Tuple
+
 import numpy as np
 from config import FunctionScoreConfig
+# (Elasticsearch field path, boost before formatting as "path^boost")
+MatchFieldSpec = Tuple[str, float]
+
 class ESQueryBuilder:
     """Builds Elasticsearch DSL queries."""
@@ -36,6 +40,7 @@ class ESQueryBuilder:
         source_boost_when_missing: float = 0.6,
         original_query_fallback_boost_when_translation_missing: float = 0.2,
         tie_breaker_base_query: float = 0.9,
+        mixed_script_merged_field_boost_scale: float = 0.6,
     ):
         """
         Initialize query builder.
@@ -51,6 +56,7 @@ class ESQueryBuilder:
             function_score_config: Function score configuration
             default_language: Default language to use when detection fails or returns "unknown"
             knn_boost: Boost value for KNN (embedding recall)
+            mixed_script_merged_field_boost_scale: Multiply per-field ^boost for cross-script merged fields
         """
         self.match_fields = match_fields
         self.field_boosts = field_boosts or {}
@@ -74,6 +80,7 @@ class ESQueryBuilder:
             original_query_fallback_boost_when_translation_missing
         )
         self.tie_breaker_base_query = float(tie_breaker_base_query)
+        self.mixed_script_merged_field_boost_scale = float(mixed_script_merged_field_boost_scale)
     def _apply_source_filter(self, es_query: Dict[str, Any]) -> None:
         """
@@ -414,7 +421,7 @@ class ESQueryBuilder:
     def _format_field_with_boost(self, field_name: str, boost: float) -> str:
         if abs(float(boost) - 1.0) < 1e-9:
             return field_name
-        return f"{field_name}^{boost}"
+        return f"{field_name}^{round(boost, 2)}"
     def _get_field_boost(self, base_field: str, language: Optional[str] = None) -> float:
         # Language-specific override first (e.g. title.de), then base field (e.g. title)
@@ -426,36 +433,74 @@ class ESQueryBuilder:
             return float(self.field_boosts[base_field])
         return 1.0
-    def _get_match_fields(self, language: str) -> Tuple[List[str], List[str]]:
+    def _build_match_field_specs(self, language: str) -> Tuple[List[MatchFieldSpec], List[MatchFieldSpec]]:
         """
-        Build dynamic match fields for one language.
-        
-        Args:
-            language: Language code (e.g. zh/en/de/fr/...)
-            
-        Returns:
-            (all_fields, core_fields) - core_fields are for phrase/keyword queries
+        Per-language match targets as (field_path, boost). Single source of truth before string formatting.
+        Returns (all_fields, core_fields); core_fields are for phrase/keyword strategies elsewhere.
         """
         lang = (language or "").strip().lower()
-        all_fields: List[str] = []
-        core_fields: List[str] = []
+        all_specs: List[MatchFieldSpec] = []
+        core_specs: List[MatchFieldSpec] = []
         for base in self.multilingual_fields:
             field = f"{base}.{lang}"
-            boost = self._get_field_boost(base, lang)
-            all_fields.append(self._format_field_with_boost(field, boost))
+            all_specs.append((field, self._get_field_boost(base, lang)))
         for shared in self.shared_fields:
-            boost = self._get_field_boost(shared, None)
-            all_fields.append(self._format_field_with_boost(shared, boost))
+            all_specs.append((shared, self._get_field_boost(shared, None)))
         for base in self.core_multilingual_fields:
             field = f"{base}.{lang}"
-            boost = self._get_field_boost(base, lang)
-            core_fields.append(self._format_field_with_boost(field, boost))
+            core_specs.append((field, self._get_field_boost(base, lang)))
+
+        return all_specs, core_specs
+
+    def _format_match_field_specs(self, specs: List[MatchFieldSpec]) -> List[str]:
+        """Format (field_path, boost) pairs for Elasticsearch multi_match ``fields``."""
+        return [self._format_field_with_boost(path, boost) for path, boost in specs]
+
+    def _merge_supplemental_lang_field_specs(
+        self,
+        specs: List[MatchFieldSpec],
+        supplemental_lang: str,
+    ) -> List[MatchFieldSpec]:
+        """Append supplemental-language columns; boosts multiplied by mixed_script scale."""
+        scale = float(self.mixed_script_merged_field_boost_scale)
+        extra_all, _ = self._build_match_field_specs(supplemental_lang)
+        seen = {path for path, _ in specs}
+        out = list(specs)
+        for path, boost in extra_all:
+            if path not in seen:
+                out.append((path, boost * scale))
+                seen.add(path)
+        return out
+
+    def _expand_match_field_specs_for_mixed_script(
+        self,
+        lang: str,
+        specs: List[MatchFieldSpec],
+        contains_chinese: bool,
+        contains_english: bool,
+        index_languages: List[str],
+    ) -> List[MatchFieldSpec]:
+        """
+        When the query mixes scripts, widen each clause to indexed fields for the other script
+        (e.g. zh clause also searches title.en when the query contains an English word token).
+        """
+        norm = {str(x or "").strip().lower() for x in (index_languages or []) if str(x or "").strip()}
+        allow = norm or {"zh", "en"}
+
+        def can_use(lcode: str) -> bool:
+            return lcode in allow if norm else True
+
+        out = list(specs)
+        lnorm = (lang or "").strip().lower()
+        if contains_english and lnorm != "en" and can_use("en"):
+            out = self._merge_supplemental_lang_field_specs(out, "en")
+        if contains_chinese and lnorm != "zh" and can_use("zh"):
+            out = self._merge_supplemental_lang_field_specs(out, "zh")
+        return out
-        return all_fields, core_fields
-    
     def _get_embedding_field(self, language: str) -> str:
         """Get embedding field name for a language."""
         # Currently using unified embedding field
@@ -486,6 +531,8 @@ class ESQueryBuilder:
         source_in_index_languages = True
         index_languages: List[str] = []
+        contains_chinese = False
+        contains_english = False
         if parsed_query:
             query_text_by_lang = getattr(parsed_query, "query_text_by_lang", None) or {}
             search_langs = getattr(parsed_query, "search_langs", None) or []
@@ -495,6 +542,8 @@ class ESQueryBuilder:
                 getattr(parsed_query, "source_in_index_languages", True)
             )
             index_languages = getattr(parsed_query, "index_languages", None) or []
+            contains_chinese = bool(getattr(parsed_query, "contains_chinese", False))
+            contains_english = bool(getattr(parsed_query, "contains_english", False))
         if not query_text_by_lang:
             query_text_by_lang = {source_lang: query_text}
@@ -508,7 +557,15 @@ class ESQueryBuilder:
             lang_query = query_text_by_lang.get(lang)
             if not lang_query:
                 continue
-            match_fields, _ = self._get_match_fields(lang)
+            all_specs, _ = self._build_match_field_specs(lang)
+            expanded_specs = self._expand_match_field_specs_for_mixed_script(
+                lang,
+                all_specs,
+                contains_chinese,
+                contains_english,
+                index_languages,
+            )
+            match_fields = self._format_match_field_specs(expanded_specs)
             if not match_fields:
                 continue
@@ -559,7 +616,15 @@ class ESQueryBuilder:
                     continue
                 if lang in query_text_by_lang:
                     continue
-                match_fields, _ = self._get_match_fields(lang)
+                fb_specs, _ = self._build_match_field_specs(lang)
+                expanded_fb = self._expand_match_field_specs_for_mixed_script(
+                    lang,
+                    fb_specs,
+                    contains_chinese,
+                    contains_english,
+                    index_languages,
+                )
+                match_fields = self._format_match_field_specs(expanded_fb)
                 if not match_fields:
                     continue
                 should_clauses.append({
@@ -80,3 +80,102 @@ def test_text_query_contains_only_base_translation_and_fallback_named_queries():
     names = [clause["multi_match"]["_name"] for clause in should]
     assert names == ["base_query", "base_query_trans_zh", "fallback_original_query_fr"]
+
+
+def test_mixed_script_merges_en_fields_into_zh_clause():
+    qb = ESQueryBuilder(
+        match_fields=["title.en^3.0"],
+        multilingual_fields=["title", "brief"],
+        shared_fields=[],
+        text_embedding_field="title_embedding",
+        default_language="en",
+    )
+    parsed_query = SimpleNamespace(
+        query_text_by_lang={"zh": "法式 dress"},
+        search_langs=["zh"],
+        detected_language="zh",
+        source_in_index_languages=True,
+        index_languages=["zh", "en"],
+        contains_chinese=True,
+        contains_english=True,
+    )
+    q = qb.build_query(query_text="法式 dress", parsed_query=parsed_query, enable_knn=False)
+    fields = q["query"]["multi_match"]["fields"]
+    bases = {f.split("^", 1)[0] for f in fields}
+    assert "title.zh" in bases and "title.en" in bases
+    assert "brief.zh" in bases and "brief.en" in bases
+    # Merged supplemental language fields use boost * 0.8 (implicit 1.0 -> ^0.8)
+    assert "title.en^0.8" in fields
+    assert "brief.en^0.8" in fields
+
+
+def test_mixed_script_merges_zh_fields_into_en_clause():
+    qb = ESQueryBuilder(
+        match_fields=["title.en^3.0"],
+        multilingual_fields=["title"],
+        shared_fields=[],
+        text_embedding_field="title_embedding",
+        default_language="en",
+    )
+    parsed_query = SimpleNamespace(
+        query_text_by_lang={"en": "red 连衣裙"},
+        search_langs=["en"],
+        detected_language="en",
+        source_in_index_languages=True,
+        index_languages=["zh", "en"],
+        contains_chinese=True,
+        contains_english=True,
+    )
+    q = qb.build_query(query_text="red 连衣裙", parsed_query=parsed_query, enable_knn=False)
+    fields = q["query"]["multi_match"]["fields"]
+    bases = {f.split("^", 1)[0] for f in fields}
+    assert "title.en" in bases and "title.zh" in bases
+    assert "title.zh^0.8" in fields
+
+
+def test_mixed_script_merged_fields_scale_configured_boosts():
+    qb = ESQueryBuilder(
+        match_fields=["title.en^3.0"],
+        multilingual_fields=["title"],
+        shared_fields=[],
+        field_boosts={"title.zh": 5.0, "title.en": 10.0},
+        text_embedding_field="title_embedding",
+        default_language="en",
+    )
+    parsed_query = SimpleNamespace(
+        query_text_by_lang={"zh": "法式 dress"},
+        search_langs=["zh"],
+        detected_language="zh",
+        source_in_index_languages=True,
+        index_languages=["zh", "en"],
+        contains_chinese=True,
+        contains_english=True,
+    )
+    q = qb.build_query(query_text="法式 dress", parsed_query=parsed_query, enable_knn=False)
+    fields = q["query"]["multi_match"]["fields"]
+    assert "title.zh^5.0" in fields
+    assert "title.en^8.0" in fields  # 10.0 * 0.8
+
+
+def test_mixed_script_does_not_merge_en_when_not_in_index_languages():
+    qb = ESQueryBuilder(
+        match_fields=["title.zh^3.0"],
+        multilingual_fields=["title"],
+        shared_fields=[],
+        text_embedding_field="title_embedding",
+        default_language="zh",
+    )
+    parsed_query = SimpleNamespace(
+        query_text_by_lang={"zh": "法式 dress"},
+        search_langs=["zh"],
+        detected_language="zh",
+        source_in_index_languages=True,
+        index_languages=["zh"],
+        contains_chinese=True,
+        contains_english=True,
+    )
+    q = qb.build_query(query_text="法式 dress", parsed_query=parsed_query, enable_knn=False)
+    fields = q["query"]["multi_match"]["fields"]
+    bases = {f.split("^", 1)[0] for f in fields}
+    assert "title.zh" in bases
+    assert "title.en" not in bases
@@ -9,6 +9,14 @@ class _DummyTranslator:
         return f"{text}-{target_lang}"
+def test_pure_english_word_token_length_and_script():
+    assert QueryParser._is_pure_english_word_token("ab") is False
+    assert QueryParser._is_pure_english_word_token("abc") is True
+    assert QueryParser._is_pure_english_word_token("wi-fi") is True
+    assert QueryParser._is_pure_english_word_token("连衣裙") is False
+    assert QueryParser._is_pure_english_word_token("ab12") is False
+
+
 def _build_config() -> SearchConfig:
     return SearchConfig(
         es_index_name="test_products",
@@ -38,6 +46,8 @@ def test_parse_adds_en_fields_for_mixed_chinese_query_with_meaningful_english(mo
     result = parser.parse("法式 dress 连衣裙", tenant_id="162", generate_vector=False)
     assert result.detected_language == "zh"
+    assert result.contains_chinese is True
+    assert result.contains_english is True
     assert "en" in result.search_langs
     # 翻译在预算内完成时会写入目标语言字段（优于仅用原文做 supplemental 探测）
     assert result.query_text_by_lang["en"] == "法式 dress 连衣裙-en"
@@ -56,6 +66,8 @@ def test_parse_adds_zh_fields_for_english_query_when_cjk_present(monkeypatch):
     result = parser.parse("red 连衣裙", tenant_id="0", generate_vector=False)
     assert result.detected_language == "en"
+    assert result.contains_chinese is True
+    assert result.contains_english is True
     assert "zh" in result.search_langs
     assert result.query_text_by_lang["zh"] == "red 连衣裙-zh"
     assert result.query_text_by_lang["en"] == "red 连衣裙"
@@ -74,6 +86,8 @@ def test_parse_waits_for_translation_when_source_in_index_languages(monkeypatch)
     result = parser.parse("off shoulder top", tenant_id="0", generate_vector=False)
     assert result.detected_language == "en"
+    assert result.contains_chinese is False
+    assert result.contains_english is True
     assert result.translations.get("zh") == "off shoulder top-zh"
     assert result.query_text_by_lang.get("zh") == "off shoulder top-zh"
     assert result.source_in_index_languages is True