图片文本两个knn召回相关参数配置

tangwang
1 parent 1681a135
Showing 8 changed files with 88 additions and 41 deletions Show diff stats
config/config.yaml
config/loader.py
config/schema.py
docs/常用查询 - sql.sql
search/es_query_builder.py
search/searcher.py
tests/test_es_query_builder.py
tests/test_es_query_builder_text_recall_languages.py
@@ -206,8 +206,18 @@ query_config:
     - specifications
     - skus
  
-  # KNN boost配置（向量召回的boost值）
-  knn_boost: 2.0  # Lower boost for embedding recall
+  # KNN：文本向量与多模态（图片）向量各自 boost 与召回（k / num_candidates）
+  knn_text_boost: 20
+  knn_image_boost: 20
+
+  knn_text_k: 150
+  knn_text_num_candidates: 400
+
+  knn_text_k_long: 300
+  knn_text_num_candidates_long: 720
+
+  knn_image_k: 300
+  knn_image_num_candidates: 720
  
 # Function Score配置（ES层打分规则）
 function_score:
@@ -376,7 +376,20 @@ class AppConfigLoader:
             text_embedding_field=query_cfg.get("text_embedding_field"),
             image_embedding_field=query_cfg.get("image_embedding_field"),
             source_fields=query_cfg.get("source_fields"),
-            knn_boost=float(query_cfg.get("knn_boost", 0.25)),
+            knn_text_boost=float(
+                query_cfg.get("knn_text_boost", query_cfg.get("knn_boost", 0.25))
+            ),
+            knn_image_boost=float(
+                query_cfg.get("knn_image_boost", query_cfg.get("knn_boost", 0.25))
+            ),
+            knn_text_k=int(query_cfg.get("knn_text_k", 120)),
+            knn_text_num_candidates=int(query_cfg.get("knn_text_num_candidates", 400)),
+            knn_text_k_long=int(query_cfg.get("knn_text_k_long", 160)),
+            knn_text_num_candidates_long=int(
+                query_cfg.get("knn_text_num_candidates_long", 500)
+            ),
+            knn_image_k=int(query_cfg.get("knn_image_k", 120)),
+            knn_image_num_candidates=int(query_cfg.get("knn_image_num_candidates", 400)),
             multilingual_fields=list(
                 search_fields.get(
                     "multilingual_fields",
@@ -34,7 +34,15 @@ class QueryConfig:
     text_embedding_field: Optional[str] = "title_embedding"
     image_embedding_field: Optional[str] = None
     source_fields: Optional[List[str]] = None
-    knn_boost: float = 0.25
+    # 文本向量 KNN 与多模态（图片）向量 KNN 各自 boost；未在 YAML 中写时由 loader 用 legacy knn_boost 回填
+    knn_text_boost: float = 20.0
+    knn_image_boost: float = 20.0
+    knn_text_k: int = 120
+    knn_text_num_candidates: int = 400
+    knn_text_k_long: int = 160
+    knn_text_num_candidates_long: int = 500
+    knn_image_k: int = 120
+    knn_image_num_candidates: int = 400
     multilingual_fields: List[str] = field(
         default_factory=lambda: []
     )
@@ -584,18 +584,14 @@ SELECT &#39;shoplazza_product_index_increment&#39;, COUNT(*) FROM shoplazza_product_inde
 "
  
 # 执行删除
-cd /data/saas-search && MYSQL_PWD='qY8tgodLoA&KT#yQ' mysql -h 10.200.16.14 -P 3316 -u root saas -N -e "
-SELECT 'shoplazza_sync_log', COUNT(*) FROM shoplazza_sync_log WHERE tenant_id = 163
-UNION ALL
-SELECT 'shoplazza_product_index_increment', COUNT(*) FROM shoplazza_product_index_increment WHERE tenant_id = 163;
+cd /data/saas-search && MYSQL_PWD='qY8tgodLoA&KT#yQ' mysql -h 10.200.16.14 -P 3316 -u root saas -e "
+SET SESSION sql_safe_updates = 0;
+DELETE FROM shoplazza_sync_log WHERE tenant_id = 163;
+SELECT ROW_COUNT() AS deleted_sync_log;
+DELETE FROM shoplazza_product_index_increment WHERE tenant_id = 163;
+SELECT ROW_COUNT() AS deleted_index_increment;
 "
  
-# 再次统计 tenant_id=163 的行数
-MYSQL_PWD='qY8tgodLoA&KT#yQ' mysql -h 10.200.16.14 -P 3316 -u root saas -N -e "
-SELECT 'shoplazza_sync_log', COUNT(*) FROM shoplazza_sync_log WHERE tenant_id = 163
-UNION ALL
-SELECT 'shoplazza_product_index_increment', COUNT(*) FROM shoplazza_product_index_increment WHERE tenant_id = 163;
-"
 ```
  
 然后触发重新安装：
@@ -29,7 +29,14 @@ class ESQueryBuilder:
         source_fields: Optional[List[str]] = None,
         function_score_config: Optional[FunctionScoreConfig] = None,
         default_language: str = "en",
-        knn_boost: float = 0.25,
+        knn_text_boost: float = 20.0,
+        knn_image_boost: float = 20.0,
+        knn_text_k: int = 120,
+        knn_text_num_candidates: int = 400,
+        knn_text_k_long: int = 160,
+        knn_text_num_candidates_long: int = 500,
+        knn_image_k: int = 120,
+        knn_image_num_candidates: int = 400,
         base_minimum_should_match: str = "70%",
         translation_minimum_should_match: str = "70%",
         translation_boost: float = 0.4,
@@ -55,7 +62,8 @@ class ESQueryBuilder:
             source_fields: Fields to return in search results (_source includes)
             function_score_config: Function score configuration
             default_language: Default language to use when detection fails or returns "unknown"
-            knn_boost: Boost value for KNN (embedding recall)
+            knn_text_boost: Boost for text-embedding KNN clause
+            knn_image_boost: Boost for image-embedding KNN clause
         """
         self.match_fields = match_fields
         self.field_boosts = field_boosts or {}
@@ -67,7 +75,14 @@ class ESQueryBuilder:
         self.source_fields = source_fields
         self.function_score_config = function_score_config
         self.default_language = default_language
-        self.knn_boost = knn_boost
+        self.knn_text_boost = float(knn_text_boost)
+        self.knn_image_boost = float(knn_image_boost)
+        self.knn_text_k = int(knn_text_k)
+        self.knn_text_num_candidates = int(knn_text_num_candidates)
+        self.knn_text_k_long = int(knn_text_k_long)
+        self.knn_text_num_candidates_long = int(knn_text_num_candidates_long)
+        self.knn_image_k = int(knn_image_k)
+        self.knn_image_num_candidates = int(knn_image_num_candidates)
         self.base_minimum_should_match = base_minimum_should_match
         self.translation_minimum_should_match = translation_minimum_should_match
         self.translation_boost = float(translation_boost)
@@ -171,8 +186,6 @@ class ESQueryBuilder:
         size: int = 10,
         from_: int = 0,
         enable_knn: bool = True,
-        knn_k: int = 50,
-        knn_num_candidates: int = 200,
         min_score: Optional[float] = None,
         parsed_query: Optional[Any] = None,
     ) -> Dict[str, Any]:
@@ -195,8 +208,6 @@ class ESQueryBuilder:
             size: Number of results
             from_: Offset for pagination
             enable_knn: Whether to use KNN search
-            knn_k: K value for KNN
-            knn_num_candidates: Number of candidates for KNN
             min_score: Minimum score threshold
  
         Returns:
@@ -234,41 +245,37 @@ class ESQueryBuilder:
             filter_clauses.append(product_title_exclusion_filter)
  
         # 3. Add KNN search clauses alongside lexical clauses under the same bool.should
-        # Adjust KNN k, num_candidates, boost by query_tokens (short query: less KNN; long: more)
-        final_knn_k, final_knn_num_candidates = knn_k, knn_num_candidates
+        # Text KNN: k / num_candidates from config; long queries use *_long and higher boost
         if has_embedding:
-            knn_boost = self.knn_boost
+            text_knn_boost = self.knn_text_boost
+            final_knn_k = self.knn_text_k
+            final_knn_num_candidates = self.knn_text_num_candidates
             if parsed_query:
                 query_tokens = getattr(parsed_query, 'query_tokens', None) or []
                 token_count = len(query_tokens)
                 if token_count >= 5:
-                    final_knn_k, final_knn_num_candidates = 160, 500
-                    knn_boost = self.knn_boost * 1.4  # Higher weight for long queries
-                else:
-                    final_knn_k, final_knn_num_candidates = 120, 400
-            else:
-                final_knn_k, final_knn_num_candidates = 120, 400
+                    final_knn_k = self.knn_text_k_long
+                    final_knn_num_candidates = self.knn_text_num_candidates_long
+                    text_knn_boost = self.knn_text_boost * 1.4
             recall_clauses.append({
                 "knn": {
                     "field": self.text_embedding_field,
                     "query_vector": query_vector.tolist(),
                     "k": final_knn_k,
                     "num_candidates": final_knn_num_candidates,
-                    "boost": knn_boost,
+                    "boost": text_knn_boost,
                     "_name": "knn_query",
                 }
             })
  
         if has_image_embedding:
-            image_knn_k = max(final_knn_k, 120)
-            image_knn_num_candidates = max(final_knn_num_candidates, 400)
             recall_clauses.append({
                 "knn": {
                     "field": self.image_embedding_field,
                     "query_vector": image_query_vector.tolist(),
-                    "k": image_knn_k,
-                    "num_candidates": image_knn_num_candidates,
-                    "boost": self.knn_boost,
+                    "k": self.knn_image_k,
+                    "num_candidates": self.knn_image_num_candidates,
+                    "boost": self.knn_image_boost,
                     "_name": "image_knn_query",
                 }
             })
@@ -133,7 +133,14 @@ class Searcher:
             source_fields=self.source_fields,
             function_score_config=self.config.function_score,
             default_language=self.config.query_config.default_language,
-            knn_boost=self.config.query_config.knn_boost,
+            knn_text_boost=self.config.query_config.knn_text_boost,
+            knn_image_boost=self.config.query_config.knn_image_boost,
+            knn_text_k=self.config.query_config.knn_text_k,
+            knn_text_num_candidates=self.config.query_config.knn_text_num_candidates,
+            knn_text_k_long=self.config.query_config.knn_text_k_long,
+            knn_text_num_candidates_long=self.config.query_config.knn_text_num_candidates_long,
+            knn_image_k=self.config.query_config.knn_image_k,
+            knn_image_num_candidates=self.config.query_config.knn_image_num_candidates,
             base_minimum_should_match=self.config.query_config.base_minimum_should_match,
             translation_minimum_should_match=self.config.query_config.translation_minimum_should_match,
             translation_boost=self.config.query_config.translation_boost,
@@ -119,9 +119,12 @@ def test_text_query_skips_duplicate_translation_same_as_base():
         enable_knn=False,
     )
  
-    root = _recall_root(q)
-    assert root["bool"]["_name"] == "base_query"
-    assert [clause["multi_match"]["type"] for clause in root["bool"]["should"]] == ["best_fields", "phrase"]
+    query_root = q["query"]
+    if "function_score" in query_root:
+        query_root = query_root["function_score"]["query"]
+    base_bool = query_root["bool"]
+    assert base_bool["_name"] == "base_query"
+    assert [clause["multi_match"]["type"] for clause in base_bool["should"]] == ["best_fields", "phrase"]
  
  
 def test_product_title_exclusion_filter_is_applied_once_on_outer_query():
@@ -351,7 +351,10 @@ def test_text_clauses_present_alongside_knn():
         parsed_query=parsed,
         enable_knn=True,
     )
-    assert "knn" in q
+    qr = q["query"]
+    if "function_score" in qr:
+        qr = qr["function_score"]["query"]
+    assert any("knn" in c for c in qr["bool"]["should"])
     idx = _clauses_index(q)
     assert set(idx) == {"base_query", "base_query_trans_zh"}