Blame view

search/searcher.py 71.9 KB
be52af70   tangwang   first commit
1
2
3
  """
  Main Searcher module - executes search queries against Elasticsearch.
  
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
4
  Handles query parsing, ranking, and result formatting.
be52af70   tangwang   first commit
5
6
  """
  
814e352b   tangwang   乘法公式配置化
7
8
  from typing import Dict, Any, List, Optional
  import json
325eec03   tangwang   1. 日志、配置基础设施,使用优化
9
  import logging
28e57bb1   tangwang   日志体系优化
10
  import hashlib
5f7d7f09   tangwang   性能测试报告.md
11
  from string import Formatter
be52af70   tangwang   first commit
12
  
be52af70   tangwang   first commit
13
14
  from utils.es_client import ESClient
  from query import QueryParser, ParsedQuery
cda1cd62   tangwang   意图分析&应用 baseline
15
  from query.style_intent import StyleIntentRegistry
07cf5a93   tangwang   START_EMBEDDING=...
16
  from embeddings.image_encoder import CLIPImageEncoder
be52af70   tangwang   first commit
17
  from .es_query_builder import ESQueryBuilder
cda1cd62   tangwang   意图分析&应用 baseline
18
  from .sku_intent_selector import SkuSelectionDecision, StyleSkuSelector
9f96d6f3   tangwang   短query不用语义搜索
19
  from config import SearchConfig
345d960b   tangwang   1. 删除全局 enable_tr...
20
  from config.tenant_config_loader import get_tenant_config_loader
ed948666   tangwang   tidy
21
  from context.request_context import RequestContext, RequestContextStage
814e352b   tangwang   乘法公式配置化
22
  from api.models import FacetResult, FacetConfig
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
23
  from api.result_formatter import ResultFormatter
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
24
  from indexer.mapping_generator import get_tenant_index_name
be52af70   tangwang   first commit
25
  
325eec03   tangwang   1. 日志、配置基础设施,使用优化
26
  logger = logging.getLogger(__name__)
28e57bb1   tangwang   日志体系优化
27
28
29
30
31
32
33
34
35
  backend_verbose_logger = logging.getLogger("backend.verbose")
  
  
  def _log_backend_verbose(payload: Dict[str, Any]) -> None:
      if not backend_verbose_logger.handlers:
          return
      backend_verbose_logger.info(
          json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
      )
325eec03   tangwang   1. 日志、配置基础设施,使用优化
36
  
be52af70   tangwang   first commit
37
  
465f90e1   tangwang   添加LTR数据收集
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
  def _summarize_ltr_features(per_result_debug: List[Dict[str, Any]], top_n: int = 20) -> Dict[str, Any]:
      rows = list(per_result_debug[:top_n])
      if not rows:
          return {"top_n": 0, "counts": {}, "averages": {}, "top_docs": []}
  
      def _feature(row: Dict[str, Any], key: str) -> Any:
          features = row.get("ltr_features")
          if isinstance(features, dict):
              return features.get(key)
          rerank_stage = row.get("ranking_funnel", {}).get("rerank", {})
          stage_features = rerank_stage.get("ltr_features")
          if isinstance(stage_features, dict):
              return stage_features.get(key)
          return None
  
      def _count(flag: str) -> int:
          return sum(1 for row in rows if bool(_feature(row, flag)))
  
      def _avg(name: str) -> float | None:
          values = [float(v) for row in rows if (v := _feature(row, name)) is not None]
          if not values:
              return None
          return round(sum(values) / len(values), 6)
  
      top_docs = []
      for row in rows[:10]:
          top_docs.append(
              {
                  "spu_id": row.get("spu_id"),
                  "final_rank": row.get("final_rank"),
                  "title_zh": row.get("title_multilingual", {}).get("zh")
                  if isinstance(row.get("title_multilingual"), dict)
                  else None,
                  "es_score": _feature(row, "es_score"),
                  "text_score": _feature(row, "text_score"),
                  "knn_score": _feature(row, "knn_score"),
                  "rerank_score": _feature(row, "rerank_score"),
                  "fine_score": _feature(row, "fine_score"),
                  "has_translation_match": _feature(row, "has_translation_match"),
                  "has_text_knn": _feature(row, "has_text_knn"),
                  "has_image_knn": _feature(row, "has_image_knn"),
                  "has_style_boost": _feature(row, "has_style_boost"),
              }
          )
  
      return {
          "top_n": len(rows),
          "counts": {
              "translation_match_docs": _count("has_translation_match"),
              "text_knn_docs": _count("has_text_knn"),
              "image_knn_docs": _count("has_image_knn"),
              "style_boost_docs": _count("has_style_boost"),
              "text_fallback_to_es_docs": _count("text_score_fallback_to_es"),
          },
          "averages": {
              "es_score": _avg("es_score"),
              "text_score": _avg("text_score"),
              "knn_score": _avg("knn_score"),
              "rerank_score": _avg("rerank_score"),
              "fine_score": _avg("fine_score"),
              "source_score": _avg("source_score"),
              "translation_score": _avg("translation_score"),
              "text_knn_score": _avg("text_knn_score"),
              "image_knn_score": _avg("image_knn_score"),
          },
          "top_docs": top_docs,
      }
  
  
be52af70   tangwang   first commit
107
  class SearchResult:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
108
      """Container for search results (外部友好格式)."""
be52af70   tangwang   first commit
109
110
111
  
      def __init__(
          self,
cadc77b6   tangwang   索引字段名、变量名、API数据结构...
112
          results: List[Any],  # List[SpuResult]
be52af70   tangwang   first commit
113
114
115
          total: int,
          max_score: float,
          took_ms: int,
6aa246be   tangwang   问题:Pydantic 应该能自动...
116
          facets: Optional[List[FacetResult]] = None,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
117
          query_info: Optional[Dict[str, Any]] = None,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
118
119
          suggestions: Optional[List[str]] = None,
          related_searches: Optional[List[str]] = None,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
120
          debug_info: Optional[Dict[str, Any]] = None
be52af70   tangwang   first commit
121
      ):
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
122
          self.results = results
be52af70   tangwang   first commit
123
124
125
          self.total = total
          self.max_score = max_score
          self.took_ms = took_ms
6aa246be   tangwang   问题:Pydantic 应该能自动...
126
          self.facets = facets
be52af70   tangwang   first commit
127
          self.query_info = query_info or {}
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
128
129
          self.suggestions = suggestions or []
          self.related_searches = related_searches or []
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
130
          self.debug_info = debug_info
43f1139f   tangwang   refactor: ES查询结构重...
131
  
be52af70   tangwang   first commit
132
133
      def to_dict(self) -> Dict[str, Any]:
          """Convert to dictionary representation."""
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
134
          result = {
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
135
              "results": [r.model_dump() if hasattr(r, 'model_dump') else r for r in self.results],
be52af70   tangwang   first commit
136
137
138
              "total": self.total,
              "max_score": self.max_score,
              "took_ms": self.took_ms,
6aa246be   tangwang   问题:Pydantic 应该能自动...
139
              "facets": [f.model_dump() for f in self.facets] if self.facets else None,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
140
141
142
              "query_info": self.query_info,
              "suggestions": self.suggestions,
              "related_searches": self.related_searches
be52af70   tangwang   first commit
143
          }
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
144
145
146
          if self.debug_info is not None:
              result["debug_info"] = self.debug_info
          return result
be52af70   tangwang   first commit
147
148
149
150
151
152
153
154
  
  
  class Searcher:
      """
      Main search engine class.
  
      Handles:
      - Query parsing and translation
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
155
      - Dynamic multi-language text recall planning
be52af70   tangwang   first commit
156
157
158
159
160
161
      - ES query building
      - Result ranking and formatting
      """
  
      def __init__(
          self,
be52af70   tangwang   first commit
162
          es_client: ESClient,
9f96d6f3   tangwang   短query不用语义搜索
163
          config: SearchConfig,
26b910bd   tangwang   refactor service ...
164
165
          query_parser: Optional[QueryParser] = None,
          image_encoder: Optional[CLIPImageEncoder] = None,
be52af70   tangwang   first commit
166
167
168
169
170
      ):
          """
          Initialize searcher.
  
          Args:
be52af70   tangwang   first commit
171
              es_client: Elasticsearch client
9f96d6f3   tangwang   短query不用语义搜索
172
              config: SearchConfig instance
be52af70   tangwang   first commit
173
              query_parser: Query parser (created if not provided)
26b910bd   tangwang   refactor service ...
174
              image_encoder: Optional pre-initialized image encoder
be52af70   tangwang   first commit
175
          """
be52af70   tangwang   first commit
176
          self.es_client = es_client
9f96d6f3   tangwang   短query不用语义搜索
177
          self.config = config
9f96d6f3   tangwang   短query不用语义搜索
178
          self.text_embedding_field = config.query_config.text_embedding_field or "title_embedding"
26b910bd   tangwang   refactor service ...
179
180
181
182
183
          self.image_embedding_field = config.query_config.image_embedding_field
          if self.image_embedding_field and image_encoder is None:
              self.image_encoder = CLIPImageEncoder()
          else:
              self.image_encoder = image_encoder
dc403578   tangwang   多模态搜索
184
185
          # Index name is now generated dynamically per tenant, no longer stored here
          self.query_parser = query_parser or QueryParser(config, image_encoder=self.image_encoder)
26b910bd   tangwang   refactor service ...
186
          self.source_fields = config.query_config.source_fields
cda1cd62   tangwang   意图分析&应用 baseline
187
188
189
190
          self.style_intent_registry = StyleIntentRegistry.from_query_config(self.config.query_config)
          self.style_sku_selector = StyleSkuSelector(
              self.style_intent_registry,
              text_encoder_getter=lambda: getattr(self.query_parser, "text_encoder", None),
cda1cd62   tangwang   意图分析&应用 baseline
191
          )
be52af70   tangwang   first commit
192
  
f0d020c3   tangwang   多语言查询改为只支持中英文两种,f...
193
          # Query builder - simplified single-layer architecture
f0d020c3   tangwang   多语言查询改为只支持中英文两种,f...
194
          self.query_builder = ESQueryBuilder(
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
195
196
197
198
199
              match_fields=[],
              field_boosts=self.config.field_boosts,
              multilingual_fields=self.config.query_config.multilingual_fields,
              shared_fields=self.config.query_config.shared_fields,
              core_multilingual_fields=self.config.query_config.core_multilingual_fields,
be52af70   tangwang   first commit
200
              text_embedding_field=self.text_embedding_field,
13377199   tangwang   接口优化
201
              image_embedding_field=self.image_embedding_field,
9f96d6f3   tangwang   短query不用语义搜索
202
              source_fields=self.source_fields,
7bc756c5   tangwang   优化 ES 查询构建
203
              function_score_config=self.config.function_score,
70dab99f   tangwang   add logs
204
              default_language=self.config.query_config.default_language,
ed13851c   tangwang   图片文本两个knn召回相关参数配置
205
206
207
208
209
210
211
212
              knn_text_boost=self.config.query_config.knn_text_boost,
              knn_image_boost=self.config.query_config.knn_image_boost,
              knn_text_k=self.config.query_config.knn_text_k,
              knn_text_num_candidates=self.config.query_config.knn_text_num_candidates,
              knn_text_k_long=self.config.query_config.knn_text_k_long,
              knn_text_num_candidates_long=self.config.query_config.knn_text_num_candidates_long,
              knn_image_k=self.config.query_config.knn_image_k,
              knn_image_num_candidates=self.config.query_config.knn_image_num_candidates,
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
213
214
215
              base_minimum_should_match=self.config.query_config.base_minimum_should_match,
              translation_minimum_should_match=self.config.query_config.translation_minimum_should_match,
              translation_boost=self.config.query_config.translation_boost,
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
216
              tie_breaker_base_query=self.config.query_config.tie_breaker_base_query,
e756b18e   tangwang   重构了文本召回构建器,现在每个 b...
217
218
219
220
              best_fields_boosts=self.config.query_config.best_fields,
              best_fields_clause_boost=self.config.query_config.best_fields_boost,
              phrase_field_boosts=self.config.query_config.phrase_fields,
              phrase_match_boost=self.config.query_config.phrase_match_boost,
be52af70   tangwang   first commit
221
222
          )
  
26b910bd   tangwang   refactor service ...
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
      def _apply_source_filter(self, es_query: Dict[str, Any]) -> None:
          """
          Apply tri-state _source semantics:
          - None: do not set _source (return full source)
          - []: _source=false (return no source fields)
          - [..]: _source.includes=[..]
          """
          if self.source_fields is None:
              return
          if not isinstance(self.source_fields, list):
              raise ValueError("query_config.source_fields must be null or list[str]")
          if len(self.source_fields) == 0:
              es_query["_source"] = False
              return
          es_query["_source"] = {"includes": self.source_fields}
  
cda1cd62   tangwang   意图分析&应用 baseline
239
240
241
242
243
      def _resolve_rerank_source_filter(
          self,
          doc_template: str,
          parsed_query: Optional[ParsedQuery] = None,
      ) -> Dict[str, Any]:
5f7d7f09   tangwang   性能测试报告.md
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
          """
          Build a lightweight _source filter for rerank prefetch.
  
          Only fetch fields required by rerank doc template to reduce ES payload.
          """
          field_map = {
              "title": "title",
              "brief": "brief",
              "vendor": "vendor",
              "description": "description",
              "category_path": "category_path",
          }
          includes: set[str] = set()
          template = str(doc_template or "{title}")
          for _, field_name, _, _ in Formatter().parse(template):
              if not field_name:
                  continue
              key = field_name.split(".", 1)[0].split("!", 1)[0].split(":", 1)[0]
              mapped = field_map.get(key)
              if mapped:
                  includes.add(mapped)
  
          # Fallback to title-only to keep rerank docs usable.
          if not includes:
              includes.add("title")
  
cda1cd62   tangwang   意图分析&应用 baseline
270
271
272
273
274
275
276
277
278
279
          if self._has_style_intent(parsed_query):
              includes.update(
                  {
                      "skus",
                      "option1_name",
                      "option2_name",
                      "option3_name",
                  }
              )
  
5f7d7f09   tangwang   性能测试报告.md
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
          return {"includes": sorted(includes)}
  
      def _fetch_hits_by_ids(
          self,
          index_name: str,
          doc_ids: List[str],
          source_spec: Optional[Any],
      ) -> tuple[Dict[str, Dict[str, Any]], int]:
          """
          Fetch page documents by IDs for final response fill.
  
          Returns:
              (hits_by_id, es_took_ms)
          """
          if not doc_ids:
              return {}, 0
  
          body: Dict[str, Any] = {
              "query": {
                  "ids": {
                      "values": doc_ids,
                  }
              }
          }
          if source_spec is not None:
              body["_source"] = source_spec
  
          resp = self.es_client.search(
              index_name=index_name,
              body=body,
              size=len(doc_ids),
              from_=0,
          )
          hits = resp.get("hits", {}).get("hits") or []
          hits_by_id: Dict[str, Dict[str, Any]] = {}
          for hit in hits:
              hid = hit.get("_id")
              if hid is None:
                  continue
              hits_by_id[str(hid)] = hit
          return hits_by_id, int(resp.get("took", 0) or 0)
  
deccd68a   tangwang   Added the SKU pre...
322
      @staticmethod
8c8b9d84   tangwang   ES 拉取 coarse_rank...
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
      def _restore_hits_in_doc_order(
          doc_ids: List[str],
          hits_by_id: Dict[str, Dict[str, Any]],
      ) -> List[Dict[str, Any]]:
          ordered_hits: List[Dict[str, Any]] = []
          for doc_id in doc_ids:
              hit = hits_by_id.get(str(doc_id))
              if hit is not None:
                  ordered_hits.append(hit)
          return ordered_hits
  
      @staticmethod
      def _merge_source_specs(*source_specs: Any) -> Optional[Dict[str, Any]]:
          includes: set[str] = set()
          for source_spec in source_specs:
              if not isinstance(source_spec, dict):
                  continue
              for field_name in source_spec.get("includes") or []:
                  includes.add(str(field_name))
          if not includes:
              return None
          return {"includes": sorted(includes)}
  
      @staticmethod
cda1cd62   tangwang   意图分析&应用 baseline
347
348
349
      def _has_style_intent(parsed_query: Optional[ParsedQuery]) -> bool:
          profile = getattr(parsed_query, "style_intent_profile", None)
          return bool(getattr(profile, "is_active", False))
deccd68a   tangwang   Added the SKU pre...
350
  
cda1cd62   tangwang   意图分析&应用 baseline
351
      def _apply_style_intent_to_hits(
deccd68a   tangwang   Added the SKU pre...
352
353
354
355
          self,
          es_hits: List[Dict[str, Any]],
          parsed_query: ParsedQuery,
          context: Optional[RequestContext] = None,
cda1cd62   tangwang   意图分析&应用 baseline
356
      ) -> Dict[str, SkuSelectionDecision]:
8ae95af0   tangwang   1. Stage Timings:...
357
358
359
          if context is not None:
              context.start_stage(RequestContextStage.STYLE_SKU_PREPARE_HITS)
          try:
814e352b   tangwang   乘法公式配置化
360
              return self.style_sku_selector.prepare_hits(es_hits, parsed_query)
8ae95af0   tangwang   1. Stage Timings:...
361
362
363
          finally:
              if context is not None:
                  context.end_stage(RequestContextStage.STYLE_SKU_PREPARE_HITS)
deccd68a   tangwang   Added the SKU pre...
364
  
be52af70   tangwang   first commit
365
366
367
      def search(
          self,
          query: str,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
368
          tenant_id: str,
be52af70   tangwang   first commit
369
370
371
          size: int = 10,
          from_: int = 0,
          filters: Optional[Dict[str, Any]] = None,
6aa246be   tangwang   问题:Pydantic 应该能自动...
372
          range_filters: Optional[Dict[str, Any]] = None,
13320ac6   tangwang   分面接口修改:
373
          facets: Optional[List[FacetConfig]] = None,
16c42787   tangwang   feat: implement r...
374
          min_score: Optional[float] = None,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
375
          context: Optional[RequestContext] = None,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
376
          sort_by: Optional[str] = None,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
377
          sort_order: Optional[str] = "desc",
577ec972   tangwang   返回给前端的字段、格式适配。主要包...
378
          debug: bool = False,
2739b281   tangwang   多语言索引调整
379
          language: str = "en",
a3a5d41b   tangwang   (sku_filter_dimen...
380
          sku_filter_dimension: Optional[List[str]] = None,
5f7d7f09   tangwang   性能测试报告.md
381
          enable_rerank: Optional[bool] = None,
ff32d894   tangwang   rerank
382
383
          rerank_query_template: Optional[str] = None,
          rerank_doc_template: Optional[str] = None,
be52af70   tangwang   first commit
384
385
      ) -> SearchResult:
          """
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
386
          Execute search query (外部友好格式).
be52af70   tangwang   first commit
387
388
389
  
          Args:
              query: Search query string
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
390
              tenant_id: Tenant ID (required for filtering)
be52af70   tangwang   first commit
391
392
              size: Number of results to return
              from_: Offset for pagination
6aa246be   tangwang   问题:Pydantic 应该能自动...
393
394
395
              filters: Exact match filters
              range_filters: Range filters for numeric fields
              facets: Facet configurations for faceted search
be52af70   tangwang   first commit
396
              min_score: Minimum score threshold
ef5baa86   tangwang   混杂语言处理
397
              context: Request context for tracking (required)
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
398
399
              sort_by: Field name for sorting
              sort_order: Sort order: 'asc' or 'desc'
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
400
              debug: Enable debug information output
ef5baa86   tangwang   混杂语言处理
401
402
403
404
405
406
407
408
409
              language: Response / field selection language hint (e.g. zh, en)
              sku_filter_dimension: SKU grouping dimensions for per-SPU variant pick
              enable_rerank: If None, use ``config.rerank.enabled``; if set, overrides
                  whether the rerank provider is invoked (subject to rerank window).
              rerank_query_template: Override for rerank query text template; None uses
                  ``config.rerank.rerank_query_template`` (e.g. ``"{query}"``).
              rerank_doc_template: Override for per-hit document text passed to rerank;
                  None uses ``config.rerank.rerank_doc_template``. Placeholders are
                  resolved in ``search/rerank_client.py``.
be52af70   tangwang   first commit
410
411
  
          Returns:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
412
              SearchResult object with formatted results
be52af70   tangwang   first commit
413
          """
16c42787   tangwang   feat: implement r...
414
          if context is None:
ed948666   tangwang   tidy
415
              raise ValueError("context is required")
16c42787   tangwang   feat: implement r...
416
  
345d960b   tangwang   1. 删除全局 enable_tr...
417
418
419
          # 根据租户配置决定翻译开关(离线/在线统一)
          tenant_loader = get_tenant_config_loader()
          tenant_cfg = tenant_loader.get_tenant_config(tenant_id)
038e4e2f   tangwang   refactor(i18n): t...
420
421
          index_langs = tenant_cfg.get("index_languages") or []
          enable_translation = len(index_langs) > 0
9f96d6f3   tangwang   短query不用语义搜索
422
          enable_embedding = self.config.query_config.enable_text_embedding
8c8b9d84   tangwang   ES 拉取 coarse_rank...
423
424
          coarse_cfg = self.config.coarse_rank
          fine_cfg = self.config.fine_rank
5f7d7f09   tangwang   性能测试报告.md
425
426
427
          rc = self.config.rerank
          effective_query_template = rerank_query_template or rc.rerank_query_template
          effective_doc_template = rerank_doc_template or rc.rerank_doc_template
8c8b9d84   tangwang   ES 拉取 coarse_rank...
428
429
          fine_query_template = fine_cfg.rerank_query_template or effective_query_template
          fine_doc_template = fine_cfg.rerank_doc_template or effective_doc_template
5f7d7f09   tangwang   性能测试报告.md
430
431
432
          # 重排开关优先级:请求参数显式传值 > 服务端配置(默认开启)
          rerank_enabled_by_config = bool(rc.enabled)
          do_rerank = rerank_enabled_by_config if enable_rerank is None else bool(enable_rerank)
c51d254f   tangwang   性能测试
433
          rerank_window = rc.rerank_window
8c8b9d84   tangwang   ES 拉取 coarse_rank...
434
435
436
437
          coarse_input_window = max(rerank_window, int(coarse_cfg.input_window))
          coarse_output_window = max(rerank_window, int(coarse_cfg.output_window))
          fine_input_window = max(rerank_window, int(fine_cfg.input_window))
          fine_output_window = max(rerank_window, int(fine_cfg.output_window))
506c39b7   tangwang   feat(search): 统一重...
438
          # 若开启重排且请求范围在窗口内:从 ES 取前 rerank_window 条、重排后再按 from/size 分页;否则不重排,按原 from/size 查 ES
ff32d894   tangwang   rerank
439
          in_rerank_window = do_rerank and (from_ + size) <= rerank_window
506c39b7   tangwang   feat(search): 统一重...
440
          es_fetch_from = 0 if in_rerank_window else from_
8c8b9d84   tangwang   ES 拉取 coarse_rank...
441
          es_fetch_size = coarse_input_window if in_rerank_window else size
814e352b   tangwang   乘法公式配置化
442
443
444
  
          es_score_normalization_factor: Optional[float] = None
          initial_ranks_by_doc: Dict[str, int] = {}
daa2690b   tangwang   漏斗参数调优&呈现优化
445
446
447
          coarse_ranks_by_doc: Dict[str, int] = {}
          fine_ranks_by_doc: Dict[str, int] = {}
          rerank_ranks_by_doc: Dict[str, int] = {}
8c8b9d84   tangwang   ES 拉取 coarse_rank...
448
449
          coarse_debug_info: Optional[Dict[str, Any]] = None
          fine_debug_info: Optional[Dict[str, Any]] = None
581dafae   tangwang   debug工具,每条结果的打分中间...
450
          rerank_debug_info: Optional[Dict[str, Any]] = None
16c42787   tangwang   feat: implement r...
451
452
453
454
455
456
  
          # Start timing
          context.start_stage(RequestContextStage.TOTAL)
  
          context.logger.info(
              f"开始搜索请求 | 查询: '{query}' | 参数: size={size}, from_={from_}, "
5f7d7f09   tangwang   性能测试报告.md
457
458
459
              f"enable_rerank(request)={enable_rerank}, enable_rerank(config)={rerank_enabled_by_config}, "
              f"enable_rerank(effective)={do_rerank}, in_rerank_window={in_rerank_window}, "
              f"es_fetch=({es_fetch_from},{es_fetch_size}) | "
581dafae   tangwang   debug工具,每条结果的打分中间...
460
              f"index_languages={index_langs} | "
506c39b7   tangwang   feat(search): 统一重...
461
              f"enable_translation={enable_translation}, enable_embedding={enable_embedding}, min_score={min_score}",
16c42787   tangwang   feat: implement r...
462
463
464
465
466
467
468
              extra={'reqid': context.reqid, 'uid': context.uid}
          )
  
          # Store search parameters in context
          context.metadata['search_params'] = {
              'size': size,
              'from_': from_,
506c39b7   tangwang   feat(search): 统一重...
469
470
471
              'es_fetch_from': es_fetch_from,
              'es_fetch_size': es_fetch_size,
              'in_rerank_window': in_rerank_window,
5f7d7f09   tangwang   性能测试报告.md
472
473
474
475
              'rerank_enabled_by_config': rerank_enabled_by_config,
              'enable_rerank_request': enable_rerank,
              'rerank_query_template': effective_query_template,
              'rerank_doc_template': effective_doc_template,
8c8b9d84   tangwang   ES 拉取 coarse_rank...
476
477
              'fine_query_template': fine_query_template,
              'fine_doc_template': fine_doc_template,
16c42787   tangwang   feat: implement r...
478
              'filters': filters,
6aa246be   tangwang   问题:Pydantic 应该能自动...
479
480
              'range_filters': range_filters,
              'facets': facets,
16c42787   tangwang   feat: implement r...
481
482
              'enable_translation': enable_translation,
              'enable_embedding': enable_embedding,
ff32d894   tangwang   rerank
483
              'enable_rerank': do_rerank,
8c8b9d84   tangwang   ES 拉取 coarse_rank...
484
485
486
487
488
              'coarse_input_window': coarse_input_window,
              'coarse_output_window': coarse_output_window,
              'fine_input_window': fine_input_window,
              'fine_output_window': fine_output_window,
              'rerank_window': rerank_window,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
489
              'min_score': min_score,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
490
491
              'sort_by': sort_by,
              'sort_order': sort_order
16c42787   tangwang   feat: implement r...
492
          }
be52af70   tangwang   first commit
493
  
16c42787   tangwang   feat: implement r...
494
495
496
          context.metadata['feature_flags'] = {
              'translation_enabled': enable_translation,
              'embedding_enabled': enable_embedding,
cda1cd62   tangwang   意图分析&应用 baseline
497
498
              'rerank_enabled': do_rerank,
              'style_intent_enabled': bool(self.style_intent_registry.enabled),
16c42787   tangwang   feat: implement r...
499
          }
be52af70   tangwang   first commit
500
501
  
          # Step 1: Parse query
16c42787   tangwang   feat: implement r...
502
503
504
505
          context.start_stage(RequestContextStage.QUERY_PARSING)
          try:
              parsed_query = self.query_parser.parse(
                  query,
16c42787   tangwang   feat: implement r...
506
                  generate_vector=enable_embedding,
814e352b   tangwang   乘法公式配置化
507
                  tenant_id=tenant_id,
ef5baa86   tangwang   混杂语言处理
508
509
                  context=context,
                  target_languages=index_langs if enable_translation else [],
16c42787   tangwang   feat: implement r...
510
511
512
513
              )
              # Store query analysis results in context
              context.store_query_analysis(
                  original_query=parsed_query.original_query,
3a5fda00   tangwang   1. ES字段 skus的 ima...
514
                  query_normalized=parsed_query.query_normalized,
16c42787   tangwang   feat: implement r...
515
516
517
                  rewritten_query=parsed_query.rewritten_query,
                  detected_language=parsed_query.detected_language,
                  translations=parsed_query.translations,
9d0214bb   tangwang   qp性能优化
518
                  keywords_queries=parsed_query.keywords_queries,
16c42787   tangwang   feat: implement r...
519
                  query_vector=parsed_query.query_vector.tolist() if parsed_query.query_vector is not None else None,
16c42787   tangwang   feat: implement r...
520
              )
cda1cd62   tangwang   意图分析&应用 baseline
521
              context.metadata["feature_flags"]["style_intent_active"] = self._has_style_intent(parsed_query)
be52af70   tangwang   first commit
522
  
16c42787   tangwang   feat: implement r...
523
524
525
526
              context.logger.info(
                  f"查询解析完成 | 原查询: '{parsed_query.original_query}' | "
                  f"重写后: '{parsed_query.rewritten_query}' | "
                  f"语言: {parsed_query.detected_language} | "
9d0214bb   tangwang   qp性能优化
527
                  f"关键词: {parsed_query.keywords_queries} | "
dc403578   tangwang   多模态搜索
528
529
                  f"文本向量: {'是' if parsed_query.query_vector is not None else '否'} | "
                  f"图片向量: {'是' if getattr(parsed_query, 'image_query_vector', None) is not None else '否'}",
16c42787   tangwang   feat: implement r...
530
531
532
533
534
535
536
537
538
539
540
541
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"查询解析失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
              context.end_stage(RequestContextStage.QUERY_PARSING)
  
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
542
          # Step 2: Query building
16c42787   tangwang   feat: implement r...
543
544
          context.start_stage(RequestContextStage.QUERY_BUILDING)
          try:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
545
546
              # Generate tenant-specific index name
              index_name = get_tenant_index_name(tenant_id)
2739b281   tangwang   多语言索引调整
547
              # index_name = "search_products"
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
548
549
              
              # No longer need to add tenant_id to filters since each tenant has its own index
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
550
  
f0d020c3   tangwang   多语言查询改为只支持中英文两种,f...
551
              es_query = self.query_builder.build_query(
3a5fda00   tangwang   1. ES字段 skus的 ima...
552
                  query_text=parsed_query.rewritten_query or parsed_query.query_normalized,
16c42787   tangwang   feat: implement r...
553
                  query_vector=parsed_query.query_vector if enable_embedding else None,
dc403578   tangwang   多模态搜索
554
555
556
557
558
                  image_query_vector=(
                      getattr(parsed_query, "image_query_vector", None)
                      if enable_embedding
                      else None
                  ),
16c42787   tangwang   feat: implement r...
559
                  filters=filters,
6aa246be   tangwang   问题:Pydantic 应该能自动...
560
                  range_filters=range_filters,
c581becd   tangwang   feat: 实现 Multi-Se...
561
                  facet_configs=facets,
506c39b7   tangwang   feat(search): 统一重...
562
563
                  size=es_fetch_size,
                  from_=es_fetch_from,
dc403578   tangwang   多模态搜索
564
565
566
567
                  enable_knn=enable_embedding and (
                      parsed_query.query_vector is not None
                      or getattr(parsed_query, "image_query_vector", None) is not None
                  ),
7bc756c5   tangwang   优化 ES 查询构建
568
                  min_score=min_score,
ef5baa86   tangwang   混杂语言处理
569
                  parsed_query=parsed_query,
16c42787   tangwang   feat: implement r...
570
              )
be52af70   tangwang   first commit
571
  
6aa246be   tangwang   问题:Pydantic 应该能自动...
572
573
574
575
576
577
578
              # Add facets for faceted search
              if facets:
                  facet_aggs = self.query_builder.build_facets(facets)
                  if facet_aggs:
                      if "aggs" not in es_query:
                          es_query["aggs"] = {}
                      es_query["aggs"].update(facet_aggs)
16c42787   tangwang   feat: implement r...
579
  
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
580
581
582
              # Add sorting if specified
              if sort_by:
                  es_query = self.query_builder.add_sorting(es_query, sort_by, sort_order)
a47416ec   tangwang   把融合逻辑改成乘法公式,并把 ES...
583
                  es_query["track_scores"] = True
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
584
  
5f7d7f09   tangwang   性能测试报告.md
585
586
587
              # Keep requested response _source semantics for the final response fill.
              response_source_spec = es_query.get("_source")
  
8c8b9d84   tangwang   ES 拉取 coarse_rank...
588
              # In multi-stage rank window, first pass only needs score signals for coarse rank.
5f7d7f09   tangwang   性能测试报告.md
589
590
591
              es_query_for_fetch = es_query
              rerank_prefetch_source = None
              if in_rerank_window:
5f7d7f09   tangwang   性能测试报告.md
592
                  es_query_for_fetch = dict(es_query)
8c8b9d84   tangwang   ES 拉取 coarse_rank...
593
                  es_query_for_fetch["_source"] = False
5f7d7f09   tangwang   性能测试报告.md
594
  
16c42787   tangwang   feat: implement r...
595
              # Extract size and from from body for ES client parameters
5f7d7f09   tangwang   性能测试报告.md
596
              body_for_es = {k: v for k, v in es_query_for_fetch.items() if k not in ['size', 'from']}
16c42787   tangwang   feat: implement r...
597
598
599
  
              # Store ES query in context
              context.store_intermediate_result('es_query', es_query)
5f7d7f09   tangwang   性能测试报告.md
600
601
              if in_rerank_window and rerank_prefetch_source is not None:
                  context.store_intermediate_result('es_query_rerank_prefetch_source', rerank_prefetch_source)
28e57bb1   tangwang   日志体系优化
602
              # Serialize ES query to compute a compact size + stable digest for correlation
5f7d7f09   tangwang   性能测试报告.md
603
              es_query_compact = json.dumps(es_query_for_fetch, ensure_ascii=False, separators=(",", ":"))
28e57bb1   tangwang   日志体系优化
604
              es_query_digest = hashlib.sha256(es_query_compact.encode("utf-8")).hexdigest()[:16]
dc403578   tangwang   多模态搜索
605
606
607
608
              knn_enabled = bool(enable_embedding and (
                  parsed_query.query_vector is not None
                  or getattr(parsed_query, "image_query_vector", None) is not None
              ))
28e57bb1   tangwang   日志体系优化
609
              vector_dims = int(len(parsed_query.query_vector)) if parsed_query.query_vector is not None else 0
dc403578   tangwang   多模态搜索
610
611
612
613
614
              image_vector_dims = (
                  int(len(parsed_query.image_query_vector))
                  if getattr(parsed_query, "image_query_vector", None) is not None
                  else 0
              )
99bea633   tangwang   add logs
615
  
16c42787   tangwang   feat: implement r...
616
              context.logger.info(
dc403578   tangwang   多模态搜索
617
                  "ES query built | size: %s chars | digest: %s | KNN: %s | vector_dims: %s | image_vector_dims: %s | facets: %s | rerank_prefetch_source: %s",
28e57bb1   tangwang   日志体系优化
618
619
620
621
                  len(es_query_compact),
                  es_query_digest,
                  "yes" if knn_enabled else "no",
                  vector_dims,
dc403578   tangwang   多模态搜索
622
                  image_vector_dims,
28e57bb1   tangwang   日志体系优化
623
                  "yes" if facets else "no",
5f7d7f09   tangwang   性能测试报告.md
624
                  rerank_prefetch_source,
16c42787   tangwang   feat: implement r...
625
626
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
28e57bb1   tangwang   日志体系优化
627
628
629
630
631
632
633
634
635
              _log_backend_verbose({
                  "event": "es_query_built",
                  "reqid": context.reqid,
                  "uid": context.uid,
                  "tenant_id": tenant_id,
                  "size_chars": len(es_query_compact),
                  "sha256_16": es_query_digest,
                  "knn_enabled": knn_enabled,
                  "vector_dims": vector_dims,
dc403578   tangwang   多模态搜索
636
                  "image_vector_dims": image_vector_dims,
28e57bb1   tangwang   日志体系优化
637
                  "has_facets": bool(facets),
5f7d7f09   tangwang   性能测试报告.md
638
                  "query": es_query_for_fetch,
28e57bb1   tangwang   日志体系优化
639
              })
16c42787   tangwang   feat: implement r...
640
641
642
643
644
645
646
647
648
649
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"ES查询构建失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
              context.end_stage(RequestContextStage.QUERY_BUILDING)
  
a99e62ba   tangwang   记录各阶段耗时
650
651
          # Step 4: Elasticsearch search (primary recall)
          context.start_stage(RequestContextStage.ELASTICSEARCH_SEARCH_PRIMARY)
16c42787   tangwang   feat: implement r...
652
          try:
506c39b7   tangwang   feat(search): 统一重...
653
              # Use tenant-specific index name(开启重排且在窗口内时已用 es_fetch_size/es_fetch_from)
16c42787   tangwang   feat: implement r...
654
              es_response = self.es_client.search(
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
655
                  index_name=index_name,
16c42787   tangwang   feat: implement r...
656
                  body=body_for_es,
506c39b7   tangwang   feat(search): 统一重...
657
                  size=es_fetch_size,
a47416ec   tangwang   把融合逻辑改成乘法公式,并把 ES...
658
659
                  from_=es_fetch_from,
                  include_named_queries_score=bool(do_rerank and in_rerank_window),
be52af70   tangwang   first commit
660
661
              )
  
16c42787   tangwang   feat: implement r...
662
663
              # Store ES response in context
              context.store_intermediate_result('es_response', es_response)
581dafae   tangwang   debug工具,每条结果的打分中间...
664
              if debug:
814e352b   tangwang   乘法公式配置化
665
                  initial_hits = es_response.get("hits", {}).get("hits") or []
581dafae   tangwang   debug工具,每条结果的打分中间...
666
667
668
                  for rank, hit in enumerate(initial_hits, 1):
                      doc_id = hit.get("_id")
                      if doc_id is not None:
814e352b   tangwang   乘法公式配置化
669
670
                          initial_ranks_by_doc[str(doc_id)] = rank
                  raw_initial_max_score = es_response.get("hits", {}).get("max_score")
581dafae   tangwang   debug工具,每条结果的打分中间...
671
                  try:
814e352b   tangwang   乘法公式配置化
672
                      es_score_normalization_factor = float(raw_initial_max_score) if raw_initial_max_score is not None else None
581dafae   tangwang   debug工具,每条结果的打分中间...
673
                  except (TypeError, ValueError):
814e352b   tangwang   乘法公式配置化
674
675
676
677
678
679
680
                      es_score_normalization_factor = None
                  if es_score_normalization_factor is None and initial_hits:
                      first_score = initial_hits[0].get("_score")
                      try:
                          es_score_normalization_factor = float(first_score) if first_score is not None else None
                      except (TypeError, ValueError):
                          es_score_normalization_factor = None
be52af70   tangwang   first commit
681
  
16c42787   tangwang   feat: implement r...
682
683
684
685
686
              # Extract timing from ES response
              es_took = es_response.get('took', 0)
              context.logger.info(
                  f"ES搜索完成 | 耗时: {es_took}ms | "
                  f"命中数: {es_response.get('hits', {}).get('total', {}).get('value', 0)} | "
814e352b   tangwang   乘法公式配置化
687
                  f"最高分: {(es_response.get('hits', {}).get('max_score') or 0):.3f}",
16c42787   tangwang   feat: implement r...
688
689
690
691
692
693
694
695
696
697
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"ES搜索执行失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
a99e62ba   tangwang   记录各阶段耗时
698
              context.end_stage(RequestContextStage.ELASTICSEARCH_SEARCH_PRIMARY)
16c42787   tangwang   feat: implement r...
699
  
cda1cd62   tangwang   意图分析&应用 baseline
700
          style_intent_decisions: Dict[str, SkuSelectionDecision] = {}
8c8b9d84   tangwang   ES 拉取 coarse_rank...
701
702
          if do_rerank and in_rerank_window:
              from dataclasses import asdict
daa2690b   tangwang   漏斗参数调优&呈现优化
703
              from config.services_config import get_rerank_backend_config, get_rerank_service_url
8c8b9d84   tangwang   ES 拉取 coarse_rank...
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
              from .rerank_client import coarse_resort_hits, run_lightweight_rerank, run_rerank
  
              rerank_query = parsed_query.text_for_rerank() if parsed_query else query
              hits = es_response.get("hits", {}).get("hits") or []
  
              context.start_stage(RequestContextStage.COARSE_RANKING)
              try:
                  coarse_debug = coarse_resort_hits(
                      hits,
                      fusion=coarse_cfg.fusion,
                      debug=debug,
                  )
                  hits = hits[:coarse_output_window]
                  es_response.setdefault("hits", {})["hits"] = hits
                  if debug:
daa2690b   tangwang   漏斗参数调优&呈现优化
719
720
721
722
                      coarse_ranks_by_doc = {
                          str(hit.get("_id")): rank
                          for rank, hit in enumerate(hits, 1)
                          if hit.get("_id") is not None
8c8b9d84   tangwang   ES 拉取 coarse_rank...
723
                      }
daa2690b   tangwang   漏斗参数调优&呈现优化
724
725
726
727
728
729
                      if debug:
                          coarse_debug_info = {
                              "docs_in": es_fetch_size,
                              "docs_out": len(hits),
                              "fusion": asdict(coarse_cfg.fusion),
                          }
8c8b9d84   tangwang   ES 拉取 coarse_rank...
730
                      context.store_intermediate_result("coarse_rank_scores", coarse_debug)
cda1cd62   tangwang   意图分析&应用 baseline
731
                  context.logger.info(
8c8b9d84   tangwang   ES 拉取 coarse_rank...
732
733
734
                      "粗排完成 | docs_in=%s | docs_out=%s",
                      es_fetch_size,
                      len(hits),
cda1cd62   tangwang   意图分析&应用 baseline
735
736
                      extra={'reqid': context.reqid, 'uid': context.uid}
                  )
8c8b9d84   tangwang   ES 拉取 coarse_rank...
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
              finally:
                  context.end_stage(RequestContextStage.COARSE_RANKING)
  
              ranking_source_spec = self._merge_source_specs(
                  self._resolve_rerank_source_filter(
                      fine_doc_template,
                      parsed_query=parsed_query,
                  ),
                  self._resolve_rerank_source_filter(
                      effective_doc_template,
                      parsed_query=parsed_query,
                  ),
              )
              candidate_ids = [str(h.get("_id")) for h in hits if h.get("_id") is not None]
              if candidate_ids:
                  details_by_id, fill_took = self._fetch_hits_by_ids(
                      index_name=index_name,
                      doc_ids=candidate_ids,
                      source_spec=ranking_source_spec,
                  )
                  for hit in hits:
                      hid = hit.get("_id")
                      if hid is None:
                          continue
                      detail_hit = details_by_id.get(str(hid))
                      if detail_hit is not None and "_source" in detail_hit:
                          hit["_source"] = detail_hit.get("_source") or {}
                  if fill_took:
                      es_response["took"] = int((es_response.get("took", 0) or 0) + fill_took)
  
              if self._has_style_intent(parsed_query):
                  style_intent_decisions = self._apply_style_intent_to_hits(
                      es_response.get("hits", {}).get("hits") or [],
                      parsed_query,
                      context=context,
                  )
                  if style_intent_decisions:
                      context.logger.info(
                          "款式意图 SKU 预筛选完成 | hits=%s",
                          len(style_intent_decisions),
                          extra={'reqid': context.reqid, 'uid': context.uid}
                      )
  
              fine_scores: Optional[List[float]] = None
              hits = es_response.get("hits", {}).get("hits") or []
              if fine_cfg.enabled and hits:
                  context.start_stage(RequestContextStage.FINE_RANKING)
                  try:
                      fine_scores, fine_meta, fine_debug_rows = run_lightweight_rerank(
                          query=rerank_query,
                          es_hits=hits[:fine_input_window],
                          language=language,
                          timeout_sec=fine_cfg.timeout_sec,
                          rerank_query_template=fine_query_template,
                          rerank_doc_template=fine_doc_template,
                          top_n=fine_output_window,
                          debug=debug,
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
794
795
                          fusion=rc.fusion,
                          style_intent_selected_sku_boost=self.config.query_config.style_intent_selected_sku_boost,
8c8b9d84   tangwang   ES 拉取 coarse_rank...
796
797
798
799
800
801
                          service_profile=fine_cfg.service_profile,
                      )
                      if fine_scores is not None:
                          hits = hits[:fine_output_window]
                          es_response["hits"]["hits"] = hits
                          if debug:
daa2690b   tangwang   漏斗参数调优&呈现优化
802
803
804
805
806
807
                              fine_ranks_by_doc = {
                                  str(hit.get("_id")): rank
                                  for rank, hit in enumerate(hits, 1)
                                  if hit.get("_id") is not None
                              }
                              fine_backend_name, fine_backend_cfg = get_rerank_backend_config(fine_cfg.service_profile)
8c8b9d84   tangwang   ES 拉取 coarse_rank...
808
                              fine_debug_info = {
daa2690b   tangwang   漏斗参数调优&呈现优化
809
                                  "service_profile": fine_cfg.service_profile,
8c8b9d84   tangwang   ES 拉取 coarse_rank...
810
                                  "service_url": get_rerank_service_url(profile=fine_cfg.service_profile),
daa2690b   tangwang   漏斗参数调优&呈现优化
811
812
813
                                  "backend": fine_backend_name,
                                  "model": fine_meta.get("model") if isinstance(fine_meta, dict) else None,
                                  "backend_model_name": fine_backend_cfg.get("model_name"),
8c8b9d84   tangwang   ES 拉取 coarse_rank...
814
815
816
                                  "query_template": fine_query_template,
                                  "doc_template": fine_doc_template,
                                  "query_text": str(fine_query_template).format_map({"query": rerank_query}),
daa2690b   tangwang   漏斗参数调优&呈现优化
817
818
                                  "docs_in": min(len(fine_scores), fine_input_window),
                                  "docs_out": len(hits),
8c8b9d84   tangwang   ES 拉取 coarse_rank...
819
820
                                  "top_n": fine_output_window,
                                  "meta": fine_meta,
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
821
                                  "fusion": asdict(rc.fusion),
8c8b9d84   tangwang   ES 拉取 coarse_rank...
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
                              }
                              context.store_intermediate_result("fine_rank_scores", fine_debug_rows)
                          context.logger.info(
                              "精排完成 | docs=%s | top_n=%s | meta=%s",
                              len(hits),
                              fine_output_window,
                              fine_meta,
                              extra={'reqid': context.reqid, 'uid': context.uid}
                          )
                  except Exception as e:
                      context.add_warning(f"Fine rerank failed: {e}")
                      context.logger.warning(
                          f"调用精排服务失败 | error: {e}",
                          extra={'reqid': context.reqid, 'uid': context.uid},
                          exc_info=True,
                      )
                  finally:
                      context.end_stage(RequestContextStage.FINE_RANKING)
cda1cd62   tangwang   意图分析&应用 baseline
840
  
506c39b7   tangwang   feat(search): 统一重...
841
842
              context.start_stage(RequestContextStage.RERANKING)
              try:
8c8b9d84   tangwang   ES 拉取 coarse_rank...
843
844
845
                  final_hits = es_response.get("hits", {}).get("hits") or []
                  final_input = final_hits[:rerank_window]
                  es_response["hits"]["hits"] = final_input
506c39b7   tangwang   feat(search): 统一重...
846
847
848
849
                  es_response, rerank_meta, fused_debug = run_rerank(
                      query=rerank_query,
                      es_response=es_response,
                      language=language,
506c39b7   tangwang   feat(search): 统一重...
850
851
852
                      timeout_sec=rc.timeout_sec,
                      weight_es=rc.weight_es,
                      weight_ai=rc.weight_ai,
ff32d894   tangwang   rerank
853
854
                      rerank_query_template=effective_query_template,
                      rerank_doc_template=effective_doc_template,
d31c7f65   tangwang   补充云服务reranker
855
                      top_n=(from_ + size),
581dafae   tangwang   debug工具,每条结果的打分中间...
856
                      debug=debug,
814e352b   tangwang   乘法公式配置化
857
                      fusion=rc.fusion,
8c8b9d84   tangwang   ES 拉取 coarse_rank...
858
                      service_profile=rc.service_profile,
87cacb1b   tangwang   融合公式优化。加入意图匹配因子
859
                      style_intent_selected_sku_boost=self.config.query_config.style_intent_selected_sku_boost,
506c39b7   tangwang   feat(search): 统一重...
860
861
862
                  )
  
                  if rerank_meta is not None:
814e352b   tangwang   乘法公式配置化
863
                      if debug:
daa2690b   tangwang   漏斗参数调优&呈现优化
864
865
866
867
868
869
                          rerank_ranks_by_doc = {
                              str(hit.get("_id")): rank
                              for rank, hit in enumerate(es_response.get("hits", {}).get("hits") or [], 1)
                              if hit.get("_id") is not None
                          }
                          rerank_backend_name, rerank_backend_cfg = get_rerank_backend_config(rc.service_profile)
814e352b   tangwang   乘法公式配置化
870
                          rerank_debug_info = {
daa2690b   tangwang   漏斗参数调优&呈现优化
871
                              "service_profile": rc.service_profile,
8c8b9d84   tangwang   ES 拉取 coarse_rank...
872
                              "service_url": get_rerank_service_url(profile=rc.service_profile),
daa2690b   tangwang   漏斗参数调优&呈现优化
873
874
875
                              "backend": rerank_backend_name,
                              "model": rerank_meta.get("model") if isinstance(rerank_meta, dict) else None,
                              "backend_model_name": rerank_backend_cfg.get("model_name"),
814e352b   tangwang   乘法公式配置化
876
877
                              "query_template": effective_query_template,
                              "doc_template": effective_doc_template,
581dafae   tangwang   debug工具,每条结果的打分中间...
878
                              "query_text": str(effective_query_template).format_map({"query": rerank_query}),
daa2690b   tangwang   漏斗参数调优&呈现优化
879
880
                              "docs_in": len(final_input),
                              "docs_out": len(es_response.get("hits", {}).get("hits") or []),
814e352b   tangwang   乘法公式配置化
881
                              "top_n": from_ + size,
581dafae   tangwang   debug工具,每条结果的打分中间...
882
                              "meta": rerank_meta,
814e352b   tangwang   乘法公式配置化
883
884
                              "fusion": asdict(rc.fusion),
                          }
581dafae   tangwang   debug工具,每条结果的打分中间...
885
                          context.store_intermediate_result("rerank_scores", fused_debug)
506c39b7   tangwang   feat(search): 统一重...
886
                      context.logger.info(
581dafae   tangwang   debug工具,每条结果的打分中间...
887
                          f"重排完成 | docs={len(es_response.get('hits', {}).get('hits') or [])} | "
814e352b   tangwang   乘法公式配置化
888
                          f"top_n={from_ + size} | meta={rerank_meta}",
506c39b7   tangwang   feat(search): 统一重...
889
890
891
892
                          extra={'reqid': context.reqid, 'uid': context.uid}
                      )
              except Exception as e:
                  context.add_warning(f"Rerank failed: {e}")
506c39b7   tangwang   feat(search): 统一重...
893
894
895
896
897
898
899
900
                  context.logger.warning(
                      f"调用重排服务失败 | error: {e}",
                      extra={'reqid': context.reqid, 'uid': context.uid},
                      exc_info=True,
                  )
              finally:
                  context.end_stage(RequestContextStage.RERANKING)
  
8c8b9d84   tangwang   ES 拉取 coarse_rank...
901
          # 当本次请求在重排窗口内时:已按多阶段排序产出前 rerank_window 条,需按请求的 from/size 做分页切片
506c39b7   tangwang   feat(search): 统一重...
902
903
904
905
906
          if in_rerank_window:
              hits = es_response.get("hits", {}).get("hits") or []
              sliced = hits[from_ : from_ + size]
              es_response.setdefault("hits", {})["hits"] = sliced
              if sliced:
af827ce9   tangwang   rerank
907
                  slice_max = max(
8c8b9d84   tangwang   ES 拉取 coarse_rank...
908
909
910
911
                      (
                          h.get("_fused_score", h.get("_fine_score", h.get("_coarse_score", h.get("_score", 0.0))))
                          for h in sliced
                      ),
af827ce9   tangwang   rerank
912
913
                      default=0.0,
                  )
506c39b7   tangwang   feat(search): 统一重...
914
915
916
917
918
919
                  try:
                      es_response["hits"]["max_score"] = float(slice_max)
                  except (TypeError, ValueError):
                      es_response["hits"]["max_score"] = 0.0
              else:
                  es_response["hits"]["max_score"] = 0.0
5f7d7f09   tangwang   性能测试报告.md
920
  
5f7d7f09   tangwang   性能测试报告.md
921
922
923
924
925
926
927
928
929
              if sliced:
                  if response_source_spec is False:
                      for hit in sliced:
                          hit.pop("_source", None)
                      context.logger.info(
                          "分页详情回填跳过 | 原查询 _source=false",
                          extra={'reqid': context.reqid, 'uid': context.uid}
                      )
                  else:
a99e62ba   tangwang   记录各阶段耗时
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
                      context.start_stage(RequestContextStage.ELASTICSEARCH_PAGE_FILL)
                      try:
                          page_ids = [str(h.get("_id")) for h in sliced if h.get("_id") is not None]
                          details_by_id, fill_took = self._fetch_hits_by_ids(
                              index_name=index_name,
                              doc_ids=page_ids,
                              source_spec=response_source_spec,
                          )
                          filled = 0
                          for hit in sliced:
                              hid = hit.get("_id")
                              if hid is None:
                                  continue
                              detail_hit = details_by_id.get(str(hid))
                              if detail_hit is None:
                                  continue
                              if "_source" in detail_hit:
                                  hit["_source"] = detail_hit.get("_source") or {}
                                  filled += 1
cda1cd62   tangwang   意图分析&应用 baseline
949
950
951
952
953
                          if style_intent_decisions:
                              self.style_sku_selector.apply_precomputed_decisions(
                                  sliced,
                                  style_intent_decisions,
                              )
a99e62ba   tangwang   记录各阶段耗时
954
955
                          if fill_took:
                              es_response["took"] = int((es_response.get("took", 0) or 0) + fill_took)
a99e62ba   tangwang   记录各阶段耗时
956
957
958
959
960
961
                          context.logger.info(
                              f"分页详情回填 | ids={len(page_ids)} | filled={filled} | took={fill_took}ms",
                              extra={'reqid': context.reqid, 'uid': context.uid}
                          )
                      finally:
                          context.end_stage(RequestContextStage.ELASTICSEARCH_PAGE_FILL)
5f7d7f09   tangwang   性能测试报告.md
962
  
506c39b7   tangwang   feat(search): 统一重...
963
964
965
966
967
              context.logger.info(
                  f"重排分页切片 | from={from_}, size={size}, 返回={len(sliced)}条",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
  
8ae95af0   tangwang   1. Stage Timings:...
968
969
970
971
972
973
974
975
976
          # 非重排窗口:款式意图在 result_processing 之前执行,便于单独计时且与 ES 召回阶段衔接
          if self._has_style_intent(parsed_query) and not in_rerank_window:
              es_hits_pre = es_response.get("hits", {}).get("hits") or []
              style_intent_decisions = self._apply_style_intent_to_hits(
                  es_hits_pre,
                  parsed_query,
                  context=context,
              )
  
16c42787   tangwang   feat: implement r...
977
978
979
          # Step 5: Result processing
          context.start_stage(RequestContextStage.RESULT_PROCESSING)
          try:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
980
981
              # Extract ES hits
              es_hits = []
16c42787   tangwang   feat: implement r...
982
              if 'hits' in es_response and 'hits' in es_response['hits']:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
983
                  es_hits = es_response['hits']['hits']
16c42787   tangwang   feat: implement r...
984
985
986
987
988
989
              # Extract total and max_score
              total = es_response.get('hits', {}).get('total', {})
              if isinstance(total, dict):
                  total_value = total.get('value', 0)
              else:
                  total_value = total
506c39b7   tangwang   feat(search): 统一重...
990
              # max_score 会在启用 AI 搜索时被更新为融合分数的最大值
25d3e81d   tangwang   fix指定sort项时候的bug
991
              max_score = es_response.get('hits', {}).get('max_score') or 0.0
be52af70   tangwang   first commit
992
  
af827ce9   tangwang   rerank
993
994
995
996
997
998
999
1000
1001
1002
1003
              # 从上下文中取出重排调试信息(若有)
              rerank_debug_raw = context.get_intermediate_result('rerank_scores', None)
              rerank_debug_by_doc: Dict[str, Dict[str, Any]] = {}
              if isinstance(rerank_debug_raw, list):
                  for item in rerank_debug_raw:
                      if not isinstance(item, dict):
                          continue
                      doc_id = item.get("doc_id")
                      if doc_id is None:
                          continue
                      rerank_debug_by_doc[str(doc_id)] = item
16d28bf8   tangwang   漏斗信息呈现,便于调整参数
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
              coarse_debug_raw = context.get_intermediate_result('coarse_rank_scores', None)
              coarse_debug_by_doc: Dict[str, Dict[str, Any]] = {}
              if isinstance(coarse_debug_raw, list):
                  for item in coarse_debug_raw:
                      if not isinstance(item, dict):
                          continue
                      doc_id = item.get("doc_id")
                      if doc_id is None:
                          continue
                      coarse_debug_by_doc[str(doc_id)] = item
8c8b9d84   tangwang   ES 拉取 coarse_rank...
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
              fine_debug_raw = context.get_intermediate_result('fine_rank_scores', None)
              fine_debug_by_doc: Dict[str, Dict[str, Any]] = {}
              if isinstance(fine_debug_raw, list):
                  for item in fine_debug_raw:
                      if not isinstance(item, dict):
                          continue
                      doc_id = item.get("doc_id")
                      if doc_id is None:
                          continue
                      fine_debug_by_doc[str(doc_id)] = item
af827ce9   tangwang   rerank
1024
  
cda1cd62   tangwang   意图分析&应用 baseline
1025
              if self._has_style_intent(parsed_query):
2efad04b   tangwang   意图匹配的性能优化:
1026
                  if style_intent_decisions:
cda1cd62   tangwang   意图分析&应用 baseline
1027
1028
1029
1030
                      self.style_sku_selector.apply_precomputed_decisions(
                          es_hits,
                          style_intent_decisions,
                      )
deccd68a   tangwang   Added the SKU pre...
1031
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1032
              # Format results using ResultFormatter
577ec972   tangwang   返回给前端的字段、格式适配。主要包...
1033
1034
1035
              formatted_results = ResultFormatter.format_search_results(
                  es_hits,
                  max_score,
ca91352a   tangwang   更新文档
1036
1037
                  language=language,
                  sku_filter_dimension=sku_filter_dimension
577ec972   tangwang   返回给前端的字段、格式适配。主要包...
1038
              )
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1039
  
985752f5   tangwang   1. 前端调试功能
1040
1041
1042
              # Build per-result debug info (per SPU) when debug mode is enabled
              per_result_debug = []
              if debug and es_hits and formatted_results:
814e352b   tangwang   乘法公式配置化
1043
1044
                  final_ranks_by_doc = {
                      str(hit.get("_id")): from_ + rank
581dafae   tangwang   debug工具,每条结果的打分中间...
1045
1046
1047
                      for rank, hit in enumerate(es_hits, 1)
                      if hit.get("_id") is not None
                  }
985752f5   tangwang   1. 前端调试功能
1048
1049
                  for hit, spu in zip(es_hits, formatted_results):
                      source = hit.get("_source", {}) or {}
af827ce9   tangwang   rerank
1050
1051
1052
1053
                      doc_id = hit.get("_id")
                      rerank_debug = None
                      if doc_id is not None:
                          rerank_debug = rerank_debug_by_doc.get(str(doc_id))
16d28bf8   tangwang   漏斗信息呈现,便于调整参数
1054
1055
1056
                      coarse_debug = None
                      if doc_id is not None:
                          coarse_debug = coarse_debug_by_doc.get(str(doc_id))
8c8b9d84   tangwang   ES 拉取 coarse_rank...
1057
1058
1059
                      fine_debug = None
                      if doc_id is not None:
                          fine_debug = fine_debug_by_doc.get(str(doc_id))
cda1cd62   tangwang   意图分析&应用 baseline
1060
1061
1062
1063
1064
                      style_intent_debug = None
                      if doc_id is not None and style_intent_decisions:
                          decision = style_intent_decisions.get(str(doc_id))
                          if decision is not None:
                              style_intent_debug = decision.to_dict()
af827ce9   tangwang   rerank
1065
  
9df421ed   tangwang   基于eval框架开始调参
1066
                      raw_score = hit.get("_raw_es_score", hit.get("_original_score", hit.get("_score")))
985752f5   tangwang   1. 前端调试功能
1067
1068
1069
1070
1071
                      try:
                          es_score = float(raw_score) if raw_score is not None else 0.0
                      except (TypeError, ValueError):
                          es_score = 0.0
                      try:
581dafae   tangwang   debug工具,每条结果的打分中间...
1072
                          normalized = (
814e352b   tangwang   乘法公式配置化
1073
1074
                              float(es_score) / float(es_score_normalization_factor)
                              if es_score_normalization_factor else None
581dafae   tangwang   debug工具,每条结果的打分中间...
1075
                          )
985752f5   tangwang   1. 前端调试功能
1076
1077
                      except (TypeError, ValueError, ZeroDivisionError):
                          normalized = None
985752f5   tangwang   1. 前端调试功能
1078
1079
1080
1081
1082
  
                      title_multilingual = source.get("title") if isinstance(source.get("title"), dict) else None
                      brief_multilingual = source.get("brief") if isinstance(source.get("brief"), dict) else None
                      vendor_multilingual = source.get("vendor") if isinstance(source.get("vendor"), dict) else None
  
af827ce9   tangwang   rerank
1083
1084
1085
1086
                      debug_entry: Dict[str, Any] = {
                          "spu_id": spu.spu_id,
                          "es_score": es_score,
                          "es_score_normalized": normalized,
814e352b   tangwang   乘法公式配置化
1087
1088
                          "initial_rank": initial_ranks_by_doc.get(str(doc_id)) if doc_id is not None else None,
                          "final_rank": final_ranks_by_doc.get(str(doc_id)) if doc_id is not None else None,
af827ce9   tangwang   rerank
1089
1090
1091
1092
1093
                          "title_multilingual": title_multilingual,
                          "brief_multilingual": brief_multilingual,
                          "vendor_multilingual": vendor_multilingual,
                      }
  
16d28bf8   tangwang   漏斗信息呈现,便于调整参数
1094
1095
                      if coarse_debug:
                          debug_entry["coarse_score"] = coarse_debug.get("coarse_score")
9df421ed   tangwang   基于eval框架开始调参
1096
                          debug_entry["coarse_es_factor"] = coarse_debug.get("coarse_es_factor")
16d28bf8   tangwang   漏斗信息呈现,便于调整参数
1097
1098
1099
                          debug_entry["coarse_text_factor"] = coarse_debug.get("coarse_text_factor")
                          debug_entry["coarse_knn_factor"] = coarse_debug.get("coarse_knn_factor")
  
af827ce9   tangwang   rerank
1100
1101
1102
                      # 若存在重排调试信息,则补充 doc 级别的融合分数信息
                      if rerank_debug:
                          debug_entry["doc_id"] = rerank_debug.get("doc_id")
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
1103
                          debug_entry["score"] = rerank_debug.get("score")
af827ce9   tangwang   rerank
1104
                          debug_entry["rerank_score"] = rerank_debug.get("rerank_score")
8c8b9d84   tangwang   ES 拉取 coarse_rank...
1105
                          debug_entry["fine_score"] = rerank_debug.get("fine_score")
9df421ed   tangwang   基于eval框架开始调参
1106
                          debug_entry["es_score"] = rerank_debug.get("es_score", es_score)
a8261ece   tangwang   检索效果优化
1107
                          debug_entry["text_score"] = rerank_debug.get("text_score")
a8261ece   tangwang   检索效果优化
1108
                          debug_entry["knn_score"] = rerank_debug.get("knn_score")
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
1109
1110
1111
                          debug_entry["fusion_inputs"] = rerank_debug.get("fusion_inputs")
                          debug_entry["fusion_factors"] = rerank_debug.get("fusion_factors")
                          debug_entry["fusion_summary"] = rerank_debug.get("fusion_summary")
581dafae   tangwang   debug工具,每条结果的打分中间...
1112
                          debug_entry["rerank_factor"] = rerank_debug.get("rerank_factor")
8c8b9d84   tangwang   ES 拉取 coarse_rank...
1113
                          debug_entry["fine_factor"] = rerank_debug.get("fine_factor")
9df421ed   tangwang   基于eval框架开始调参
1114
                          debug_entry["es_factor"] = rerank_debug.get("es_factor")
581dafae   tangwang   debug工具,每条结果的打分中间...
1115
1116
                          debug_entry["text_factor"] = rerank_debug.get("text_factor")
                          debug_entry["knn_factor"] = rerank_debug.get("knn_factor")
af827ce9   tangwang   rerank
1117
                          debug_entry["fused_score"] = rerank_debug.get("fused_score")
581dafae   tangwang   debug工具,每条结果的打分中间...
1118
                          debug_entry["rerank_input"] = rerank_debug.get("rerank_input")
a8261ece   tangwang   检索效果优化
1119
                          debug_entry["matched_queries"] = rerank_debug.get("matched_queries")
465f90e1   tangwang   添加LTR数据收集
1120
                          debug_entry["ltr_features"] = rerank_debug.get("ltr_features")
8c8b9d84   tangwang   ES 拉取 coarse_rank...
1121
1122
                      elif fine_debug:
                          debug_entry["doc_id"] = fine_debug.get("doc_id")
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
1123
                          debug_entry["score"] = fine_debug.get("score")
8c8b9d84   tangwang   ES 拉取 coarse_rank...
1124
                          debug_entry["fine_score"] = fine_debug.get("fine_score")
9df421ed   tangwang   基于eval框架开始调参
1125
                          debug_entry["es_score"] = fine_debug.get("es_score", es_score)
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
1126
1127
1128
1129
1130
                          debug_entry["text_score"] = fine_debug.get("text_score")
                          debug_entry["knn_score"] = fine_debug.get("knn_score")
                          debug_entry["fusion_inputs"] = fine_debug.get("fusion_inputs")
                          debug_entry["fusion_factors"] = fine_debug.get("fusion_factors")
                          debug_entry["fusion_summary"] = fine_debug.get("fusion_summary")
9df421ed   tangwang   基于eval框架开始调参
1131
                          debug_entry["es_factor"] = fine_debug.get("es_factor")
8c8b9d84   tangwang   ES 拉取 coarse_rank...
1132
                          debug_entry["rerank_input"] = fine_debug.get("rerank_input")
465f90e1   tangwang   添加LTR数据收集
1133
                          debug_entry["ltr_features"] = fine_debug.get("ltr_features")
af827ce9   tangwang   rerank
1134
  
daa2690b   tangwang   漏斗参数调优&呈现优化
1135
1136
1137
1138
1139
                      initial_rank = initial_ranks_by_doc.get(str(doc_id)) if doc_id is not None else None
                      coarse_rank = coarse_ranks_by_doc.get(str(doc_id)) if doc_id is not None else None
                      fine_rank = fine_ranks_by_doc.get(str(doc_id)) if doc_id is not None else None
                      rerank_rank = rerank_ranks_by_doc.get(str(doc_id)) if doc_id is not None else None
                      final_rank = final_ranks_by_doc.get(str(doc_id)) if doc_id is not None else None
9df421ed   tangwang   基于eval框架开始调参
1140
1141
1142
1143
1144
1145
1146
1147
                      rerank_previous_rank = fine_rank if fine_rank is not None else coarse_rank
                      final_previous_rank = rerank_rank
                      if final_previous_rank is None:
                          final_previous_rank = fine_rank
                      if final_previous_rank is None:
                          final_previous_rank = coarse_rank
                      if final_previous_rank is None:
                          final_previous_rank = initial_rank
daa2690b   tangwang   漏斗参数调优&呈现优化
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
  
                      def _rank_change(previous_rank: Optional[int], current_rank: Optional[int]) -> Optional[int]:
                          if previous_rank is None or current_rank is None:
                              return None
                          return previous_rank - current_rank
  
                      debug_entry["ranking_funnel"] = {
                          "es_recall": {
                              "rank": initial_rank,
                              "score": es_score,
                              "normalized_score": normalized,
                              "matched_queries": hit.get("matched_queries"),
                          },
                          "coarse_rank": {
                              "rank": coarse_rank,
                              "rank_change": _rank_change(initial_rank, coarse_rank),
                              "score": coarse_debug.get("coarse_score") if coarse_debug else None,
9df421ed   tangwang   基于eval框架开始调参
1165
                              "es_score": coarse_debug.get("es_score") if coarse_debug else es_score,
daa2690b   tangwang   漏斗参数调优&呈现优化
1166
1167
                              "text_score": coarse_debug.get("text_score") if coarse_debug else None,
                              "knn_score": coarse_debug.get("knn_score") if coarse_debug else None,
9df421ed   tangwang   基于eval框架开始调参
1168
                              "es_factor": coarse_debug.get("coarse_es_factor") if coarse_debug else None,
daa2690b   tangwang   漏斗参数调优&呈现优化
1169
1170
1171
                              "text_factor": coarse_debug.get("coarse_text_factor") if coarse_debug else None,
                              "knn_factor": coarse_debug.get("coarse_knn_factor") if coarse_debug else None,
                              "signals": coarse_debug,
465f90e1   tangwang   添加LTR数据收集
1172
                              "ltr_features": coarse_debug.get("ltr_features") if coarse_debug else None,
daa2690b   tangwang   漏斗参数调优&呈现优化
1173
1174
1175
1176
                          },
                          "fine_rank": {
                              "rank": fine_rank,
                              "rank_change": _rank_change(coarse_rank, fine_rank),
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
1177
1178
1179
1180
1181
1182
                              "score": (
                                  fine_debug.get("score")
                                  if fine_debug and fine_debug.get("score") is not None
                                  else hit.get("_fine_fused_score", hit.get("_fine_score"))
                              ),
                              "fine_score": fine_debug.get("fine_score") if fine_debug else hit.get("_fine_score"),
9df421ed   tangwang   基于eval框架开始调参
1183
                              "es_score": fine_debug.get("es_score") if fine_debug else es_score,
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
1184
1185
                              "text_score": fine_debug.get("text_score") if fine_debug else hit.get("_text_score"),
                              "knn_score": fine_debug.get("knn_score") if fine_debug else hit.get("_knn_score"),
9df421ed   tangwang   基于eval框架开始调参
1186
                              "es_factor": fine_debug.get("es_factor") if fine_debug else None,
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
1187
1188
1189
                              "fusion_summary": fine_debug.get("fusion_summary") if fine_debug else None,
                              "fusion_inputs": fine_debug.get("fusion_inputs") if fine_debug else None,
                              "fusion_factors": fine_debug.get("fusion_factors") if fine_debug else None,
daa2690b   tangwang   漏斗参数调优&呈现优化
1190
                              "rerank_input": fine_debug.get("rerank_input") if fine_debug else None,
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
1191
                              "signals": fine_debug,
465f90e1   tangwang   添加LTR数据收集
1192
                              "ltr_features": fine_debug.get("ltr_features") if fine_debug else None,
daa2690b   tangwang   漏斗参数调优&呈现优化
1193
1194
1195
                          },
                          "rerank": {
                              "rank": rerank_rank,
9df421ed   tangwang   基于eval框架开始调参
1196
                              "rank_change": _rank_change(rerank_previous_rank, rerank_rank),
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
1197
                              "score": rerank_debug.get("score") if rerank_debug else hit.get("_fused_score"),
9df421ed   tangwang   基于eval框架开始调参
1198
                              "es_score": rerank_debug.get("es_score") if rerank_debug else es_score,
daa2690b   tangwang   漏斗参数调优&呈现优化
1199
1200
1201
1202
1203
                              "rerank_score": rerank_debug.get("rerank_score") if rerank_debug else hit.get("_rerank_score"),
                              "fine_score": rerank_debug.get("fine_score") if rerank_debug else hit.get("_fine_score"),
                              "fused_score": rerank_debug.get("fused_score") if rerank_debug else hit.get("_fused_score"),
                              "text_score": rerank_debug.get("text_score") if rerank_debug else hit.get("_text_score"),
                              "knn_score": rerank_debug.get("knn_score") if rerank_debug else hit.get("_knn_score"),
c3425429   tangwang   在以下文件中完成精排/融合清理工作...
1204
1205
1206
                              "fusion_summary": rerank_debug.get("fusion_summary") if rerank_debug else None,
                              "fusion_inputs": rerank_debug.get("fusion_inputs") if rerank_debug else None,
                              "fusion_factors": rerank_debug.get("fusion_factors") if rerank_debug else None,
daa2690b   tangwang   漏斗参数调优&呈现优化
1207
1208
                              "rerank_factor": rerank_debug.get("rerank_factor") if rerank_debug else None,
                              "fine_factor": rerank_debug.get("fine_factor") if rerank_debug else None,
9df421ed   tangwang   基于eval框架开始调参
1209
                              "es_factor": rerank_debug.get("es_factor") if rerank_debug else None,
daa2690b   tangwang   漏斗参数调优&呈现优化
1210
1211
1212
                              "text_factor": rerank_debug.get("text_factor") if rerank_debug else None,
                              "knn_factor": rerank_debug.get("knn_factor") if rerank_debug else None,
                              "signals": rerank_debug,
465f90e1   tangwang   添加LTR数据收集
1213
                              "ltr_features": rerank_debug.get("ltr_features") if rerank_debug else None,
daa2690b   tangwang   漏斗参数调优&呈现优化
1214
1215
1216
                          },
                          "final_page": {
                              "rank": final_rank,
9df421ed   tangwang   基于eval框架开始调参
1217
                              "rank_change": _rank_change(final_previous_rank, final_rank),
daa2690b   tangwang   漏斗参数调优&呈现优化
1218
1219
1220
                          },
                      }
  
cda1cd62   tangwang   意图分析&应用 baseline
1221
1222
1223
                      if style_intent_debug:
                          debug_entry["style_intent_sku"] = style_intent_debug
  
af827ce9   tangwang   rerank
1224
                      per_result_debug.append(debug_entry)
985752f5   tangwang   1. 前端调试功能
1225
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1226
1227
1228
1229
1230
              # Format facets
              standardized_facets = None
              if facets:
                  standardized_facets = ResultFormatter.format_facets(
                      es_response.get('aggregations', {}),
c581becd   tangwang   feat: 实现 Multi-Se...
1231
1232
                      facets,
                      filters
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1233
1234
1235
1236
1237
1238
                  )
  
              # Generate suggestions and related searches
              query_text = parsed_query.original_query if parsed_query else query
              suggestions = ResultFormatter.generate_suggestions(query_text, formatted_results)
              related_searches = ResultFormatter.generate_related_searches(query_text, formatted_results)
be52af70   tangwang   first commit
1239
  
16c42787   tangwang   feat: implement r...
1240
              context.logger.info(
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1241
                  f"结果处理完成 | 返回: {len(formatted_results)}条 | 总计: {total_value}条",
16c42787   tangwang   feat: implement r...
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
  
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"结果处理失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
              context.end_stage(RequestContextStage.RESULT_PROCESSING)
be52af70   tangwang   first commit
1254
  
16c42787   tangwang   feat: implement r...
1255
1256
1257
          # End total timing and build result
          total_duration = context.end_stage(RequestContextStage.TOTAL)
          context.performance_metrics.total_duration = total_duration
be52af70   tangwang   first commit
1258
  
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1259
1260
1261
          # Collect debug information if requested
          debug_info = None
          if debug:
465f90e1   tangwang   添加LTR数据收集
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
              query_tokens = getattr(parsed_query, "query_tokens", []) if parsed_query else []
              token_count = len(query_tokens)
              text_knn_is_long = token_count >= 5
              text_knn_k = self.query_builder.knn_text_k_long if text_knn_is_long else self.query_builder.knn_text_k
              text_knn_num_candidates = (
                  self.query_builder.knn_text_num_candidates_long
                  if text_knn_is_long
                  else self.query_builder.knn_text_num_candidates
              )
              ltr_summary = _summarize_ltr_features(per_result_debug)
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1272
1273
1274
              debug_info = {
                  "query_analysis": {
                      "original_query": context.query_analysis.original_query,
3a5fda00   tangwang   1. ES字段 skus的 ima...
1275
                      "query_normalized": context.query_analysis.query_normalized,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1276
1277
                      "rewritten_query": context.query_analysis.rewritten_query,
                      "detected_language": context.query_analysis.detected_language,
581dafae   tangwang   debug工具,每条结果的打分中间...
1278
                      "index_languages": index_langs,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1279
                      "translations": context.query_analysis.translations,
9d0214bb   tangwang   qp性能优化
1280
                      "keywords_queries": context.query_analysis.keywords_queries,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1281
                      "has_vector": context.query_analysis.query_vector is not None,
24edc208   tangwang   修改_extract_combin...
1282
                      "has_image_vector": getattr(parsed_query, "image_query_vector", None) is not None,
465f90e1   tangwang   添加LTR数据收集
1283
                      "query_tokens": query_tokens,
2efad04b   tangwang   意图匹配的性能优化:
1284
                      "intent_detection": context.get_intermediate_result("style_intent_profile"),
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1285
                  },
465f90e1   tangwang   添加LTR数据收集
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
                  "retrieval_plan": {
                      "text_knn": {
                          "enabled": bool(enable_embedding and parsed_query and parsed_query.query_vector is not None),
                          "is_long_query_plan": text_knn_is_long,
                          "token_count": token_count,
                          "k": text_knn_k,
                          "num_candidates": text_knn_num_candidates,
                          "boost": (
                              self.query_builder.knn_text_boost * 1.4
                              if text_knn_is_long
                              else self.query_builder.knn_text_boost
                          ),
                      },
                      "image_knn": {
                          "enabled": bool(
                              enable_embedding
                              and parsed_query
                              and getattr(parsed_query, "image_query_vector", None) is not None
                          ),
                          "k": self.query_builder.knn_image_k,
                          "num_candidates": self.query_builder.knn_image_num_candidates,
                          "boost": self.query_builder.knn_image_boost,
                      },
                  },
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1310
                  "es_query": context.get_intermediate_result('es_query', {}),
581dafae   tangwang   debug工具,每条结果的打分中间...
1311
                  "es_query_context": {
581dafae   tangwang   debug工具,每条结果的打分中间...
1312
1313
1314
1315
1316
1317
                      "es_fetch_from": es_fetch_from,
                      "es_fetch_size": es_fetch_size,
                      "in_rerank_window": in_rerank_window,
                      "rerank_prefetch_source": context.get_intermediate_result('es_query_rerank_prefetch_source'),
                      "include_named_queries_score": bool(do_rerank and in_rerank_window),
                  },
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1318
1319
1320
1321
                  "es_response": {
                      "took_ms": es_response.get('took', 0),
                      "total_hits": total_value,
                      "max_score": max_score,
581dafae   tangwang   debug工具,每条结果的打分中间...
1322
                      "shards": es_response.get('_shards', {}),
814e352b   tangwang   乘法公式配置化
1323
                      "es_score_normalization_factor": es_score_normalization_factor,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1324
                  },
8c8b9d84   tangwang   ES 拉取 coarse_rank...
1325
1326
                  "coarse_rank": coarse_debug_info,
                  "fine_rank": fine_debug_info,
581dafae   tangwang   debug工具,每条结果的打分中间...
1327
                  "rerank": rerank_debug_info,
daa2690b   tangwang   漏斗参数调优&呈现优化
1328
1329
1330
1331
1332
1333
1334
1335
1336
                  "ranking_funnel": {
                      "es_recall": {
                          "docs_out": es_fetch_size,
                          "score_normalization_factor": es_score_normalization_factor,
                      },
                      "coarse_rank": coarse_debug_info,
                      "fine_rank": fine_debug_info,
                      "rerank": rerank_debug_info,
                  },
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1337
1338
1339
1340
                  "feature_flags": context.metadata.get('feature_flags', {}),
                  "stage_timings": {
                      k: round(v, 2) for k, v in context.performance_metrics.stage_timings.items()
                  },
8ae95af0   tangwang   1. Stage Timings:...
1341
1342
1343
1344
1345
1346
                  "stage_time_bounds_ms": {
                      stage: {
                          kk: round(vv, 3) for kk, vv in bounds.items()
                      }
                      for stage, bounds in context.performance_metrics.stage_time_bounds_ms.items()
                  },
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1347
1348
                  "search_params": context.metadata.get('search_params', {})
              }
985752f5   tangwang   1. 前端调试功能
1349
1350
              if per_result_debug:
                  debug_info["per_result"] = per_result_debug
465f90e1   tangwang   添加LTR数据收集
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
                  debug_info["ltr_summary"] = ltr_summary
                  _log_backend_verbose({
                      "event": "search_debug_ltr_summary",
                      "reqid": context.reqid,
                      "uid": context.uid,
                      "tenant_id": tenant_id,
                      "query": query,
                      "language": language,
                      "top_n": ltr_summary.get("top_n"),
                      "counts": ltr_summary.get("counts"),
                      "averages": ltr_summary.get("averages"),
                      "top_docs": ltr_summary.get("top_docs"),
                      "query_analysis": {
                          "rewritten_query": context.query_analysis.rewritten_query,
                          "detected_language": context.query_analysis.detected_language,
                          "translations": context.query_analysis.translations,
                          "query_tokens": query_tokens,
                      },
                      "retrieval_plan": debug_info["retrieval_plan"],
                      "ranking_windows": {
                          "es_fetch_size": es_fetch_size,
                          "coarse_output_window": coarse_output_window if do_rerank and in_rerank_window else None,
                          "fine_input_window": fine_input_window if do_rerank and in_rerank_window else None,
                          "fine_output_window": fine_output_window if do_rerank and in_rerank_window else None,
                          "rerank_window": rerank_window if do_rerank and in_rerank_window else None,
                          "page_from": from_,
                          "page_size": size,
                      },
                  })
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1380
  
be52af70   tangwang   first commit
1381
1382
          # Build result
          result = SearchResult(
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1383
              results=formatted_results,
be52af70   tangwang   first commit
1384
1385
              total=total_value,
              max_score=max_score,
16c42787   tangwang   feat: implement r...
1386
              took_ms=int(total_duration),
6aa246be   tangwang   问题:Pydantic 应该能自动...
1387
              facets=standardized_facets,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1388
              query_info=parsed_query.to_dict(),
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1389
1390
              suggestions=suggestions,
              related_searches=related_searches,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1391
              debug_info=debug_info
be52af70   tangwang   first commit
1392
1393
          )
  
16c42787   tangwang   feat: implement r...
1394
1395
          # Log complete performance summary
          context.log_performance_summary()
be52af70   tangwang   first commit
1396
1397
1398
1399
1400
1401
  
          return result
  
      def search_by_image(
          self,
          image_url: str,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1402
          tenant_id: str,
be52af70   tangwang   first commit
1403
          size: int = 10,
6aa246be   tangwang   问题:Pydantic 应该能自动...
1404
1405
          filters: Optional[Dict[str, Any]] = None,
          range_filters: Optional[Dict[str, Any]] = None
be52af70   tangwang   first commit
1406
1407
      ) -> SearchResult:
          """
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1408
          Search by image similarity (外部友好格式).
be52af70   tangwang   first commit
1409
1410
1411
  
          Args:
              image_url: URL of query image
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1412
              tenant_id: Tenant ID (required for filtering)
be52af70   tangwang   first commit
1413
              size: Number of results
6aa246be   tangwang   问题:Pydantic 应该能自动...
1414
1415
              filters: Exact match filters
              range_filters: Range filters for numeric fields
be52af70   tangwang   first commit
1416
1417
  
          Returns:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1418
              SearchResult object with formatted results
be52af70   tangwang   first commit
1419
1420
1421
1422
1423
          """
          if not self.image_embedding_field:
              raise ValueError("Image embedding field not configured")
  
          # Generate image embedding
26b910bd   tangwang   refactor service ...
1424
1425
          if self.image_encoder is None:
              raise RuntimeError("Image encoder is not initialized at startup")
b754fd41   tangwang   图片向量化支持优先级参数
1426
          image_vector = self.image_encoder.encode_image_from_url(image_url, priority=1)
be52af70   tangwang   first commit
1427
1428
1429
1430
  
          if image_vector is None:
              raise ValueError(f"Failed to encode image: {image_url}")
  
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1431
1432
1433
1434
          # Generate tenant-specific index name
          index_name = get_tenant_index_name(tenant_id)
          
          # No longer need to add tenant_id to filters since each tenant has its own index
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1435
  
be52af70   tangwang   first commit
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
          # Build KNN query
          es_query = {
              "size": size,
              "knn": {
                  "field": self.image_embedding_field,
                  "query_vector": image_vector.tolist(),
                  "k": size,
                  "num_candidates": size * 10
              }
          }
  
26b910bd   tangwang   refactor service ...
1447
1448
          # Apply source filtering semantics (None / [] / list)
          self._apply_source_filter(es_query)
13377199   tangwang   接口优化
1449
  
6aa246be   tangwang   问题:Pydantic 应该能自动...
1450
1451
1452
          if filters or range_filters:
              filter_clauses = self.query_builder._build_filters(filters, range_filters)
              if filter_clauses:
7fbca0d7   tangwang   启动脚本优化
1453
1454
1455
1456
1457
1458
1459
                  if len(filter_clauses) == 1:
                      es_query["knn"]["filter"] = filter_clauses[0]
                  else:
                      es_query["knn"]["filter"] = {
                          "bool": {
                              "filter": filter_clauses
                          }
6aa246be   tangwang   问题:Pydantic 应该能自动...
1460
                      }
be52af70   tangwang   first commit
1461
1462
1463
  
          # Execute search
          es_response = self.es_client.search(
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1464
              index_name=index_name,
be52af70   tangwang   first commit
1465
1466
1467
1468
              body=es_query,
              size=size
          )
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1469
1470
          # Extract ES hits
          es_hits = []
be52af70   tangwang   first commit
1471
          if 'hits' in es_response and 'hits' in es_response['hits']:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1472
              es_hits = es_response['hits']['hits']
be52af70   tangwang   first commit
1473
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1474
          # Extract total and max_score
be52af70   tangwang   first commit
1475
1476
1477
1478
1479
1480
          total = es_response.get('hits', {}).get('total', {})
          if isinstance(total, dict):
              total_value = total.get('value', 0)
          else:
              total_value = total
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1481
1482
1483
          max_score = es_response.get('hits', {}).get('max_score') or 0.0
  
          # Format results using ResultFormatter
ca91352a   tangwang   更新文档
1484
1485
1486
          formatted_results = ResultFormatter.format_search_results(
              es_hits, 
              max_score,
2739b281   tangwang   多语言索引调整
1487
              language="en",  # Default language for image search
ca91352a   tangwang   更新文档
1488
1489
              sku_filter_dimension=None  # Image search doesn't support SKU filtering
          )
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1490
  
be52af70   tangwang   first commit
1491
          return SearchResult(
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1492
              results=formatted_results,
be52af70   tangwang   first commit
1493
              total=total_value,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1494
              max_score=max_score,
be52af70   tangwang   first commit
1495
              took_ms=es_response.get('took', 0),
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1496
1497
1498
1499
              facets=None,
              query_info={'image_url': image_url, 'search_type': 'image_similarity'},
              suggestions=[],
              related_searches=[]
be52af70   tangwang   first commit
1500
1501
          )
  
b926f678   tangwang   多语言查询
1502
1503
      def get_domain_summary(self) -> Dict[str, Any]:
          """
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
1504
          Get summary of dynamic text retrieval configuration.
b926f678   tangwang   多语言查询
1505
1506
  
          Returns:
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
1507
              Dictionary with language-aware field information
b926f678   tangwang   多语言查询
1508
          """
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
1509
1510
1511
1512
1513
1514
1515
          return {
              "mode": "dynamic_language_fields",
              "multilingual_fields": self.config.query_config.multilingual_fields,
              "shared_fields": self.config.query_config.shared_fields,
              "core_multilingual_fields": self.config.query_config.core_multilingual_fields,
              "field_boosts": self.config.field_boosts,
          }
b926f678   tangwang   多语言查询
1516
  
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1517
      def get_document(self, tenant_id: str, doc_id: str) -> Optional[Dict[str, Any]]:
be52af70   tangwang   first commit
1518
1519
1520
1521
          """
          Get single document by ID.
  
          Args:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1522
              tenant_id: Tenant ID (required to determine which index to query)
be52af70   tangwang   first commit
1523
1524
1525
1526
1527
1528
              doc_id: Document ID
  
          Returns:
              Document or None if not found
          """
          try:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1529
              index_name = get_tenant_index_name(tenant_id)
be52af70   tangwang   first commit
1530
              response = self.es_client.client.get(
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1531
                  index=index_name,
be52af70   tangwang   first commit
1532
1533
1534
1535
                  id=doc_id
              )
              return response.get('_source')
          except Exception as e:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1536
              logger.error(f"Failed to get document {doc_id} from tenant {tenant_id}: {e}", exc_info=True)
be52af70   tangwang   first commit
1537
              return None