Blame view

search/searcher.py 49.1 KB
be52af70   tangwang   first commit
1
2
3
  """
  Main Searcher module - executes search queries against Elasticsearch.
  
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
4
  Handles query parsing, ranking, and result formatting.
be52af70   tangwang   first commit
5
6
  """
  
deccd68a   tangwang   Added the SKU pre...
7
  from typing import Dict, Any, List, Optional, Union, Tuple
d1d356f8   tangwang   脚本优化
8
  import os
99bea633   tangwang   add logs
9
  import time, json
325eec03   tangwang   1. 日志、配置基础设施,使用优化
10
  import logging
28e57bb1   tangwang   日志体系优化
11
  import hashlib
5f7d7f09   tangwang   性能测试报告.md
12
  from string import Formatter
deccd68a   tangwang   Added the SKU pre...
13
  import numpy as np
be52af70   tangwang   first commit
14
  
be52af70   tangwang   first commit
15
16
  from utils.es_client import ESClient
  from query import QueryParser, ParsedQuery
07cf5a93   tangwang   START_EMBEDDING=...
17
  from embeddings.image_encoder import CLIPImageEncoder
be52af70   tangwang   first commit
18
  from .es_query_builder import ESQueryBuilder
9f96d6f3   tangwang   短query不用语义搜索
19
  from config import SearchConfig
345d960b   tangwang   1. 删除全局 enable_tr...
20
  from config.tenant_config_loader import get_tenant_config_loader
ed948666   tangwang   tidy
21
  from context.request_context import RequestContext, RequestContextStage
13320ac6   tangwang   分面接口修改:
22
  from api.models import FacetResult, FacetValue, FacetConfig
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
23
  from api.result_formatter import ResultFormatter
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
24
  from indexer.mapping_generator import get_tenant_index_name
be52af70   tangwang   first commit
25
  
325eec03   tangwang   1. 日志、配置基础设施,使用优化
26
  logger = logging.getLogger(__name__)
28e57bb1   tangwang   日志体系优化
27
28
29
30
31
32
33
34
35
  backend_verbose_logger = logging.getLogger("backend.verbose")
  
  
  def _log_backend_verbose(payload: Dict[str, Any]) -> None:
      if not backend_verbose_logger.handlers:
          return
      backend_verbose_logger.info(
          json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
      )
325eec03   tangwang   1. 日志、配置基础设施,使用优化
36
  
be52af70   tangwang   first commit
37
38
  
  class SearchResult:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
39
      """Container for search results (外部友好格式)."""
be52af70   tangwang   first commit
40
41
42
  
      def __init__(
          self,
cadc77b6   tangwang   索引字段名、变量名、API数据结构...
43
          results: List[Any],  # List[SpuResult]
be52af70   tangwang   first commit
44
45
46
          total: int,
          max_score: float,
          took_ms: int,
6aa246be   tangwang   问题:Pydantic 应该能自动...
47
          facets: Optional[List[FacetResult]] = None,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
48
          query_info: Optional[Dict[str, Any]] = None,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
49
50
          suggestions: Optional[List[str]] = None,
          related_searches: Optional[List[str]] = None,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
51
          debug_info: Optional[Dict[str, Any]] = None
be52af70   tangwang   first commit
52
      ):
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
53
          self.results = results
be52af70   tangwang   first commit
54
55
56
          self.total = total
          self.max_score = max_score
          self.took_ms = took_ms
6aa246be   tangwang   问题:Pydantic 应该能自动...
57
          self.facets = facets
be52af70   tangwang   first commit
58
          self.query_info = query_info or {}
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
59
60
          self.suggestions = suggestions or []
          self.related_searches = related_searches or []
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
61
          self.debug_info = debug_info
43f1139f   tangwang   refactor: ES查询结构重...
62
  
be52af70   tangwang   first commit
63
64
      def to_dict(self) -> Dict[str, Any]:
          """Convert to dictionary representation."""
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
65
          result = {
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
66
              "results": [r.model_dump() if hasattr(r, 'model_dump') else r for r in self.results],
be52af70   tangwang   first commit
67
68
69
              "total": self.total,
              "max_score": self.max_score,
              "took_ms": self.took_ms,
6aa246be   tangwang   问题:Pydantic 应该能自动...
70
              "facets": [f.model_dump() for f in self.facets] if self.facets else None,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
71
72
73
              "query_info": self.query_info,
              "suggestions": self.suggestions,
              "related_searches": self.related_searches
be52af70   tangwang   first commit
74
          }
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
75
76
77
          if self.debug_info is not None:
              result["debug_info"] = self.debug_info
          return result
be52af70   tangwang   first commit
78
79
80
81
82
83
84
85
  
  
  class Searcher:
      """
      Main search engine class.
  
      Handles:
      - Query parsing and translation
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
86
      - Dynamic multi-language text recall planning
be52af70   tangwang   first commit
87
88
89
90
91
92
      - ES query building
      - Result ranking and formatting
      """
  
      def __init__(
          self,
be52af70   tangwang   first commit
93
          es_client: ESClient,
9f96d6f3   tangwang   短query不用语义搜索
94
          config: SearchConfig,
26b910bd   tangwang   refactor service ...
95
96
          query_parser: Optional[QueryParser] = None,
          image_encoder: Optional[CLIPImageEncoder] = None,
be52af70   tangwang   first commit
97
98
99
100
101
      ):
          """
          Initialize searcher.
  
          Args:
be52af70   tangwang   first commit
102
              es_client: Elasticsearch client
9f96d6f3   tangwang   短query不用语义搜索
103
              config: SearchConfig instance
be52af70   tangwang   first commit
104
              query_parser: Query parser (created if not provided)
26b910bd   tangwang   refactor service ...
105
              image_encoder: Optional pre-initialized image encoder
be52af70   tangwang   first commit
106
          """
be52af70   tangwang   first commit
107
          self.es_client = es_client
9f96d6f3   tangwang   短query不用语义搜索
108
          self.config = config
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
109
          # Index name is now generated dynamically per tenant, no longer stored here
9f96d6f3   tangwang   短query不用语义搜索
110
          self.query_parser = query_parser or QueryParser(config)
9f96d6f3   tangwang   短query不用语义搜索
111
          self.text_embedding_field = config.query_config.text_embedding_field or "title_embedding"
26b910bd   tangwang   refactor service ...
112
113
114
115
116
117
          self.image_embedding_field = config.query_config.image_embedding_field
          if self.image_embedding_field and image_encoder is None:
              self.image_encoder = CLIPImageEncoder()
          else:
              self.image_encoder = image_encoder
          self.source_fields = config.query_config.source_fields
be52af70   tangwang   first commit
118
  
f0d020c3   tangwang   多语言查询改为只支持中英文两种,f...
119
          # Query builder - simplified single-layer architecture
f0d020c3   tangwang   多语言查询改为只支持中英文两种,f...
120
          self.query_builder = ESQueryBuilder(
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
121
122
123
124
125
              match_fields=[],
              field_boosts=self.config.field_boosts,
              multilingual_fields=self.config.query_config.multilingual_fields,
              shared_fields=self.config.query_config.shared_fields,
              core_multilingual_fields=self.config.query_config.core_multilingual_fields,
be52af70   tangwang   first commit
126
              text_embedding_field=self.text_embedding_field,
13377199   tangwang   接口优化
127
              image_embedding_field=self.image_embedding_field,
9f96d6f3   tangwang   短query不用语义搜索
128
              source_fields=self.source_fields,
7bc756c5   tangwang   优化 ES 查询构建
129
              function_score_config=self.config.function_score,
70dab99f   tangwang   add logs
130
              default_language=self.config.query_config.default_language,
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
131
132
133
134
135
136
              knn_boost=self.config.query_config.knn_boost,
              base_minimum_should_match=self.config.query_config.base_minimum_should_match,
              translation_minimum_should_match=self.config.query_config.translation_minimum_should_match,
              translation_boost=self.config.query_config.translation_boost,
              translation_boost_when_source_missing=self.config.query_config.translation_boost_when_source_missing,
              source_boost_when_missing=self.config.query_config.source_boost_when_missing,
bcada818   tangwang   last
137
138
139
              original_query_fallback_boost_when_translation_missing=(
                  self.config.query_config.original_query_fallback_boost_when_translation_missing
              ),
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
140
              tie_breaker_base_query=self.config.query_config.tie_breaker_base_query,
be52af70   tangwang   first commit
141
142
          )
  
26b910bd   tangwang   refactor service ...
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
      def _apply_source_filter(self, es_query: Dict[str, Any]) -> None:
          """
          Apply tri-state _source semantics:
          - None: do not set _source (return full source)
          - []: _source=false (return no source fields)
          - [..]: _source.includes=[..]
          """
          if self.source_fields is None:
              return
          if not isinstance(self.source_fields, list):
              raise ValueError("query_config.source_fields must be null or list[str]")
          if len(self.source_fields) == 0:
              es_query["_source"] = False
              return
          es_query["_source"] = {"includes": self.source_fields}
  
5f7d7f09   tangwang   性能测试报告.md
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
      def _resolve_rerank_source_filter(self, doc_template: str) -> Dict[str, Any]:
          """
          Build a lightweight _source filter for rerank prefetch.
  
          Only fetch fields required by rerank doc template to reduce ES payload.
          """
          field_map = {
              "title": "title",
              "brief": "brief",
              "vendor": "vendor",
              "description": "description",
              "category_path": "category_path",
          }
          includes: set[str] = set()
          template = str(doc_template or "{title}")
          for _, field_name, _, _ in Formatter().parse(template):
              if not field_name:
                  continue
              key = field_name.split(".", 1)[0].split("!", 1)[0].split(":", 1)[0]
              mapped = field_map.get(key)
              if mapped:
                  includes.add(mapped)
  
          # Fallback to title-only to keep rerank docs usable.
          if not includes:
              includes.add("title")
  
          return {"includes": sorted(includes)}
  
      def _fetch_hits_by_ids(
          self,
          index_name: str,
          doc_ids: List[str],
          source_spec: Optional[Any],
      ) -> tuple[Dict[str, Dict[str, Any]], int]:
          """
          Fetch page documents by IDs for final response fill.
  
          Returns:
              (hits_by_id, es_took_ms)
          """
          if not doc_ids:
              return {}, 0
  
          body: Dict[str, Any] = {
              "query": {
                  "ids": {
                      "values": doc_ids,
                  }
              }
          }
          if source_spec is not None:
              body["_source"] = source_spec
  
          resp = self.es_client.search(
              index_name=index_name,
              body=body,
              size=len(doc_ids),
              from_=0,
          )
          hits = resp.get("hits", {}).get("hits") or []
          hits_by_id: Dict[str, Dict[str, Any]] = {}
          for hit in hits:
              hid = hit.get("_id")
              if hid is None:
                  continue
              hits_by_id[str(hid)] = hit
          return hits_by_id, int(resp.get("took", 0) or 0)
  
deccd68a   tangwang   Added the SKU pre...
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
      @staticmethod
      def _normalize_sku_match_text(value: Optional[str]) -> str:
          """Normalize free text for lightweight SKU option matching."""
          if value is None:
              return ""
          return " ".join(str(value).strip().casefold().split())
  
      def _build_sku_query_texts(self, parsed_query: ParsedQuery) -> List[str]:
          """Collect original and translated query texts for SKU option matching."""
          candidates: List[str] = []
          for text in (
              getattr(parsed_query, "original_query", None),
              getattr(parsed_query, "query_normalized", None),
              getattr(parsed_query, "rewritten_query", None),
          ):
              normalized = self._normalize_sku_match_text(text)
              if normalized:
                  candidates.append(normalized)
  
          query_text_by_lang = getattr(parsed_query, "query_text_by_lang", {}) or {}
          if isinstance(query_text_by_lang, dict):
              for text in query_text_by_lang.values():
                  normalized = self._normalize_sku_match_text(text)
                  if normalized:
                      candidates.append(normalized)
  
          translations = getattr(parsed_query, "translations", {}) or {}
          if isinstance(translations, dict):
              for text in translations.values():
                  normalized = self._normalize_sku_match_text(text)
                  if normalized:
                      candidates.append(normalized)
  
          deduped: List[str] = []
          seen = set()
          for text in candidates:
              if text in seen:
                  continue
              seen.add(text)
              deduped.append(text)
          return deduped
  
      def _find_query_matching_sku_index(
          self,
          skus: List[Dict[str, Any]],
          query_texts: List[str],
      ) -> Optional[int]:
          """Return the first SKU whose option1_value appears in query texts."""
          if not skus or not query_texts:
              return None
  
          for index, sku in enumerate(skus):
              option1_value = self._normalize_sku_match_text(sku.get("option1_value"))
              if not option1_value:
                  continue
              if any(option1_value in query_text for query_text in query_texts):
                  return index
          return None
  
      def _encode_query_vector_for_sku_matching(
          self,
          parsed_query: ParsedQuery,
          context: Optional[RequestContext] = None,
      ) -> Optional[np.ndarray]:
          """Best-effort fallback query embedding for final-page SKU matching."""
          query_text = (
              getattr(parsed_query, "rewritten_query", None)
              or getattr(parsed_query, "query_normalized", None)
              or getattr(parsed_query, "original_query", None)
          )
          if not query_text:
              return None
  
          text_encoder = getattr(self.query_parser, "text_encoder", None)
          if text_encoder is None:
              return None
  
          try:
              vectors = text_encoder.encode([query_text], priority=1)
          except Exception as exc:
              logger.warning("Failed to encode query vector for SKU matching: %s", exc, exc_info=True)
              if context is not None:
                  context.add_warning(f"SKU query embedding failed: {exc}")
              return None
  
          if vectors is None or len(vectors) == 0:
              return None
  
          vector = vectors[0]
          if vector is None:
              return None
          return np.asarray(vector, dtype=np.float32)
  
      def _select_sku_by_embedding(
          self,
          skus: List[Dict[str, Any]],
          option1_vectors: Dict[str, np.ndarray],
          query_vector: np.ndarray,
      ) -> Tuple[Optional[int], Optional[float]]:
          """Select the SKU whose option1_value is most similar to the query."""
          best_index: Optional[int] = None
          best_score: Optional[float] = None
  
          for index, sku in enumerate(skus):
              option1_value_raw = sku.get("option1_value")
              if option1_value_raw is None:
                  continue
              option1_value = str(option1_value_raw).strip()
              if not option1_value:
                  continue
              option_vector = option1_vectors.get(option1_value)
              if option_vector is None:
                  continue
              score = float(np.inner(query_vector, option_vector))
              if best_score is None or score > best_score:
                  best_index = index
                  best_score = score
  
          return best_index, best_score
  
      @staticmethod
      def _promote_matching_sku(source: Dict[str, Any], match_index: int) -> Optional[Dict[str, Any]]:
          """Move the matched SKU to the front and swap the SPU image."""
          skus = source.get("skus")
          if not isinstance(skus, list) or match_index < 0 or match_index >= len(skus):
              return None
  
          matched_sku = skus.pop(match_index)
          skus.insert(0, matched_sku)
  
          image_src = matched_sku.get("image_src") or matched_sku.get("imageSrc")
          if image_src:
              source["image_url"] = image_src
          return matched_sku
  
      def _apply_sku_sorting_for_page_hits(
          self,
          es_hits: List[Dict[str, Any]],
          parsed_query: ParsedQuery,
          context: Optional[RequestContext] = None,
      ) -> None:
          """Sort each page hit's SKUs so the best-matching SKU is first."""
          if not es_hits:
              return
  
          query_texts = self._build_sku_query_texts(parsed_query)
          unmatched_hits: List[Dict[str, Any]] = []
          option1_values_to_encode: List[str] = []
          seen_option1_values = set()
          text_matched = 0
          embedding_matched = 0
  
          for hit in es_hits:
              source = hit.get("_source")
              if not isinstance(source, dict):
                  continue
              skus = source.get("skus")
              if not isinstance(skus, list) or not skus:
                  continue
  
              match_index = self._find_query_matching_sku_index(skus, query_texts)
              if match_index is not None:
                  self._promote_matching_sku(source, match_index)
                  text_matched += 1
                  continue
  
              unmatched_hits.append(hit)
              for sku in skus:
                  option1_value_raw = sku.get("option1_value")
                  if option1_value_raw is None:
                      continue
                  option1_value = str(option1_value_raw).strip()
                  if not option1_value or option1_value in seen_option1_values:
                      continue
                  seen_option1_values.add(option1_value)
                  option1_values_to_encode.append(option1_value)
  
          if not unmatched_hits or not option1_values_to_encode:
              return
  
          query_vector = getattr(parsed_query, "query_vector", None)
          if query_vector is None:
              query_vector = self._encode_query_vector_for_sku_matching(parsed_query, context=context)
          if query_vector is None:
              return
  
          text_encoder = getattr(self.query_parser, "text_encoder", None)
          if text_encoder is None:
              return
  
          try:
              encoded_option_vectors = text_encoder.encode(option1_values_to_encode, priority=1)
          except Exception as exc:
              logger.warning("Failed to encode SKU option1 values for final-page sorting: %s", exc, exc_info=True)
              if context is not None:
                  context.add_warning(f"SKU option embedding failed: {exc}")
              return
  
          option1_vectors: Dict[str, np.ndarray] = {}
          for option1_value, vector in zip(option1_values_to_encode, encoded_option_vectors):
              if vector is None:
                  continue
              option1_vectors[option1_value] = np.asarray(vector, dtype=np.float32)
  
          query_vector_array = np.asarray(query_vector, dtype=np.float32)
          for hit in unmatched_hits:
              source = hit.get("_source")
              if not isinstance(source, dict):
                  continue
              skus = source.get("skus")
              if not isinstance(skus, list) or not skus:
                  continue
              match_index, _ = self._select_sku_by_embedding(skus, option1_vectors, query_vector_array)
              if match_index is None:
                  continue
              self._promote_matching_sku(source, match_index)
              embedding_matched += 1
  
          if text_matched or embedding_matched:
              logger.info(
                  "Final-page SKU sorting completed | text_matched=%s | embedding_matched=%s",
                  text_matched,
                  embedding_matched,
              )
  
be52af70   tangwang   first commit
453
454
455
      def search(
          self,
          query: str,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
456
          tenant_id: str,
be52af70   tangwang   first commit
457
458
459
          size: int = 10,
          from_: int = 0,
          filters: Optional[Dict[str, Any]] = None,
6aa246be   tangwang   问题:Pydantic 应该能自动...
460
          range_filters: Optional[Dict[str, Any]] = None,
13320ac6   tangwang   分面接口修改:
461
          facets: Optional[List[FacetConfig]] = None,
16c42787   tangwang   feat: implement r...
462
          min_score: Optional[float] = None,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
463
          context: Optional[RequestContext] = None,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
464
          sort_by: Optional[str] = None,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
465
          sort_order: Optional[str] = "desc",
577ec972   tangwang   返回给前端的字段、格式适配。主要包...
466
          debug: bool = False,
2739b281   tangwang   多语言索引调整
467
          language: str = "en",
a3a5d41b   tangwang   (sku_filter_dimen...
468
          sku_filter_dimension: Optional[List[str]] = None,
5f7d7f09   tangwang   性能测试报告.md
469
          enable_rerank: Optional[bool] = None,
ff32d894   tangwang   rerank
470
471
          rerank_query_template: Optional[str] = None,
          rerank_doc_template: Optional[str] = None,
be52af70   tangwang   first commit
472
473
      ) -> SearchResult:
          """
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
474
          Execute search query (外部友好格式).
be52af70   tangwang   first commit
475
476
477
  
          Args:
              query: Search query string
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
478
              tenant_id: Tenant ID (required for filtering)
be52af70   tangwang   first commit
479
480
              size: Number of results to return
              from_: Offset for pagination
6aa246be   tangwang   问题:Pydantic 应该能自动...
481
482
483
              filters: Exact match filters
              range_filters: Range filters for numeric fields
              facets: Facet configurations for faceted search
be52af70   tangwang   first commit
484
              min_score: Minimum score threshold
16c42787   tangwang   feat: implement r...
485
              context: Request context for tracking (created if not provided)
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
486
487
              sort_by: Field name for sorting
              sort_order: Sort order: 'asc' or 'desc'
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
488
              debug: Enable debug information output
be52af70   tangwang   first commit
489
490
  
          Returns:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
491
              SearchResult object with formatted results
be52af70   tangwang   first commit
492
          """
16c42787   tangwang   feat: implement r...
493
          if context is None:
ed948666   tangwang   tidy
494
              raise ValueError("context is required")
16c42787   tangwang   feat: implement r...
495
  
345d960b   tangwang   1. 删除全局 enable_tr...
496
497
498
          # 根据租户配置决定翻译开关(离线/在线统一)
          tenant_loader = get_tenant_config_loader()
          tenant_cfg = tenant_loader.get_tenant_config(tenant_id)
038e4e2f   tangwang   refactor(i18n): t...
499
500
          index_langs = tenant_cfg.get("index_languages") or []
          enable_translation = len(index_langs) > 0
9f96d6f3   tangwang   短query不用语义搜索
501
          enable_embedding = self.config.query_config.enable_text_embedding
5f7d7f09   tangwang   性能测试报告.md
502
503
504
505
506
507
          rc = self.config.rerank
          effective_query_template = rerank_query_template or rc.rerank_query_template
          effective_doc_template = rerank_doc_template or rc.rerank_doc_template
          # 重排开关优先级:请求参数显式传值 > 服务端配置(默认开启)
          rerank_enabled_by_config = bool(rc.enabled)
          do_rerank = rerank_enabled_by_config if enable_rerank is None else bool(enable_rerank)
c51d254f   tangwang   性能测试
508
          rerank_window = rc.rerank_window
506c39b7   tangwang   feat(search): 统一重...
509
          # 若开启重排且请求范围在窗口内:从 ES 取前 rerank_window 条、重排后再按 from/size 分页;否则不重排,按原 from/size 查 ES
ff32d894   tangwang   rerank
510
          in_rerank_window = do_rerank and (from_ + size) <= rerank_window
506c39b7   tangwang   feat(search): 统一重...
511
512
          es_fetch_from = 0 if in_rerank_window else from_
          es_fetch_size = rerank_window if in_rerank_window else size
16c42787   tangwang   feat: implement r...
513
514
515
516
517
518
  
          # Start timing
          context.start_stage(RequestContextStage.TOTAL)
  
          context.logger.info(
              f"开始搜索请求 | 查询: '{query}' | 参数: size={size}, from_={from_}, "
5f7d7f09   tangwang   性能测试报告.md
519
520
521
              f"enable_rerank(request)={enable_rerank}, enable_rerank(config)={rerank_enabled_by_config}, "
              f"enable_rerank(effective)={do_rerank}, in_rerank_window={in_rerank_window}, "
              f"es_fetch=({es_fetch_from},{es_fetch_size}) | "
506c39b7   tangwang   feat(search): 统一重...
522
              f"enable_translation={enable_translation}, enable_embedding={enable_embedding}, min_score={min_score}",
16c42787   tangwang   feat: implement r...
523
524
525
526
527
528
529
              extra={'reqid': context.reqid, 'uid': context.uid}
          )
  
          # Store search parameters in context
          context.metadata['search_params'] = {
              'size': size,
              'from_': from_,
506c39b7   tangwang   feat(search): 统一重...
530
531
532
              'es_fetch_from': es_fetch_from,
              'es_fetch_size': es_fetch_size,
              'in_rerank_window': in_rerank_window,
5f7d7f09   tangwang   性能测试报告.md
533
534
535
536
              'rerank_enabled_by_config': rerank_enabled_by_config,
              'enable_rerank_request': enable_rerank,
              'rerank_query_template': effective_query_template,
              'rerank_doc_template': effective_doc_template,
16c42787   tangwang   feat: implement r...
537
              'filters': filters,
6aa246be   tangwang   问题:Pydantic 应该能自动...
538
539
              'range_filters': range_filters,
              'facets': facets,
16c42787   tangwang   feat: implement r...
540
541
              'enable_translation': enable_translation,
              'enable_embedding': enable_embedding,
ff32d894   tangwang   rerank
542
              'enable_rerank': do_rerank,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
543
              'min_score': min_score,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
544
545
              'sort_by': sort_by,
              'sort_order': sort_order
16c42787   tangwang   feat: implement r...
546
          }
be52af70   tangwang   first commit
547
  
16c42787   tangwang   feat: implement r...
548
549
550
          context.metadata['feature_flags'] = {
              'translation_enabled': enable_translation,
              'embedding_enabled': enable_embedding,
ff32d894   tangwang   rerank
551
              'rerank_enabled': do_rerank
16c42787   tangwang   feat: implement r...
552
          }
be52af70   tangwang   first commit
553
554
  
          # Step 1: Parse query
16c42787   tangwang   feat: implement r...
555
556
557
558
          context.start_stage(RequestContextStage.QUERY_PARSING)
          try:
              parsed_query = self.query_parser.parse(
                  query,
345d960b   tangwang   1. 删除全局 enable_tr...
559
                  tenant_id=tenant_id,
16c42787   tangwang   feat: implement r...
560
561
562
563
564
565
                  generate_vector=enable_embedding,
                  context=context
              )
              # Store query analysis results in context
              context.store_query_analysis(
                  original_query=parsed_query.original_query,
3a5fda00   tangwang   1. ES字段 skus的 ima...
566
                  query_normalized=parsed_query.query_normalized,
16c42787   tangwang   feat: implement r...
567
568
569
570
571
                  rewritten_query=parsed_query.rewritten_query,
                  detected_language=parsed_query.detected_language,
                  translations=parsed_query.translations,
                  query_vector=parsed_query.query_vector.tolist() if parsed_query.query_vector is not None else None,
                  domain=parsed_query.domain,
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
572
                  is_simple_query=True
16c42787   tangwang   feat: implement r...
573
              )
be52af70   tangwang   first commit
574
  
16c42787   tangwang   feat: implement r...
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
              context.logger.info(
                  f"查询解析完成 | 原查询: '{parsed_query.original_query}' | "
                  f"重写后: '{parsed_query.rewritten_query}' | "
                  f"语言: {parsed_query.detected_language} | "
                  f"域: {parsed_query.domain} | "
                  f"向量: {'是' if parsed_query.query_vector is not None else '否'}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"查询解析失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
              context.end_stage(RequestContextStage.QUERY_PARSING)
  
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
593
          # Step 2: Query building
16c42787   tangwang   feat: implement r...
594
595
          context.start_stage(RequestContextStage.QUERY_BUILDING)
          try:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
596
597
              # Generate tenant-specific index name
              index_name = get_tenant_index_name(tenant_id)
2739b281   tangwang   多语言索引调整
598
              # index_name = "search_products"
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
599
600
              
              # No longer need to add tenant_id to filters since each tenant has its own index
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
601
  
f0d020c3   tangwang   多语言查询改为只支持中英文两种,f...
602
              es_query = self.query_builder.build_query(
3a5fda00   tangwang   1. ES字段 skus的 ima...
603
                  query_text=parsed_query.rewritten_query or parsed_query.query_normalized,
16c42787   tangwang   feat: implement r...
604
                  query_vector=parsed_query.query_vector if enable_embedding else None,
16c42787   tangwang   feat: implement r...
605
                  filters=filters,
6aa246be   tangwang   问题:Pydantic 应该能自动...
606
                  range_filters=range_filters,
c581becd   tangwang   feat: 实现 Multi-Se...
607
                  facet_configs=facets,
506c39b7   tangwang   feat(search): 统一重...
608
609
                  size=es_fetch_size,
                  from_=es_fetch_from,
16c42787   tangwang   feat: implement r...
610
                  enable_knn=enable_embedding and parsed_query.query_vector is not None,
7bc756c5   tangwang   优化 ES 查询构建
611
612
                  min_score=min_score,
                  parsed_query=parsed_query
16c42787   tangwang   feat: implement r...
613
              )
be52af70   tangwang   first commit
614
  
6aa246be   tangwang   问题:Pydantic 应该能自动...
615
616
617
618
619
620
621
              # Add facets for faceted search
              if facets:
                  facet_aggs = self.query_builder.build_facets(facets)
                  if facet_aggs:
                      if "aggs" not in es_query:
                          es_query["aggs"] = {}
                      es_query["aggs"].update(facet_aggs)
16c42787   tangwang   feat: implement r...
622
  
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
623
624
625
              # Add sorting if specified
              if sort_by:
                  es_query = self.query_builder.add_sorting(es_query, sort_by, sort_order)
a47416ec   tangwang   把融合逻辑改成乘法公式,并把 ES...
626
                  es_query["track_scores"] = True
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
627
  
5f7d7f09   tangwang   性能测试报告.md
628
629
630
631
632
633
634
635
636
637
638
              # Keep requested response _source semantics for the final response fill.
              response_source_spec = es_query.get("_source")
  
              # In rerank window, first pass only fetches minimal fields required by rerank template.
              es_query_for_fetch = es_query
              rerank_prefetch_source = None
              if in_rerank_window:
                  rerank_prefetch_source = self._resolve_rerank_source_filter(effective_doc_template)
                  es_query_for_fetch = dict(es_query)
                  es_query_for_fetch["_source"] = rerank_prefetch_source
  
16c42787   tangwang   feat: implement r...
639
              # Extract size and from from body for ES client parameters
5f7d7f09   tangwang   性能测试报告.md
640
              body_for_es = {k: v for k, v in es_query_for_fetch.items() if k not in ['size', 'from']}
16c42787   tangwang   feat: implement r...
641
642
643
  
              # Store ES query in context
              context.store_intermediate_result('es_query', es_query)
5f7d7f09   tangwang   性能测试报告.md
644
645
              if in_rerank_window and rerank_prefetch_source is not None:
                  context.store_intermediate_result('es_query_rerank_prefetch_source', rerank_prefetch_source)
16c42787   tangwang   feat: implement r...
646
647
              context.store_intermediate_result('es_body_for_search', body_for_es)
  
28e57bb1   tangwang   日志体系优化
648
              # Serialize ES query to compute a compact size + stable digest for correlation
5f7d7f09   tangwang   性能测试报告.md
649
              es_query_compact = json.dumps(es_query_for_fetch, ensure_ascii=False, separators=(",", ":"))
28e57bb1   tangwang   日志体系优化
650
651
652
              es_query_digest = hashlib.sha256(es_query_compact.encode("utf-8")).hexdigest()[:16]
              knn_enabled = bool(enable_embedding and parsed_query.query_vector is not None)
              vector_dims = int(len(parsed_query.query_vector)) if parsed_query.query_vector is not None else 0
99bea633   tangwang   add logs
653
  
16c42787   tangwang   feat: implement r...
654
              context.logger.info(
5f7d7f09   tangwang   性能测试报告.md
655
                  "ES query built | size: %s chars | digest: %s | KNN: %s | vector_dims: %s | facets: %s | rerank_prefetch_source: %s",
28e57bb1   tangwang   日志体系优化
656
657
658
659
660
                  len(es_query_compact),
                  es_query_digest,
                  "yes" if knn_enabled else "no",
                  vector_dims,
                  "yes" if facets else "no",
5f7d7f09   tangwang   性能测试报告.md
661
                  rerank_prefetch_source,
16c42787   tangwang   feat: implement r...
662
663
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
28e57bb1   tangwang   日志体系优化
664
665
666
667
668
669
670
671
672
673
              _log_backend_verbose({
                  "event": "es_query_built",
                  "reqid": context.reqid,
                  "uid": context.uid,
                  "tenant_id": tenant_id,
                  "size_chars": len(es_query_compact),
                  "sha256_16": es_query_digest,
                  "knn_enabled": knn_enabled,
                  "vector_dims": vector_dims,
                  "has_facets": bool(facets),
5f7d7f09   tangwang   性能测试报告.md
674
                  "query": es_query_for_fetch,
28e57bb1   tangwang   日志体系优化
675
              })
16c42787   tangwang   feat: implement r...
676
677
678
679
680
681
682
683
684
685
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"ES查询构建失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
              context.end_stage(RequestContextStage.QUERY_BUILDING)
  
a99e62ba   tangwang   记录各阶段耗时
686
687
          # Step 4: Elasticsearch search (primary recall)
          context.start_stage(RequestContextStage.ELASTICSEARCH_SEARCH_PRIMARY)
16c42787   tangwang   feat: implement r...
688
          try:
506c39b7   tangwang   feat(search): 统一重...
689
              # Use tenant-specific index name(开启重排且在窗口内时已用 es_fetch_size/es_fetch_from)
16c42787   tangwang   feat: implement r...
690
              es_response = self.es_client.search(
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
691
                  index_name=index_name,
16c42787   tangwang   feat: implement r...
692
                  body=body_for_es,
506c39b7   tangwang   feat(search): 统一重...
693
                  size=es_fetch_size,
a47416ec   tangwang   把融合逻辑改成乘法公式,并把 ES...
694
695
                  from_=es_fetch_from,
                  include_named_queries_score=bool(do_rerank and in_rerank_window),
be52af70   tangwang   first commit
696
697
              )
  
16c42787   tangwang   feat: implement r...
698
699
              # Store ES response in context
              context.store_intermediate_result('es_response', es_response)
be52af70   tangwang   first commit
700
  
16c42787   tangwang   feat: implement r...
701
702
703
704
705
              # Extract timing from ES response
              es_took = es_response.get('took', 0)
              context.logger.info(
                  f"ES搜索完成 | 耗时: {es_took}ms | "
                  f"命中数: {es_response.get('hits', {}).get('total', {}).get('value', 0)} | "
25d3e81d   tangwang   fix指定sort项时候的bug
706
                  f"最高分: {(es_response.get('hits', {}).get('max_score') or 0):.3f}",
16c42787   tangwang   feat: implement r...
707
708
709
710
711
712
713
714
715
716
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"ES搜索执行失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
a99e62ba   tangwang   记录各阶段耗时
717
              context.end_stage(RequestContextStage.ELASTICSEARCH_SEARCH_PRIMARY)
16c42787   tangwang   feat: implement r...
718
  
506c39b7   tangwang   feat(search): 统一重...
719
          # Optional Step 4.5: AI reranking(仅当请求范围在重排窗口内时执行)
ff32d894   tangwang   rerank
720
          if do_rerank and in_rerank_window:
506c39b7   tangwang   feat(search): 统一重...
721
722
723
724
725
              context.start_stage(RequestContextStage.RERANKING)
              try:
                  from .rerank_client import run_rerank
  
                  rerank_query = parsed_query.original_query if parsed_query else query
506c39b7   tangwang   feat(search): 统一重...
726
727
728
729
                  es_response, rerank_meta, fused_debug = run_rerank(
                      query=rerank_query,
                      es_response=es_response,
                      language=language,
506c39b7   tangwang   feat(search): 统一重...
730
731
732
                      timeout_sec=rc.timeout_sec,
                      weight_es=rc.weight_es,
                      weight_ai=rc.weight_ai,
ff32d894   tangwang   rerank
733
734
                      rerank_query_template=effective_query_template,
                      rerank_doc_template=effective_doc_template,
d31c7f65   tangwang   补充云服务reranker
735
                      top_n=(from_ + size),
506c39b7   tangwang   feat(search): 统一重...
736
737
738
                  )
  
                  if rerank_meta is not None:
42e3aea6   tangwang   tidy
739
740
                      from config.services_config import get_rerank_service_url
                      rerank_url = get_rerank_service_url()
506c39b7   tangwang   feat(search): 统一重...
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
                      context.metadata.setdefault("rerank_info", {})
                      context.metadata["rerank_info"].update({
                          "service_url": rerank_url,
                          "docs": len(es_response.get("hits", {}).get("hits") or []),
                          "meta": rerank_meta,
                      })
                      context.store_intermediate_result("rerank_scores", fused_debug)
                      context.logger.info(
                          f"重排完成 | docs={len(fused_debug)} | meta={rerank_meta}",
                          extra={'reqid': context.reqid, 'uid': context.uid}
                      )
              except Exception as e:
                  context.add_warning(f"Rerank failed: {e}")
                  context.logger.warning(
                      f"调用重排服务失败 | error: {e}",
                      extra={'reqid': context.reqid, 'uid': context.uid},
                      exc_info=True,
                  )
              finally:
                  context.end_stage(RequestContextStage.RERANKING)
  
          # 当本次请求在重排窗口内时:已从 ES 取了 rerank_window 条并可能已重排,需按请求的 from/size 做分页切片
          if in_rerank_window:
              hits = es_response.get("hits", {}).get("hits") or []
              sliced = hits[from_ : from_ + size]
              es_response.setdefault("hits", {})["hits"] = sliced
              if sliced:
af827ce9   tangwang   rerank
768
769
770
771
772
                  # 对于启用重排的结果,优先使用 _fused_score 计算 max_score;否则退回原始 _score
                  slice_max = max(
                      (h.get("_fused_score", h.get("_score", 0.0)) for h in sliced),
                      default=0.0,
                  )
506c39b7   tangwang   feat(search): 统一重...
773
774
775
776
777
778
                  try:
                      es_response["hits"]["max_score"] = float(slice_max)
                  except (TypeError, ValueError):
                      es_response["hits"]["max_score"] = 0.0
              else:
                  es_response["hits"]["max_score"] = 0.0
5f7d7f09   tangwang   性能测试报告.md
779
780
781
782
783
784
785
786
787
788
789
  
              # Page fill: fetch detailed fields only for final page hits.
              if sliced:
                  if response_source_spec is False:
                      for hit in sliced:
                          hit.pop("_source", None)
                      context.logger.info(
                          "分页详情回填跳过 | 原查询 _source=false",
                          extra={'reqid': context.reqid, 'uid': context.uid}
                      )
                  else:
a99e62ba   tangwang   记录各阶段耗时
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
                      context.start_stage(RequestContextStage.ELASTICSEARCH_PAGE_FILL)
                      try:
                          page_ids = [str(h.get("_id")) for h in sliced if h.get("_id") is not None]
                          details_by_id, fill_took = self._fetch_hits_by_ids(
                              index_name=index_name,
                              doc_ids=page_ids,
                              source_spec=response_source_spec,
                          )
                          filled = 0
                          for hit in sliced:
                              hid = hit.get("_id")
                              if hid is None:
                                  continue
                              detail_hit = details_by_id.get(str(hid))
                              if detail_hit is None:
                                  continue
                              if "_source" in detail_hit:
                                  hit["_source"] = detail_hit.get("_source") or {}
                                  filled += 1
                          if fill_took:
                              es_response["took"] = int((es_response.get("took", 0) or 0) + fill_took)
                          context.logger.info(
                              f"分页详情回填 | ids={len(page_ids)} | filled={filled} | took={fill_took}ms",
                              extra={'reqid': context.reqid, 'uid': context.uid}
                          )
                      finally:
                          context.end_stage(RequestContextStage.ELASTICSEARCH_PAGE_FILL)
5f7d7f09   tangwang   性能测试报告.md
817
  
506c39b7   tangwang   feat(search): 统一重...
818
819
820
821
822
              context.logger.info(
                  f"重排分页切片 | from={from_}, size={size}, 返回={len(sliced)}条",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
  
16c42787   tangwang   feat: implement r...
823
824
825
          # Step 5: Result processing
          context.start_stage(RequestContextStage.RESULT_PROCESSING)
          try:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
826
827
              # Extract ES hits
              es_hits = []
16c42787   tangwang   feat: implement r...
828
              if 'hits' in es_response and 'hits' in es_response['hits']:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
829
                  es_hits = es_response['hits']['hits']
16c42787   tangwang   feat: implement r...
830
831
832
833
834
835
              # Extract total and max_score
              total = es_response.get('hits', {}).get('total', {})
              if isinstance(total, dict):
                  total_value = total.get('value', 0)
              else:
                  total_value = total
506c39b7   tangwang   feat(search): 统一重...
836
              # max_score 会在启用 AI 搜索时被更新为融合分数的最大值
25d3e81d   tangwang   fix指定sort项时候的bug
837
              max_score = es_response.get('hits', {}).get('max_score') or 0.0
be52af70   tangwang   first commit
838
  
af827ce9   tangwang   rerank
839
840
841
842
843
844
845
846
847
848
849
850
              # 从上下文中取出重排调试信息(若有)
              rerank_debug_raw = context.get_intermediate_result('rerank_scores', None)
              rerank_debug_by_doc: Dict[str, Dict[str, Any]] = {}
              if isinstance(rerank_debug_raw, list):
                  for item in rerank_debug_raw:
                      if not isinstance(item, dict):
                          continue
                      doc_id = item.get("doc_id")
                      if doc_id is None:
                          continue
                      rerank_debug_by_doc[str(doc_id)] = item
  
deccd68a   tangwang   Added the SKU pre...
851
852
              self._apply_sku_sorting_for_page_hits(es_hits, parsed_query, context=context)
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
853
              # Format results using ResultFormatter
577ec972   tangwang   返回给前端的字段、格式适配。主要包...
854
855
856
              formatted_results = ResultFormatter.format_search_results(
                  es_hits,
                  max_score,
ca91352a   tangwang   更新文档
857
858
                  language=language,
                  sku_filter_dimension=sku_filter_dimension
577ec972   tangwang   返回给前端的字段、格式适配。主要包...
859
              )
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
860
  
985752f5   tangwang   1. 前端调试功能
861
862
863
864
865
              # Build per-result debug info (per SPU) when debug mode is enabled
              per_result_debug = []
              if debug and es_hits and formatted_results:
                  for hit, spu in zip(es_hits, formatted_results):
                      source = hit.get("_source", {}) or {}
af827ce9   tangwang   rerank
866
867
868
869
870
                      doc_id = hit.get("_id")
                      rerank_debug = None
                      if doc_id is not None:
                          rerank_debug = rerank_debug_by_doc.get(str(doc_id))
  
985752f5   tangwang   1. 前端调试功能
871
872
873
874
875
876
877
878
879
880
881
882
883
884
                      raw_score = hit.get("_score")
                      try:
                          es_score = float(raw_score) if raw_score is not None else 0.0
                      except (TypeError, ValueError):
                          es_score = 0.0
                      try:
                          normalized = float(es_score) / float(max_score) if max_score else None
                      except (TypeError, ValueError, ZeroDivisionError):
                          normalized = None
  
                      title_multilingual = source.get("title") if isinstance(source.get("title"), dict) else None
                      brief_multilingual = source.get("brief") if isinstance(source.get("brief"), dict) else None
                      vendor_multilingual = source.get("vendor") if isinstance(source.get("vendor"), dict) else None
  
af827ce9   tangwang   rerank
885
886
887
888
889
890
891
892
893
894
895
896
897
                      debug_entry: Dict[str, Any] = {
                          "spu_id": spu.spu_id,
                          "es_score": es_score,
                          "es_score_normalized": normalized,
                          "title_multilingual": title_multilingual,
                          "brief_multilingual": brief_multilingual,
                          "vendor_multilingual": vendor_multilingual,
                      }
  
                      # 若存在重排调试信息,则补充 doc 级别的融合分数信息
                      if rerank_debug:
                          debug_entry["doc_id"] = rerank_debug.get("doc_id")
                          # 与 rerank_client 中字段保持一致,便于前端直接使用
af827ce9   tangwang   rerank
898
                          debug_entry["rerank_score"] = rerank_debug.get("rerank_score")
a8261ece   tangwang   检索效果优化
899
                          debug_entry["text_score"] = rerank_debug.get("text_score")
c90f80ed   tangwang   相关性优化
900
901
902
903
904
                          debug_entry["text_source_score"] = rerank_debug.get("text_source_score")
                          debug_entry["text_translation_score"] = rerank_debug.get("text_translation_score")
                          debug_entry["text_fallback_score"] = rerank_debug.get("text_fallback_score")
                          debug_entry["text_primary_score"] = rerank_debug.get("text_primary_score")
                          debug_entry["text_support_score"] = rerank_debug.get("text_support_score")
a8261ece   tangwang   检索效果优化
905
                          debug_entry["knn_score"] = rerank_debug.get("knn_score")
af827ce9   tangwang   rerank
906
                          debug_entry["fused_score"] = rerank_debug.get("fused_score")
a8261ece   tangwang   检索效果优化
907
                          debug_entry["matched_queries"] = rerank_debug.get("matched_queries")
af827ce9   tangwang   rerank
908
909
  
                      per_result_debug.append(debug_entry)
985752f5   tangwang   1. 前端调试功能
910
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
911
912
913
914
915
              # Format facets
              standardized_facets = None
              if facets:
                  standardized_facets = ResultFormatter.format_facets(
                      es_response.get('aggregations', {}),
c581becd   tangwang   feat: 实现 Multi-Se...
916
917
                      facets,
                      filters
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
918
919
920
921
922
923
                  )
  
              # Generate suggestions and related searches
              query_text = parsed_query.original_query if parsed_query else query
              suggestions = ResultFormatter.generate_suggestions(query_text, formatted_results)
              related_searches = ResultFormatter.generate_related_searches(query_text, formatted_results)
be52af70   tangwang   first commit
924
  
16c42787   tangwang   feat: implement r...
925
              context.logger.info(
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
926
                  f"结果处理完成 | 返回: {len(formatted_results)}条 | 总计: {total_value}条",
16c42787   tangwang   feat: implement r...
927
928
929
930
931
932
933
934
935
936
937
938
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
  
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"结果处理失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
              context.end_stage(RequestContextStage.RESULT_PROCESSING)
be52af70   tangwang   first commit
939
  
16c42787   tangwang   feat: implement r...
940
941
942
          # End total timing and build result
          total_duration = context.end_stage(RequestContextStage.TOTAL)
          context.performance_metrics.total_duration = total_duration
be52af70   tangwang   first commit
943
  
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
944
945
946
947
948
949
          # Collect debug information if requested
          debug_info = None
          if debug:
              debug_info = {
                  "query_analysis": {
                      "original_query": context.query_analysis.original_query,
3a5fda00   tangwang   1. ES字段 skus的 ima...
950
                      "query_normalized": context.query_analysis.query_normalized,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
951
952
953
                      "rewritten_query": context.query_analysis.rewritten_query,
                      "detected_language": context.query_analysis.detected_language,
                      "translations": context.query_analysis.translations,
a8261ece   tangwang   检索效果优化
954
955
956
                      "query_text_by_lang": context.get_intermediate_result("query_text_by_lang", {}),
                      "search_langs": context.get_intermediate_result("search_langs", []),
                      "supplemental_search_langs": context.get_intermediate_result("supplemental_search_langs", []),
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
957
958
                      "has_vector": context.query_analysis.query_vector is not None,
                      "is_simple_query": context.query_analysis.is_simple_query,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
                      "domain": context.query_analysis.domain
                  },
                  "es_query": context.get_intermediate_result('es_query', {}),
                  "es_response": {
                      "took_ms": es_response.get('took', 0),
                      "total_hits": total_value,
                      "max_score": max_score,
                      "shards": es_response.get('_shards', {})
                  },
                  "feature_flags": context.metadata.get('feature_flags', {}),
                  "stage_timings": {
                      k: round(v, 2) for k, v in context.performance_metrics.stage_timings.items()
                  },
                  "search_params": context.metadata.get('search_params', {})
              }
985752f5   tangwang   1. 前端调试功能
974
975
              if per_result_debug:
                  debug_info["per_result"] = per_result_debug
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
976
  
be52af70   tangwang   first commit
977
978
          # Build result
          result = SearchResult(
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
979
              results=formatted_results,
be52af70   tangwang   first commit
980
981
              total=total_value,
              max_score=max_score,
16c42787   tangwang   feat: implement r...
982
              took_ms=int(total_duration),
6aa246be   tangwang   问题:Pydantic 应该能自动...
983
              facets=standardized_facets,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
984
              query_info=parsed_query.to_dict(),
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
985
986
              suggestions=suggestions,
              related_searches=related_searches,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
987
              debug_info=debug_info
be52af70   tangwang   first commit
988
989
          )
  
16c42787   tangwang   feat: implement r...
990
991
          # Log complete performance summary
          context.log_performance_summary()
be52af70   tangwang   first commit
992
993
994
995
996
997
  
          return result
  
      def search_by_image(
          self,
          image_url: str,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
998
          tenant_id: str,
be52af70   tangwang   first commit
999
          size: int = 10,
6aa246be   tangwang   问题:Pydantic 应该能自动...
1000
1001
          filters: Optional[Dict[str, Any]] = None,
          range_filters: Optional[Dict[str, Any]] = None
be52af70   tangwang   first commit
1002
1003
      ) -> SearchResult:
          """
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1004
          Search by image similarity (外部友好格式).
be52af70   tangwang   first commit
1005
1006
1007
  
          Args:
              image_url: URL of query image
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1008
              tenant_id: Tenant ID (required for filtering)
be52af70   tangwang   first commit
1009
              size: Number of results
6aa246be   tangwang   问题:Pydantic 应该能自动...
1010
1011
              filters: Exact match filters
              range_filters: Range filters for numeric fields
be52af70   tangwang   first commit
1012
1013
  
          Returns:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1014
              SearchResult object with formatted results
be52af70   tangwang   first commit
1015
1016
1017
1018
1019
          """
          if not self.image_embedding_field:
              raise ValueError("Image embedding field not configured")
  
          # Generate image embedding
26b910bd   tangwang   refactor service ...
1020
1021
          if self.image_encoder is None:
              raise RuntimeError("Image encoder is not initialized at startup")
b754fd41   tangwang   图片向量化支持优先级参数
1022
          image_vector = self.image_encoder.encode_image_from_url(image_url, priority=1)
be52af70   tangwang   first commit
1023
1024
1025
1026
  
          if image_vector is None:
              raise ValueError(f"Failed to encode image: {image_url}")
  
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1027
1028
1029
1030
          # Generate tenant-specific index name
          index_name = get_tenant_index_name(tenant_id)
          
          # No longer need to add tenant_id to filters since each tenant has its own index
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1031
  
be52af70   tangwang   first commit
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
          # Build KNN query
          es_query = {
              "size": size,
              "knn": {
                  "field": self.image_embedding_field,
                  "query_vector": image_vector.tolist(),
                  "k": size,
                  "num_candidates": size * 10
              }
          }
  
26b910bd   tangwang   refactor service ...
1043
1044
          # Apply source filtering semantics (None / [] / list)
          self._apply_source_filter(es_query)
13377199   tangwang   接口优化
1045
  
6aa246be   tangwang   问题:Pydantic 应该能自动...
1046
1047
1048
          if filters or range_filters:
              filter_clauses = self.query_builder._build_filters(filters, range_filters)
              if filter_clauses:
7fbca0d7   tangwang   启动脚本优化
1049
1050
1051
1052
1053
1054
1055
                  if len(filter_clauses) == 1:
                      es_query["knn"]["filter"] = filter_clauses[0]
                  else:
                      es_query["knn"]["filter"] = {
                          "bool": {
                              "filter": filter_clauses
                          }
6aa246be   tangwang   问题:Pydantic 应该能自动...
1056
                      }
be52af70   tangwang   first commit
1057
1058
1059
  
          # Execute search
          es_response = self.es_client.search(
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1060
              index_name=index_name,
be52af70   tangwang   first commit
1061
1062
1063
1064
              body=es_query,
              size=size
          )
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1065
1066
          # Extract ES hits
          es_hits = []
be52af70   tangwang   first commit
1067
          if 'hits' in es_response and 'hits' in es_response['hits']:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1068
              es_hits = es_response['hits']['hits']
be52af70   tangwang   first commit
1069
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1070
          # Extract total and max_score
be52af70   tangwang   first commit
1071
1072
1073
1074
1075
1076
          total = es_response.get('hits', {}).get('total', {})
          if isinstance(total, dict):
              total_value = total.get('value', 0)
          else:
              total_value = total
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1077
1078
1079
          max_score = es_response.get('hits', {}).get('max_score') or 0.0
  
          # Format results using ResultFormatter
ca91352a   tangwang   更新文档
1080
1081
1082
          formatted_results = ResultFormatter.format_search_results(
              es_hits, 
              max_score,
2739b281   tangwang   多语言索引调整
1083
              language="en",  # Default language for image search
ca91352a   tangwang   更新文档
1084
1085
              sku_filter_dimension=None  # Image search doesn't support SKU filtering
          )
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1086
  
be52af70   tangwang   first commit
1087
          return SearchResult(
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1088
              results=formatted_results,
be52af70   tangwang   first commit
1089
              total=total_value,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1090
              max_score=max_score,
be52af70   tangwang   first commit
1091
              took_ms=es_response.get('took', 0),
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1092
1093
1094
1095
              facets=None,
              query_info={'image_url': image_url, 'search_type': 'image_similarity'},
              suggestions=[],
              related_searches=[]
be52af70   tangwang   first commit
1096
1097
          )
  
b926f678   tangwang   多语言查询
1098
1099
      def get_domain_summary(self) -> Dict[str, Any]:
          """
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
1100
          Get summary of dynamic text retrieval configuration.
b926f678   tangwang   多语言查询
1101
1102
  
          Returns:
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
1103
              Dictionary with language-aware field information
b926f678   tangwang   多语言查询
1104
          """
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
1105
1106
1107
1108
1109
1110
1111
          return {
              "mode": "dynamic_language_fields",
              "multilingual_fields": self.config.query_config.multilingual_fields,
              "shared_fields": self.config.query_config.shared_fields,
              "core_multilingual_fields": self.config.query_config.core_multilingual_fields,
              "field_boosts": self.config.field_boosts,
          }
b926f678   tangwang   多语言查询
1112
  
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1113
      def get_document(self, tenant_id: str, doc_id: str) -> Optional[Dict[str, Any]]:
be52af70   tangwang   first commit
1114
1115
1116
1117
          """
          Get single document by ID.
  
          Args:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1118
              tenant_id: Tenant ID (required to determine which index to query)
be52af70   tangwang   first commit
1119
1120
1121
1122
1123
1124
              doc_id: Document ID
  
          Returns:
              Document or None if not found
          """
          try:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1125
              index_name = get_tenant_index_name(tenant_id)
be52af70   tangwang   first commit
1126
              response = self.es_client.client.get(
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1127
                  index=index_name,
be52af70   tangwang   first commit
1128
1129
1130
1131
                  id=doc_id
              )
              return response.get('_source')
          except Exception as e:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1132
              logger.error(f"Failed to get document {doc_id} from tenant {tenant_id}: {e}", exc_info=True)
be52af70   tangwang   first commit
1133
              return None
6aa246be   tangwang   问题:Pydantic 应该能自动...
1134
1135
1136
1137
  
      def _standardize_facets(
          self,
          es_aggregations: Dict[str, Any],
ff5325fa   tangwang   修复:直接在 Searcher 层...
1138
          facet_configs: Optional[List[Union[str, Any]]],
6aa246be   tangwang   问题:Pydantic 应该能自动...
1139
1140
1141
1142
1143
1144
1145
          current_filters: Optional[Dict[str, Any]]
      ) -> Optional[List[FacetResult]]:
          """
           ES 聚合结果转换为标准化的分面格式(返回 Pydantic 模型)。
          
          Args:
              es_aggregations: ES 原始聚合结果
ff5325fa   tangwang   修复:直接在 Searcher 层...
1146
              facet_configs: 分面配置列表(str  FacetConfig
6aa246be   tangwang   问题:Pydantic 应该能自动...
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
              current_filters: 当前应用的过滤器
          
          Returns:
              标准化的分面结果列表(FacetResult 对象)
          """
          if not es_aggregations or not facet_configs:
              return None
          
          standardized_facets: List[FacetResult] = []
          
          for config in facet_configs:
              # 解析配置
              if isinstance(config, str):
                  field = config
                  facet_type = "terms"
              else:
ff5325fa   tangwang   修复:直接在 Searcher 层...
1163
1164
1165
                  # FacetConfig 对象
                  field = config.field
                  facet_type = config.type
6aa246be   tangwang   问题:Pydantic 应该能自动...
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
              
              agg_name = f"{field}_facet"
              
              if agg_name not in es_aggregations:
                  continue
              
              agg_result = es_aggregations[agg_name]
              
              # 获取当前字段的选中值
              selected_values = set()
              if current_filters and field in current_filters:
                  filter_value = current_filters[field]
                  if isinstance(filter_value, list):
                      selected_values = set(filter_value)
                  else:
                      selected_values = {filter_value}
              
              # 转换 buckets 为 FacetValue 对象
              facet_values: List[FacetValue] = []
              if 'buckets' in agg_result:
                  for bucket in agg_result['buckets']:
                      value = bucket.get('key')
                      count = bucket.get('doc_count', 0)
                      
                      facet_values.append(FacetValue(
                          value=value,
                          label=str(value),
                          count=count,
                          selected=value in selected_values
                      ))
              
              # 构建 FacetResult 对象
              facet_result = FacetResult(
                  field=field,
e7ad2b4a   tangwang   测试页面分页配置
1200
                  label=field,
6aa246be   tangwang   问题:Pydantic 应该能自动...
1201
1202
1203
1204
1205
1206
1207
                  type=facet_type,
                  values=facet_values
              )
              
              standardized_facets.append(facet_result)
          
          return standardized_facets if standardized_facets else None