Blame view

search/searcher.py 50.1 KB
be52af70   tangwang   first commit
1
2
3
  """
  Main Searcher module - executes search queries against Elasticsearch.
  
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
4
  Handles query parsing, ranking, and result formatting.
be52af70   tangwang   first commit
5
6
  """
  
deccd68a   tangwang   Added the SKU pre...
7
  from typing import Dict, Any, List, Optional, Union, Tuple
d1d356f8   tangwang   脚本优化
8
  import os
99bea633   tangwang   add logs
9
  import time, json
325eec03   tangwang   1. 日志、配置基础设施,使用优化
10
  import logging
28e57bb1   tangwang   日志体系优化
11
  import hashlib
5f7d7f09   tangwang   性能测试报告.md
12
  from string import Formatter
deccd68a   tangwang   Added the SKU pre...
13
  import numpy as np
be52af70   tangwang   first commit
14
  
be52af70   tangwang   first commit
15
16
  from utils.es_client import ESClient
  from query import QueryParser, ParsedQuery
07cf5a93   tangwang   START_EMBEDDING=...
17
  from embeddings.image_encoder import CLIPImageEncoder
be52af70   tangwang   first commit
18
  from .es_query_builder import ESQueryBuilder
9f96d6f3   tangwang   短query不用语义搜索
19
  from config import SearchConfig
345d960b   tangwang   1. 删除全局 enable_tr...
20
  from config.tenant_config_loader import get_tenant_config_loader
ed948666   tangwang   tidy
21
  from context.request_context import RequestContext, RequestContextStage
13320ac6   tangwang   分面接口修改:
22
  from api.models import FacetResult, FacetValue, FacetConfig
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
23
  from api.result_formatter import ResultFormatter
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
24
  from indexer.mapping_generator import get_tenant_index_name
be52af70   tangwang   first commit
25
  
325eec03   tangwang   1. 日志、配置基础设施,使用优化
26
  logger = logging.getLogger(__name__)
28e57bb1   tangwang   日志体系优化
27
28
29
30
31
32
33
34
35
  backend_verbose_logger = logging.getLogger("backend.verbose")
  
  
  def _log_backend_verbose(payload: Dict[str, Any]) -> None:
      if not backend_verbose_logger.handlers:
          return
      backend_verbose_logger.info(
          json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
      )
325eec03   tangwang   1. 日志、配置基础设施,使用优化
36
  
be52af70   tangwang   first commit
37
38
  
  class SearchResult:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
39
      """Container for search results (外部友好格式)."""
be52af70   tangwang   first commit
40
41
42
  
      def __init__(
          self,
cadc77b6   tangwang   索引字段名、变量名、API数据结构...
43
          results: List[Any],  # List[SpuResult]
be52af70   tangwang   first commit
44
45
46
          total: int,
          max_score: float,
          took_ms: int,
6aa246be   tangwang   问题:Pydantic 应该能自动...
47
          facets: Optional[List[FacetResult]] = None,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
48
          query_info: Optional[Dict[str, Any]] = None,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
49
50
          suggestions: Optional[List[str]] = None,
          related_searches: Optional[List[str]] = None,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
51
          debug_info: Optional[Dict[str, Any]] = None
be52af70   tangwang   first commit
52
      ):
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
53
          self.results = results
be52af70   tangwang   first commit
54
55
56
          self.total = total
          self.max_score = max_score
          self.took_ms = took_ms
6aa246be   tangwang   问题:Pydantic 应该能自动...
57
          self.facets = facets
be52af70   tangwang   first commit
58
          self.query_info = query_info or {}
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
59
60
          self.suggestions = suggestions or []
          self.related_searches = related_searches or []
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
61
          self.debug_info = debug_info
43f1139f   tangwang   refactor: ES查询结构重...
62
  
be52af70   tangwang   first commit
63
64
      def to_dict(self) -> Dict[str, Any]:
          """Convert to dictionary representation."""
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
65
          result = {
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
66
              "results": [r.model_dump() if hasattr(r, 'model_dump') else r for r in self.results],
be52af70   tangwang   first commit
67
68
69
              "total": self.total,
              "max_score": self.max_score,
              "took_ms": self.took_ms,
6aa246be   tangwang   问题:Pydantic 应该能自动...
70
              "facets": [f.model_dump() for f in self.facets] if self.facets else None,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
71
72
73
              "query_info": self.query_info,
              "suggestions": self.suggestions,
              "related_searches": self.related_searches
be52af70   tangwang   first commit
74
          }
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
75
76
77
          if self.debug_info is not None:
              result["debug_info"] = self.debug_info
          return result
be52af70   tangwang   first commit
78
79
80
81
82
83
84
85
  
  
  class Searcher:
      """
      Main search engine class.
  
      Handles:
      - Query parsing and translation
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
86
      - Dynamic multi-language text recall planning
be52af70   tangwang   first commit
87
88
89
90
91
92
      - ES query building
      - Result ranking and formatting
      """
  
      def __init__(
          self,
be52af70   tangwang   first commit
93
          es_client: ESClient,
9f96d6f3   tangwang   短query不用语义搜索
94
          config: SearchConfig,
26b910bd   tangwang   refactor service ...
95
96
          query_parser: Optional[QueryParser] = None,
          image_encoder: Optional[CLIPImageEncoder] = None,
be52af70   tangwang   first commit
97
98
99
100
101
      ):
          """
          Initialize searcher.
  
          Args:
be52af70   tangwang   first commit
102
              es_client: Elasticsearch client
9f96d6f3   tangwang   短query不用语义搜索
103
              config: SearchConfig instance
be52af70   tangwang   first commit
104
              query_parser: Query parser (created if not provided)
26b910bd   tangwang   refactor service ...
105
              image_encoder: Optional pre-initialized image encoder
be52af70   tangwang   first commit
106
          """
be52af70   tangwang   first commit
107
          self.es_client = es_client
9f96d6f3   tangwang   短query不用语义搜索
108
          self.config = config
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
109
          # Index name is now generated dynamically per tenant, no longer stored here
9f96d6f3   tangwang   短query不用语义搜索
110
          self.query_parser = query_parser or QueryParser(config)
9f96d6f3   tangwang   短query不用语义搜索
111
          self.text_embedding_field = config.query_config.text_embedding_field or "title_embedding"
26b910bd   tangwang   refactor service ...
112
113
114
115
116
117
          self.image_embedding_field = config.query_config.image_embedding_field
          if self.image_embedding_field and image_encoder is None:
              self.image_encoder = CLIPImageEncoder()
          else:
              self.image_encoder = image_encoder
          self.source_fields = config.query_config.source_fields
be52af70   tangwang   first commit
118
  
f0d020c3   tangwang   多语言查询改为只支持中英文两种,f...
119
          # Query builder - simplified single-layer architecture
f0d020c3   tangwang   多语言查询改为只支持中英文两种,f...
120
          self.query_builder = ESQueryBuilder(
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
121
122
123
124
125
              match_fields=[],
              field_boosts=self.config.field_boosts,
              multilingual_fields=self.config.query_config.multilingual_fields,
              shared_fields=self.config.query_config.shared_fields,
              core_multilingual_fields=self.config.query_config.core_multilingual_fields,
be52af70   tangwang   first commit
126
              text_embedding_field=self.text_embedding_field,
13377199   tangwang   接口优化
127
              image_embedding_field=self.image_embedding_field,
9f96d6f3   tangwang   短query不用语义搜索
128
              source_fields=self.source_fields,
7bc756c5   tangwang   优化 ES 查询构建
129
              function_score_config=self.config.function_score,
70dab99f   tangwang   add logs
130
              default_language=self.config.query_config.default_language,
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
131
132
133
134
              knn_boost=self.config.query_config.knn_boost,
              base_minimum_should_match=self.config.query_config.base_minimum_should_match,
              translation_minimum_should_match=self.config.query_config.translation_minimum_should_match,
              translation_boost=self.config.query_config.translation_boost,
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
135
              tie_breaker_base_query=self.config.query_config.tie_breaker_base_query,
be52af70   tangwang   first commit
136
137
          )
  
26b910bd   tangwang   refactor service ...
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
      def _apply_source_filter(self, es_query: Dict[str, Any]) -> None:
          """
          Apply tri-state _source semantics:
          - None: do not set _source (return full source)
          - []: _source=false (return no source fields)
          - [..]: _source.includes=[..]
          """
          if self.source_fields is None:
              return
          if not isinstance(self.source_fields, list):
              raise ValueError("query_config.source_fields must be null or list[str]")
          if len(self.source_fields) == 0:
              es_query["_source"] = False
              return
          es_query["_source"] = {"includes": self.source_fields}
  
5f7d7f09   tangwang   性能测试报告.md
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
      def _resolve_rerank_source_filter(self, doc_template: str) -> Dict[str, Any]:
          """
          Build a lightweight _source filter for rerank prefetch.
  
          Only fetch fields required by rerank doc template to reduce ES payload.
          """
          field_map = {
              "title": "title",
              "brief": "brief",
              "vendor": "vendor",
              "description": "description",
              "category_path": "category_path",
          }
          includes: set[str] = set()
          template = str(doc_template or "{title}")
          for _, field_name, _, _ in Formatter().parse(template):
              if not field_name:
                  continue
              key = field_name.split(".", 1)[0].split("!", 1)[0].split(":", 1)[0]
              mapped = field_map.get(key)
              if mapped:
                  includes.add(mapped)
  
          # Fallback to title-only to keep rerank docs usable.
          if not includes:
              includes.add("title")
  
          return {"includes": sorted(includes)}
  
      def _fetch_hits_by_ids(
          self,
          index_name: str,
          doc_ids: List[str],
          source_spec: Optional[Any],
      ) -> tuple[Dict[str, Dict[str, Any]], int]:
          """
          Fetch page documents by IDs for final response fill.
  
          Returns:
              (hits_by_id, es_took_ms)
          """
          if not doc_ids:
              return {}, 0
  
          body: Dict[str, Any] = {
              "query": {
                  "ids": {
                      "values": doc_ids,
                  }
              }
          }
          if source_spec is not None:
              body["_source"] = source_spec
  
          resp = self.es_client.search(
              index_name=index_name,
              body=body,
              size=len(doc_ids),
              from_=0,
          )
          hits = resp.get("hits", {}).get("hits") or []
          hits_by_id: Dict[str, Dict[str, Any]] = {}
          for hit in hits:
              hid = hit.get("_id")
              if hid is None:
                  continue
              hits_by_id[str(hid)] = hit
          return hits_by_id, int(resp.get("took", 0) or 0)
  
deccd68a   tangwang   Added the SKU pre...
223
224
225
226
227
228
229
      @staticmethod
      def _normalize_sku_match_text(value: Optional[str]) -> str:
          """Normalize free text for lightweight SKU option matching."""
          if value is None:
              return ""
          return " ".join(str(value).strip().casefold().split())
  
a7cc9078   tangwang   sku排序
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
      @staticmethod
      def _sku_option1_embedding_key(
          sku: Dict[str, Any],
          spu_option1_name: Optional[Any] = None,
      ) -> Optional[str]:
          """
          Text sent to the embedding service for option1 must be "name:value"
          (option name from SKU row or SPU-level option1_name).
          """
          value_raw = sku.get("option1_value")
          if value_raw is None:
              return None
          value = str(value_raw).strip()
          if not value:
              return None
          name = sku.get("option1_name")
          if name is None or not str(name).strip():
              name = spu_option1_name
          name_str = str(name).strip() if name is not None and str(name).strip() else ""
          if name_str:
              value = f"{name_str}:{value}"
          return value.casefold()
  
deccd68a   tangwang   Added the SKU pre...
253
254
255
256
257
258
259
260
261
262
263
264
      def _build_sku_query_texts(self, parsed_query: ParsedQuery) -> List[str]:
          """Collect original and translated query texts for SKU option matching."""
          candidates: List[str] = []
          for text in (
              getattr(parsed_query, "original_query", None),
              getattr(parsed_query, "query_normalized", None),
              getattr(parsed_query, "rewritten_query", None),
          ):
              normalized = self._normalize_sku_match_text(text)
              if normalized:
                  candidates.append(normalized)
  
deccd68a   tangwang   Added the SKU pre...
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
          translations = getattr(parsed_query, "translations", {}) or {}
          if isinstance(translations, dict):
              for text in translations.values():
                  normalized = self._normalize_sku_match_text(text)
                  if normalized:
                      candidates.append(normalized)
  
          deduped: List[str] = []
          seen = set()
          for text in candidates:
              if text in seen:
                  continue
              seen.add(text)
              deduped.append(text)
          return deduped
  
      def _find_query_matching_sku_index(
          self,
          skus: List[Dict[str, Any]],
          query_texts: List[str],
a7cc9078   tangwang   sku排序
285
          spu_option1_name: Optional[Any] = None,
deccd68a   tangwang   Added the SKU pre...
286
      ) -> Optional[int]:
a7cc9078   tangwang   sku排序
287
          """Return the first SKU whose option1_value (or name:value) appears in query texts."""
deccd68a   tangwang   Added the SKU pre...
288
289
290
291
292
293
294
295
296
          if not skus or not query_texts:
              return None
  
          for index, sku in enumerate(skus):
              option1_value = self._normalize_sku_match_text(sku.get("option1_value"))
              if not option1_value:
                  continue
              if any(option1_value in query_text for query_text in query_texts):
                  return index
a7cc9078   tangwang   sku排序
297
298
299
300
301
302
303
              embed_key = self._sku_option1_embedding_key(sku, spu_option1_name)
              if embed_key and embed_key != option1_value:
                  composite_norm = self._normalize_sku_match_text(embed_key.replace(":", " "))
                  if any(composite_norm in query_text for query_text in query_texts):
                      return index
                  if any(embed_key.casefold() in query_text for query_text in query_texts):
                      return index
deccd68a   tangwang   Added the SKU pre...
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
          return None
  
      def _encode_query_vector_for_sku_matching(
          self,
          parsed_query: ParsedQuery,
          context: Optional[RequestContext] = None,
      ) -> Optional[np.ndarray]:
          """Best-effort fallback query embedding for final-page SKU matching."""
          query_text = (
              getattr(parsed_query, "rewritten_query", None)
              or getattr(parsed_query, "query_normalized", None)
              or getattr(parsed_query, "original_query", None)
          )
          if not query_text:
              return None
  
          text_encoder = getattr(self.query_parser, "text_encoder", None)
          if text_encoder is None:
              return None
  
          try:
              vectors = text_encoder.encode([query_text], priority=1)
          except Exception as exc:
              logger.warning("Failed to encode query vector for SKU matching: %s", exc, exc_info=True)
              if context is not None:
                  context.add_warning(f"SKU query embedding failed: {exc}")
              return None
  
          if vectors is None or len(vectors) == 0:
              return None
  
          vector = vectors[0]
          if vector is None:
              return None
          return np.asarray(vector, dtype=np.float32)
  
      def _select_sku_by_embedding(
          self,
          skus: List[Dict[str, Any]],
          option1_vectors: Dict[str, np.ndarray],
          query_vector: np.ndarray,
a7cc9078   tangwang   sku排序
345
          spu_option1_name: Optional[Any] = None,
deccd68a   tangwang   Added the SKU pre...
346
      ) -> Tuple[Optional[int], Optional[float]]:
a7cc9078   tangwang   sku排序
347
          """Select the SKU whose option1 embedding key (name:value) is most similar to the query."""
deccd68a   tangwang   Added the SKU pre...
348
349
350
351
          best_index: Optional[int] = None
          best_score: Optional[float] = None
  
          for index, sku in enumerate(skus):
a7cc9078   tangwang   sku排序
352
353
              embed_key = self._sku_option1_embedding_key(sku, spu_option1_name)
              if not embed_key:
deccd68a   tangwang   Added the SKU pre...
354
                  continue
a7cc9078   tangwang   sku排序
355
              option_vector = option1_vectors.get(embed_key)
deccd68a   tangwang   Added the SKU pre...
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
              if option_vector is None:
                  continue
              score = float(np.inner(query_vector, option_vector))
              if best_score is None or score > best_score:
                  best_index = index
                  best_score = score
  
          return best_index, best_score
  
      @staticmethod
      def _promote_matching_sku(source: Dict[str, Any], match_index: int) -> Optional[Dict[str, Any]]:
          """Move the matched SKU to the front and swap the SPU image."""
          skus = source.get("skus")
          if not isinstance(skus, list) or match_index < 0 or match_index >= len(skus):
              return None
  
          matched_sku = skus.pop(match_index)
          skus.insert(0, matched_sku)
  
          image_src = matched_sku.get("image_src") or matched_sku.get("imageSrc")
          if image_src:
              source["image_url"] = image_src
          return matched_sku
  
      def _apply_sku_sorting_for_page_hits(
          self,
          es_hits: List[Dict[str, Any]],
          parsed_query: ParsedQuery,
          context: Optional[RequestContext] = None,
      ) -> None:
          """Sort each page hit's SKUs so the best-matching SKU is first."""
          if not es_hits:
              return
  
          query_texts = self._build_sku_query_texts(parsed_query)
          unmatched_hits: List[Dict[str, Any]] = []
          option1_values_to_encode: List[str] = []
          seen_option1_values = set()
          text_matched = 0
          embedding_matched = 0
  
          for hit in es_hits:
              source = hit.get("_source")
              if not isinstance(source, dict):
                  continue
              skus = source.get("skus")
              if not isinstance(skus, list) or not skus:
                  continue
  
a7cc9078   tangwang   sku排序
405
406
407
408
              spu_option1_name = source.get("option1_name")
              match_index = self._find_query_matching_sku_index(
                  skus, query_texts, spu_option1_name=spu_option1_name
              )
deccd68a   tangwang   Added the SKU pre...
409
410
411
412
413
414
415
              if match_index is not None:
                  self._promote_matching_sku(source, match_index)
                  text_matched += 1
                  continue
  
              unmatched_hits.append(hit)
              for sku in skus:
a7cc9078   tangwang   sku排序
416
417
                  embed_key = self._sku_option1_embedding_key(sku, spu_option1_name)
                  if not embed_key or embed_key in seen_option1_values:
deccd68a   tangwang   Added the SKU pre...
418
                      continue
a7cc9078   tangwang   sku排序
419
420
                  seen_option1_values.add(embed_key)
                  option1_values_to_encode.append(embed_key)
deccd68a   tangwang   Added the SKU pre...
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
  
          if not unmatched_hits or not option1_values_to_encode:
              return
  
          query_vector = getattr(parsed_query, "query_vector", None)
          if query_vector is None:
              query_vector = self._encode_query_vector_for_sku_matching(parsed_query, context=context)
          if query_vector is None:
              return
  
          text_encoder = getattr(self.query_parser, "text_encoder", None)
          if text_encoder is None:
              return
  
          try:
              encoded_option_vectors = text_encoder.encode(option1_values_to_encode, priority=1)
          except Exception as exc:
              logger.warning("Failed to encode SKU option1 values for final-page sorting: %s", exc, exc_info=True)
              if context is not None:
                  context.add_warning(f"SKU option embedding failed: {exc}")
              return
  
          option1_vectors: Dict[str, np.ndarray] = {}
          for option1_value, vector in zip(option1_values_to_encode, encoded_option_vectors):
              if vector is None:
                  continue
              option1_vectors[option1_value] = np.asarray(vector, dtype=np.float32)
  
          query_vector_array = np.asarray(query_vector, dtype=np.float32)
          for hit in unmatched_hits:
              source = hit.get("_source")
              if not isinstance(source, dict):
                  continue
              skus = source.get("skus")
              if not isinstance(skus, list) or not skus:
                  continue
a7cc9078   tangwang   sku排序
457
458
459
460
461
462
              match_index, _ = self._select_sku_by_embedding(
                  skus,
                  option1_vectors,
                  query_vector_array,
                  spu_option1_name=source.get("option1_name"),
              )
deccd68a   tangwang   Added the SKU pre...
463
464
465
466
467
468
469
470
471
472
473
474
              if match_index is None:
                  continue
              self._promote_matching_sku(source, match_index)
              embedding_matched += 1
  
          if text_matched or embedding_matched:
              logger.info(
                  "Final-page SKU sorting completed | text_matched=%s | embedding_matched=%s",
                  text_matched,
                  embedding_matched,
              )
  
be52af70   tangwang   first commit
475
476
477
      def search(
          self,
          query: str,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
478
          tenant_id: str,
be52af70   tangwang   first commit
479
480
481
          size: int = 10,
          from_: int = 0,
          filters: Optional[Dict[str, Any]] = None,
6aa246be   tangwang   问题:Pydantic 应该能自动...
482
          range_filters: Optional[Dict[str, Any]] = None,
13320ac6   tangwang   分面接口修改:
483
          facets: Optional[List[FacetConfig]] = None,
16c42787   tangwang   feat: implement r...
484
          min_score: Optional[float] = None,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
485
          context: Optional[RequestContext] = None,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
486
          sort_by: Optional[str] = None,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
487
          sort_order: Optional[str] = "desc",
577ec972   tangwang   返回给前端的字段、格式适配。主要包...
488
          debug: bool = False,
2739b281   tangwang   多语言索引调整
489
          language: str = "en",
a3a5d41b   tangwang   (sku_filter_dimen...
490
          sku_filter_dimension: Optional[List[str]] = None,
5f7d7f09   tangwang   性能测试报告.md
491
          enable_rerank: Optional[bool] = None,
ff32d894   tangwang   rerank
492
493
          rerank_query_template: Optional[str] = None,
          rerank_doc_template: Optional[str] = None,
be52af70   tangwang   first commit
494
495
      ) -> SearchResult:
          """
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
496
          Execute search query (外部友好格式).
be52af70   tangwang   first commit
497
498
499
  
          Args:
              query: Search query string
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
500
              tenant_id: Tenant ID (required for filtering)
be52af70   tangwang   first commit
501
502
              size: Number of results to return
              from_: Offset for pagination
6aa246be   tangwang   问题:Pydantic 应该能自动...
503
504
505
              filters: Exact match filters
              range_filters: Range filters for numeric fields
              facets: Facet configurations for faceted search
be52af70   tangwang   first commit
506
              min_score: Minimum score threshold
ef5baa86   tangwang   混杂语言处理
507
              context: Request context for tracking (required)
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
508
509
              sort_by: Field name for sorting
              sort_order: Sort order: 'asc' or 'desc'
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
510
              debug: Enable debug information output
ef5baa86   tangwang   混杂语言处理
511
512
513
514
515
516
517
518
519
              language: Response / field selection language hint (e.g. zh, en)
              sku_filter_dimension: SKU grouping dimensions for per-SPU variant pick
              enable_rerank: If None, use ``config.rerank.enabled``; if set, overrides
                  whether the rerank provider is invoked (subject to rerank window).
              rerank_query_template: Override for rerank query text template; None uses
                  ``config.rerank.rerank_query_template`` (e.g. ``"{query}"``).
              rerank_doc_template: Override for per-hit document text passed to rerank;
                  None uses ``config.rerank.rerank_doc_template``. Placeholders are
                  resolved in ``search/rerank_client.py``.
be52af70   tangwang   first commit
520
521
  
          Returns:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
522
              SearchResult object with formatted results
be52af70   tangwang   first commit
523
          """
16c42787   tangwang   feat: implement r...
524
          if context is None:
ed948666   tangwang   tidy
525
              raise ValueError("context is required")
16c42787   tangwang   feat: implement r...
526
  
345d960b   tangwang   1. 删除全局 enable_tr...
527
528
529
          # 根据租户配置决定翻译开关(离线/在线统一)
          tenant_loader = get_tenant_config_loader()
          tenant_cfg = tenant_loader.get_tenant_config(tenant_id)
038e4e2f   tangwang   refactor(i18n): t...
530
531
          index_langs = tenant_cfg.get("index_languages") or []
          enable_translation = len(index_langs) > 0
9f96d6f3   tangwang   短query不用语义搜索
532
          enable_embedding = self.config.query_config.enable_text_embedding
5f7d7f09   tangwang   性能测试报告.md
533
534
535
536
537
538
          rc = self.config.rerank
          effective_query_template = rerank_query_template or rc.rerank_query_template
          effective_doc_template = rerank_doc_template or rc.rerank_doc_template
          # 重排开关优先级:请求参数显式传值 > 服务端配置(默认开启)
          rerank_enabled_by_config = bool(rc.enabled)
          do_rerank = rerank_enabled_by_config if enable_rerank is None else bool(enable_rerank)
c51d254f   tangwang   性能测试
539
          rerank_window = rc.rerank_window
506c39b7   tangwang   feat(search): 统一重...
540
          # 若开启重排且请求范围在窗口内:从 ES 取前 rerank_window 条、重排后再按 from/size 分页;否则不重排,按原 from/size 查 ES
ff32d894   tangwang   rerank
541
          in_rerank_window = do_rerank and (from_ + size) <= rerank_window
506c39b7   tangwang   feat(search): 统一重...
542
543
          es_fetch_from = 0 if in_rerank_window else from_
          es_fetch_size = rerank_window if in_rerank_window else size
16c42787   tangwang   feat: implement r...
544
545
546
547
548
549
  
          # Start timing
          context.start_stage(RequestContextStage.TOTAL)
  
          context.logger.info(
              f"开始搜索请求 | 查询: '{query}' | 参数: size={size}, from_={from_}, "
5f7d7f09   tangwang   性能测试报告.md
550
551
552
              f"enable_rerank(request)={enable_rerank}, enable_rerank(config)={rerank_enabled_by_config}, "
              f"enable_rerank(effective)={do_rerank}, in_rerank_window={in_rerank_window}, "
              f"es_fetch=({es_fetch_from},{es_fetch_size}) | "
506c39b7   tangwang   feat(search): 统一重...
553
              f"enable_translation={enable_translation}, enable_embedding={enable_embedding}, min_score={min_score}",
16c42787   tangwang   feat: implement r...
554
555
556
557
558
559
560
              extra={'reqid': context.reqid, 'uid': context.uid}
          )
  
          # Store search parameters in context
          context.metadata['search_params'] = {
              'size': size,
              'from_': from_,
506c39b7   tangwang   feat(search): 统一重...
561
562
563
              'es_fetch_from': es_fetch_from,
              'es_fetch_size': es_fetch_size,
              'in_rerank_window': in_rerank_window,
5f7d7f09   tangwang   性能测试报告.md
564
565
566
567
              'rerank_enabled_by_config': rerank_enabled_by_config,
              'enable_rerank_request': enable_rerank,
              'rerank_query_template': effective_query_template,
              'rerank_doc_template': effective_doc_template,
16c42787   tangwang   feat: implement r...
568
              'filters': filters,
6aa246be   tangwang   问题:Pydantic 应该能自动...
569
570
              'range_filters': range_filters,
              'facets': facets,
16c42787   tangwang   feat: implement r...
571
572
              'enable_translation': enable_translation,
              'enable_embedding': enable_embedding,
ff32d894   tangwang   rerank
573
              'enable_rerank': do_rerank,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
574
              'min_score': min_score,
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
575
576
              'sort_by': sort_by,
              'sort_order': sort_order
16c42787   tangwang   feat: implement r...
577
          }
be52af70   tangwang   first commit
578
  
16c42787   tangwang   feat: implement r...
579
580
581
          context.metadata['feature_flags'] = {
              'translation_enabled': enable_translation,
              'embedding_enabled': enable_embedding,
ff32d894   tangwang   rerank
582
              'rerank_enabled': do_rerank
16c42787   tangwang   feat: implement r...
583
          }
be52af70   tangwang   first commit
584
585
  
          # Step 1: Parse query
16c42787   tangwang   feat: implement r...
586
587
588
589
          context.start_stage(RequestContextStage.QUERY_PARSING)
          try:
              parsed_query = self.query_parser.parse(
                  query,
345d960b   tangwang   1. 删除全局 enable_tr...
590
                  tenant_id=tenant_id,
16c42787   tangwang   feat: implement r...
591
                  generate_vector=enable_embedding,
ef5baa86   tangwang   混杂语言处理
592
593
                  context=context,
                  target_languages=index_langs if enable_translation else [],
16c42787   tangwang   feat: implement r...
594
595
596
597
              )
              # Store query analysis results in context
              context.store_query_analysis(
                  original_query=parsed_query.original_query,
3a5fda00   tangwang   1. ES字段 skus的 ima...
598
                  query_normalized=parsed_query.query_normalized,
16c42787   tangwang   feat: implement r...
599
600
601
602
                  rewritten_query=parsed_query.rewritten_query,
                  detected_language=parsed_query.detected_language,
                  translations=parsed_query.translations,
                  query_vector=parsed_query.query_vector.tolist() if parsed_query.query_vector is not None else None,
ef5baa86   tangwang   混杂语言处理
603
                  domain="default",
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
604
                  is_simple_query=True
16c42787   tangwang   feat: implement r...
605
              )
be52af70   tangwang   first commit
606
  
16c42787   tangwang   feat: implement r...
607
608
609
610
              context.logger.info(
                  f"查询解析完成 | 原查询: '{parsed_query.original_query}' | "
                  f"重写后: '{parsed_query.rewritten_query}' | "
                  f"语言: {parsed_query.detected_language} | "
16c42787   tangwang   feat: implement r...
611
612
613
614
615
616
617
618
619
620
621
622
623
                  f"向量: {'是' if parsed_query.query_vector is not None else '否'}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"查询解析失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
              context.end_stage(RequestContextStage.QUERY_PARSING)
  
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
624
          # Step 2: Query building
16c42787   tangwang   feat: implement r...
625
626
          context.start_stage(RequestContextStage.QUERY_BUILDING)
          try:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
627
628
              # Generate tenant-specific index name
              index_name = get_tenant_index_name(tenant_id)
2739b281   tangwang   多语言索引调整
629
              # index_name = "search_products"
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
630
631
              
              # No longer need to add tenant_id to filters since each tenant has its own index
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
632
  
f0d020c3   tangwang   多语言查询改为只支持中英文两种,f...
633
              es_query = self.query_builder.build_query(
3a5fda00   tangwang   1. ES字段 skus的 ima...
634
                  query_text=parsed_query.rewritten_query or parsed_query.query_normalized,
16c42787   tangwang   feat: implement r...
635
                  query_vector=parsed_query.query_vector if enable_embedding else None,
16c42787   tangwang   feat: implement r...
636
                  filters=filters,
6aa246be   tangwang   问题:Pydantic 应该能自动...
637
                  range_filters=range_filters,
c581becd   tangwang   feat: 实现 Multi-Se...
638
                  facet_configs=facets,
506c39b7   tangwang   feat(search): 统一重...
639
640
                  size=es_fetch_size,
                  from_=es_fetch_from,
16c42787   tangwang   feat: implement r...
641
                  enable_knn=enable_embedding and parsed_query.query_vector is not None,
7bc756c5   tangwang   优化 ES 查询构建
642
                  min_score=min_score,
ef5baa86   tangwang   混杂语言处理
643
644
                  parsed_query=parsed_query,
                  index_languages=index_langs,
16c42787   tangwang   feat: implement r...
645
              )
be52af70   tangwang   first commit
646
  
6aa246be   tangwang   问题:Pydantic 应该能自动...
647
648
649
650
651
652
653
              # Add facets for faceted search
              if facets:
                  facet_aggs = self.query_builder.build_facets(facets)
                  if facet_aggs:
                      if "aggs" not in es_query:
                          es_query["aggs"] = {}
                      es_query["aggs"].update(facet_aggs)
16c42787   tangwang   feat: implement r...
654
  
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
655
656
657
              # Add sorting if specified
              if sort_by:
                  es_query = self.query_builder.add_sorting(es_query, sort_by, sort_order)
a47416ec   tangwang   把融合逻辑改成乘法公式,并把 ES...
658
                  es_query["track_scores"] = True
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
659
  
5f7d7f09   tangwang   性能测试报告.md
660
661
662
663
664
665
666
667
668
669
670
              # Keep requested response _source semantics for the final response fill.
              response_source_spec = es_query.get("_source")
  
              # In rerank window, first pass only fetches minimal fields required by rerank template.
              es_query_for_fetch = es_query
              rerank_prefetch_source = None
              if in_rerank_window:
                  rerank_prefetch_source = self._resolve_rerank_source_filter(effective_doc_template)
                  es_query_for_fetch = dict(es_query)
                  es_query_for_fetch["_source"] = rerank_prefetch_source
  
16c42787   tangwang   feat: implement r...
671
              # Extract size and from from body for ES client parameters
5f7d7f09   tangwang   性能测试报告.md
672
              body_for_es = {k: v for k, v in es_query_for_fetch.items() if k not in ['size', 'from']}
16c42787   tangwang   feat: implement r...
673
674
675
  
              # Store ES query in context
              context.store_intermediate_result('es_query', es_query)
5f7d7f09   tangwang   性能测试报告.md
676
677
              if in_rerank_window and rerank_prefetch_source is not None:
                  context.store_intermediate_result('es_query_rerank_prefetch_source', rerank_prefetch_source)
16c42787   tangwang   feat: implement r...
678
679
              context.store_intermediate_result('es_body_for_search', body_for_es)
  
28e57bb1   tangwang   日志体系优化
680
              # Serialize ES query to compute a compact size + stable digest for correlation
5f7d7f09   tangwang   性能测试报告.md
681
              es_query_compact = json.dumps(es_query_for_fetch, ensure_ascii=False, separators=(",", ":"))
28e57bb1   tangwang   日志体系优化
682
683
684
              es_query_digest = hashlib.sha256(es_query_compact.encode("utf-8")).hexdigest()[:16]
              knn_enabled = bool(enable_embedding and parsed_query.query_vector is not None)
              vector_dims = int(len(parsed_query.query_vector)) if parsed_query.query_vector is not None else 0
99bea633   tangwang   add logs
685
  
16c42787   tangwang   feat: implement r...
686
              context.logger.info(
5f7d7f09   tangwang   性能测试报告.md
687
                  "ES query built | size: %s chars | digest: %s | KNN: %s | vector_dims: %s | facets: %s | rerank_prefetch_source: %s",
28e57bb1   tangwang   日志体系优化
688
689
690
691
692
                  len(es_query_compact),
                  es_query_digest,
                  "yes" if knn_enabled else "no",
                  vector_dims,
                  "yes" if facets else "no",
5f7d7f09   tangwang   性能测试报告.md
693
                  rerank_prefetch_source,
16c42787   tangwang   feat: implement r...
694
695
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
28e57bb1   tangwang   日志体系优化
696
697
698
699
700
701
702
703
704
705
              _log_backend_verbose({
                  "event": "es_query_built",
                  "reqid": context.reqid,
                  "uid": context.uid,
                  "tenant_id": tenant_id,
                  "size_chars": len(es_query_compact),
                  "sha256_16": es_query_digest,
                  "knn_enabled": knn_enabled,
                  "vector_dims": vector_dims,
                  "has_facets": bool(facets),
5f7d7f09   tangwang   性能测试报告.md
706
                  "query": es_query_for_fetch,
28e57bb1   tangwang   日志体系优化
707
              })
16c42787   tangwang   feat: implement r...
708
709
710
711
712
713
714
715
716
717
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"ES查询构建失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
              context.end_stage(RequestContextStage.QUERY_BUILDING)
  
a99e62ba   tangwang   记录各阶段耗时
718
719
          # Step 4: Elasticsearch search (primary recall)
          context.start_stage(RequestContextStage.ELASTICSEARCH_SEARCH_PRIMARY)
16c42787   tangwang   feat: implement r...
720
          try:
506c39b7   tangwang   feat(search): 统一重...
721
              # Use tenant-specific index name(开启重排且在窗口内时已用 es_fetch_size/es_fetch_from)
16c42787   tangwang   feat: implement r...
722
              es_response = self.es_client.search(
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
723
                  index_name=index_name,
16c42787   tangwang   feat: implement r...
724
                  body=body_for_es,
506c39b7   tangwang   feat(search): 统一重...
725
                  size=es_fetch_size,
a47416ec   tangwang   把融合逻辑改成乘法公式,并把 ES...
726
727
                  from_=es_fetch_from,
                  include_named_queries_score=bool(do_rerank and in_rerank_window),
be52af70   tangwang   first commit
728
729
              )
  
16c42787   tangwang   feat: implement r...
730
731
              # Store ES response in context
              context.store_intermediate_result('es_response', es_response)
be52af70   tangwang   first commit
732
  
16c42787   tangwang   feat: implement r...
733
734
735
736
737
              # Extract timing from ES response
              es_took = es_response.get('took', 0)
              context.logger.info(
                  f"ES搜索完成 | 耗时: {es_took}ms | "
                  f"命中数: {es_response.get('hits', {}).get('total', {}).get('value', 0)} | "
25d3e81d   tangwang   fix指定sort项时候的bug
738
                  f"最高分: {(es_response.get('hits', {}).get('max_score') or 0):.3f}",
16c42787   tangwang   feat: implement r...
739
740
741
742
743
744
745
746
747
748
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"ES搜索执行失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
a99e62ba   tangwang   记录各阶段耗时
749
              context.end_stage(RequestContextStage.ELASTICSEARCH_SEARCH_PRIMARY)
16c42787   tangwang   feat: implement r...
750
  
506c39b7   tangwang   feat(search): 统一重...
751
          # Optional Step 4.5: AI reranking(仅当请求范围在重排窗口内时执行)
ff32d894   tangwang   rerank
752
          if do_rerank and in_rerank_window:
506c39b7   tangwang   feat(search): 统一重...
753
754
755
756
757
              context.start_stage(RequestContextStage.RERANKING)
              try:
                  from .rerank_client import run_rerank
  
                  rerank_query = parsed_query.original_query if parsed_query else query
506c39b7   tangwang   feat(search): 统一重...
758
759
760
761
                  es_response, rerank_meta, fused_debug = run_rerank(
                      query=rerank_query,
                      es_response=es_response,
                      language=language,
506c39b7   tangwang   feat(search): 统一重...
762
763
764
                      timeout_sec=rc.timeout_sec,
                      weight_es=rc.weight_es,
                      weight_ai=rc.weight_ai,
ff32d894   tangwang   rerank
765
766
                      rerank_query_template=effective_query_template,
                      rerank_doc_template=effective_doc_template,
d31c7f65   tangwang   补充云服务reranker
767
                      top_n=(from_ + size),
506c39b7   tangwang   feat(search): 统一重...
768
769
770
                  )
  
                  if rerank_meta is not None:
42e3aea6   tangwang   tidy
771
772
                      from config.services_config import get_rerank_service_url
                      rerank_url = get_rerank_service_url()
506c39b7   tangwang   feat(search): 统一重...
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
                      context.metadata.setdefault("rerank_info", {})
                      context.metadata["rerank_info"].update({
                          "service_url": rerank_url,
                          "docs": len(es_response.get("hits", {}).get("hits") or []),
                          "meta": rerank_meta,
                      })
                      context.store_intermediate_result("rerank_scores", fused_debug)
                      context.logger.info(
                          f"重排完成 | docs={len(fused_debug)} | meta={rerank_meta}",
                          extra={'reqid': context.reqid, 'uid': context.uid}
                      )
              except Exception as e:
                  context.add_warning(f"Rerank failed: {e}")
                  context.logger.warning(
                      f"调用重排服务失败 | error: {e}",
                      extra={'reqid': context.reqid, 'uid': context.uid},
                      exc_info=True,
                  )
              finally:
                  context.end_stage(RequestContextStage.RERANKING)
  
          # 当本次请求在重排窗口内时:已从 ES 取了 rerank_window 条并可能已重排,需按请求的 from/size 做分页切片
          if in_rerank_window:
              hits = es_response.get("hits", {}).get("hits") or []
              sliced = hits[from_ : from_ + size]
              es_response.setdefault("hits", {})["hits"] = sliced
              if sliced:
af827ce9   tangwang   rerank
800
801
802
803
804
                  # 对于启用重排的结果,优先使用 _fused_score 计算 max_score;否则退回原始 _score
                  slice_max = max(
                      (h.get("_fused_score", h.get("_score", 0.0)) for h in sliced),
                      default=0.0,
                  )
506c39b7   tangwang   feat(search): 统一重...
805
806
807
808
809
810
                  try:
                      es_response["hits"]["max_score"] = float(slice_max)
                  except (TypeError, ValueError):
                      es_response["hits"]["max_score"] = 0.0
              else:
                  es_response["hits"]["max_score"] = 0.0
5f7d7f09   tangwang   性能测试报告.md
811
812
813
814
815
816
817
818
819
820
821
  
              # Page fill: fetch detailed fields only for final page hits.
              if sliced:
                  if response_source_spec is False:
                      for hit in sliced:
                          hit.pop("_source", None)
                      context.logger.info(
                          "分页详情回填跳过 | 原查询 _source=false",
                          extra={'reqid': context.reqid, 'uid': context.uid}
                      )
                  else:
a99e62ba   tangwang   记录各阶段耗时
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
                      context.start_stage(RequestContextStage.ELASTICSEARCH_PAGE_FILL)
                      try:
                          page_ids = [str(h.get("_id")) for h in sliced if h.get("_id") is not None]
                          details_by_id, fill_took = self._fetch_hits_by_ids(
                              index_name=index_name,
                              doc_ids=page_ids,
                              source_spec=response_source_spec,
                          )
                          filled = 0
                          for hit in sliced:
                              hid = hit.get("_id")
                              if hid is None:
                                  continue
                              detail_hit = details_by_id.get(str(hid))
                              if detail_hit is None:
                                  continue
                              if "_source" in detail_hit:
                                  hit["_source"] = detail_hit.get("_source") or {}
                                  filled += 1
                          if fill_took:
                              es_response["took"] = int((es_response.get("took", 0) or 0) + fill_took)
                          context.logger.info(
                              f"分页详情回填 | ids={len(page_ids)} | filled={filled} | took={fill_took}ms",
                              extra={'reqid': context.reqid, 'uid': context.uid}
                          )
                      finally:
                          context.end_stage(RequestContextStage.ELASTICSEARCH_PAGE_FILL)
5f7d7f09   tangwang   性能测试报告.md
849
  
506c39b7   tangwang   feat(search): 统一重...
850
851
852
853
854
              context.logger.info(
                  f"重排分页切片 | from={from_}, size={size}, 返回={len(sliced)}条",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
  
16c42787   tangwang   feat: implement r...
855
856
857
          # Step 5: Result processing
          context.start_stage(RequestContextStage.RESULT_PROCESSING)
          try:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
858
859
              # Extract ES hits
              es_hits = []
16c42787   tangwang   feat: implement r...
860
              if 'hits' in es_response and 'hits' in es_response['hits']:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
861
                  es_hits = es_response['hits']['hits']
16c42787   tangwang   feat: implement r...
862
863
864
865
866
867
              # Extract total and max_score
              total = es_response.get('hits', {}).get('total', {})
              if isinstance(total, dict):
                  total_value = total.get('value', 0)
              else:
                  total_value = total
506c39b7   tangwang   feat(search): 统一重...
868
              # max_score 会在启用 AI 搜索时被更新为融合分数的最大值
25d3e81d   tangwang   fix指定sort项时候的bug
869
              max_score = es_response.get('hits', {}).get('max_score') or 0.0
be52af70   tangwang   first commit
870
  
af827ce9   tangwang   rerank
871
872
873
874
875
876
877
878
879
880
881
882
              # 从上下文中取出重排调试信息(若有)
              rerank_debug_raw = context.get_intermediate_result('rerank_scores', None)
              rerank_debug_by_doc: Dict[str, Dict[str, Any]] = {}
              if isinstance(rerank_debug_raw, list):
                  for item in rerank_debug_raw:
                      if not isinstance(item, dict):
                          continue
                      doc_id = item.get("doc_id")
                      if doc_id is None:
                          continue
                      rerank_debug_by_doc[str(doc_id)] = item
  
deccd68a   tangwang   Added the SKU pre...
883
884
              self._apply_sku_sorting_for_page_hits(es_hits, parsed_query, context=context)
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
885
              # Format results using ResultFormatter
577ec972   tangwang   返回给前端的字段、格式适配。主要包...
886
887
888
              formatted_results = ResultFormatter.format_search_results(
                  es_hits,
                  max_score,
ca91352a   tangwang   更新文档
889
890
                  language=language,
                  sku_filter_dimension=sku_filter_dimension
577ec972   tangwang   返回给前端的字段、格式适配。主要包...
891
              )
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
892
  
985752f5   tangwang   1. 前端调试功能
893
894
895
896
897
              # Build per-result debug info (per SPU) when debug mode is enabled
              per_result_debug = []
              if debug and es_hits and formatted_results:
                  for hit, spu in zip(es_hits, formatted_results):
                      source = hit.get("_source", {}) or {}
af827ce9   tangwang   rerank
898
899
900
901
902
                      doc_id = hit.get("_id")
                      rerank_debug = None
                      if doc_id is not None:
                          rerank_debug = rerank_debug_by_doc.get(str(doc_id))
  
985752f5   tangwang   1. 前端调试功能
903
904
905
906
907
908
909
910
911
912
913
914
915
916
                      raw_score = hit.get("_score")
                      try:
                          es_score = float(raw_score) if raw_score is not None else 0.0
                      except (TypeError, ValueError):
                          es_score = 0.0
                      try:
                          normalized = float(es_score) / float(max_score) if max_score else None
                      except (TypeError, ValueError, ZeroDivisionError):
                          normalized = None
  
                      title_multilingual = source.get("title") if isinstance(source.get("title"), dict) else None
                      brief_multilingual = source.get("brief") if isinstance(source.get("brief"), dict) else None
                      vendor_multilingual = source.get("vendor") if isinstance(source.get("vendor"), dict) else None
  
af827ce9   tangwang   rerank
917
918
919
920
921
922
923
924
925
926
927
928
929
                      debug_entry: Dict[str, Any] = {
                          "spu_id": spu.spu_id,
                          "es_score": es_score,
                          "es_score_normalized": normalized,
                          "title_multilingual": title_multilingual,
                          "brief_multilingual": brief_multilingual,
                          "vendor_multilingual": vendor_multilingual,
                      }
  
                      # 若存在重排调试信息,则补充 doc 级别的融合分数信息
                      if rerank_debug:
                          debug_entry["doc_id"] = rerank_debug.get("doc_id")
                          # 与 rerank_client 中字段保持一致,便于前端直接使用
af827ce9   tangwang   rerank
930
                          debug_entry["rerank_score"] = rerank_debug.get("rerank_score")
a8261ece   tangwang   检索效果优化
931
                          debug_entry["text_score"] = rerank_debug.get("text_score")
c90f80ed   tangwang   相关性优化
932
933
                          debug_entry["text_source_score"] = rerank_debug.get("text_source_score")
                          debug_entry["text_translation_score"] = rerank_debug.get("text_translation_score")
c90f80ed   tangwang   相关性优化
934
935
                          debug_entry["text_primary_score"] = rerank_debug.get("text_primary_score")
                          debug_entry["text_support_score"] = rerank_debug.get("text_support_score")
a8261ece   tangwang   检索效果优化
936
                          debug_entry["knn_score"] = rerank_debug.get("knn_score")
af827ce9   tangwang   rerank
937
                          debug_entry["fused_score"] = rerank_debug.get("fused_score")
a8261ece   tangwang   检索效果优化
938
                          debug_entry["matched_queries"] = rerank_debug.get("matched_queries")
af827ce9   tangwang   rerank
939
940
  
                      per_result_debug.append(debug_entry)
985752f5   tangwang   1. 前端调试功能
941
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
942
943
944
945
946
              # Format facets
              standardized_facets = None
              if facets:
                  standardized_facets = ResultFormatter.format_facets(
                      es_response.get('aggregations', {}),
c581becd   tangwang   feat: 实现 Multi-Se...
947
948
                      facets,
                      filters
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
949
950
951
952
953
954
                  )
  
              # Generate suggestions and related searches
              query_text = parsed_query.original_query if parsed_query else query
              suggestions = ResultFormatter.generate_suggestions(query_text, formatted_results)
              related_searches = ResultFormatter.generate_related_searches(query_text, formatted_results)
be52af70   tangwang   first commit
955
  
16c42787   tangwang   feat: implement r...
956
              context.logger.info(
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
957
                  f"结果处理完成 | 返回: {len(formatted_results)}条 | 总计: {total_value}条",
16c42787   tangwang   feat: implement r...
958
959
960
961
962
963
964
965
966
967
968
969
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
  
          except Exception as e:
              context.set_error(e)
              context.logger.error(
                  f"结果处理失败 | 错误: {str(e)}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
              raise
          finally:
              context.end_stage(RequestContextStage.RESULT_PROCESSING)
be52af70   tangwang   first commit
970
  
16c42787   tangwang   feat: implement r...
971
972
973
          # End total timing and build result
          total_duration = context.end_stage(RequestContextStage.TOTAL)
          context.performance_metrics.total_duration = total_duration
be52af70   tangwang   first commit
974
  
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
975
976
977
978
979
980
          # Collect debug information if requested
          debug_info = None
          if debug:
              debug_info = {
                  "query_analysis": {
                      "original_query": context.query_analysis.original_query,
3a5fda00   tangwang   1. ES字段 skus的 ima...
981
                      "query_normalized": context.query_analysis.query_normalized,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
982
983
984
                      "rewritten_query": context.query_analysis.rewritten_query,
                      "detected_language": context.query_analysis.detected_language,
                      "translations": context.query_analysis.translations,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
985
986
                      "has_vector": context.query_analysis.query_vector is not None,
                      "is_simple_query": context.query_analysis.is_simple_query,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
                      "domain": context.query_analysis.domain
                  },
                  "es_query": context.get_intermediate_result('es_query', {}),
                  "es_response": {
                      "took_ms": es_response.get('took', 0),
                      "total_hits": total_value,
                      "max_score": max_score,
                      "shards": es_response.get('_shards', {})
                  },
                  "feature_flags": context.metadata.get('feature_flags', {}),
                  "stage_timings": {
                      k: round(v, 2) for k, v in context.performance_metrics.stage_timings.items()
                  },
                  "search_params": context.metadata.get('search_params', {})
              }
985752f5   tangwang   1. 前端调试功能
1002
1003
              if per_result_debug:
                  debug_info["per_result"] = per_result_debug
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1004
  
be52af70   tangwang   first commit
1005
1006
          # Build result
          result = SearchResult(
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1007
              results=formatted_results,
be52af70   tangwang   first commit
1008
1009
              total=total_value,
              max_score=max_score,
16c42787   tangwang   feat: implement r...
1010
              took_ms=int(total_duration),
6aa246be   tangwang   问题:Pydantic 应该能自动...
1011
              facets=standardized_facets,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1012
              query_info=parsed_query.to_dict(),
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1013
1014
              suggestions=suggestions,
              related_searches=related_searches,
1f071951   tangwang   补充调试信息,记录包括各个阶段的 ...
1015
              debug_info=debug_info
be52af70   tangwang   first commit
1016
1017
          )
  
16c42787   tangwang   feat: implement r...
1018
1019
          # Log complete performance summary
          context.log_performance_summary()
be52af70   tangwang   first commit
1020
1021
1022
1023
1024
1025
  
          return result
  
      def search_by_image(
          self,
          image_url: str,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1026
          tenant_id: str,
be52af70   tangwang   first commit
1027
          size: int = 10,
6aa246be   tangwang   问题:Pydantic 应该能自动...
1028
1029
          filters: Optional[Dict[str, Any]] = None,
          range_filters: Optional[Dict[str, Any]] = None
be52af70   tangwang   first commit
1030
1031
      ) -> SearchResult:
          """
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1032
          Search by image similarity (外部友好格式).
be52af70   tangwang   first commit
1033
1034
1035
  
          Args:
              image_url: URL of query image
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1036
              tenant_id: Tenant ID (required for filtering)
be52af70   tangwang   first commit
1037
              size: Number of results
6aa246be   tangwang   问题:Pydantic 应该能自动...
1038
1039
              filters: Exact match filters
              range_filters: Range filters for numeric fields
be52af70   tangwang   first commit
1040
1041
  
          Returns:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1042
              SearchResult object with formatted results
be52af70   tangwang   first commit
1043
1044
1045
1046
1047
          """
          if not self.image_embedding_field:
              raise ValueError("Image embedding field not configured")
  
          # Generate image embedding
26b910bd   tangwang   refactor service ...
1048
1049
          if self.image_encoder is None:
              raise RuntimeError("Image encoder is not initialized at startup")
b754fd41   tangwang   图片向量化支持优先级参数
1050
          image_vector = self.image_encoder.encode_image_from_url(image_url, priority=1)
be52af70   tangwang   first commit
1051
1052
1053
1054
  
          if image_vector is None:
              raise ValueError(f"Failed to encode image: {image_url}")
  
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1055
1056
1057
1058
          # Generate tenant-specific index name
          index_name = get_tenant_index_name(tenant_id)
          
          # No longer need to add tenant_id to filters since each tenant has its own index
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1059
  
be52af70   tangwang   first commit
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
          # Build KNN query
          es_query = {
              "size": size,
              "knn": {
                  "field": self.image_embedding_field,
                  "query_vector": image_vector.tolist(),
                  "k": size,
                  "num_candidates": size * 10
              }
          }
  
26b910bd   tangwang   refactor service ...
1071
1072
          # Apply source filtering semantics (None / [] / list)
          self._apply_source_filter(es_query)
13377199   tangwang   接口优化
1073
  
6aa246be   tangwang   问题:Pydantic 应该能自动...
1074
1075
1076
          if filters or range_filters:
              filter_clauses = self.query_builder._build_filters(filters, range_filters)
              if filter_clauses:
7fbca0d7   tangwang   启动脚本优化
1077
1078
1079
1080
1081
1082
1083
                  if len(filter_clauses) == 1:
                      es_query["knn"]["filter"] = filter_clauses[0]
                  else:
                      es_query["knn"]["filter"] = {
                          "bool": {
                              "filter": filter_clauses
                          }
6aa246be   tangwang   问题:Pydantic 应该能自动...
1084
                      }
be52af70   tangwang   first commit
1085
1086
1087
  
          # Execute search
          es_response = self.es_client.search(
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1088
              index_name=index_name,
be52af70   tangwang   first commit
1089
1090
1091
1092
              body=es_query,
              size=size
          )
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1093
1094
          # Extract ES hits
          es_hits = []
be52af70   tangwang   first commit
1095
          if 'hits' in es_response and 'hits' in es_response['hits']:
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1096
              es_hits = es_response['hits']['hits']
be52af70   tangwang   first commit
1097
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1098
          # Extract total and max_score
be52af70   tangwang   first commit
1099
1100
1101
1102
1103
1104
          total = es_response.get('hits', {}).get('total', {})
          if isinstance(total, dict):
              total_value = total.get('value', 0)
          else:
              total_value = total
  
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1105
1106
1107
          max_score = es_response.get('hits', {}).get('max_score') or 0.0
  
          # Format results using ResultFormatter
ca91352a   tangwang   更新文档
1108
1109
1110
          formatted_results = ResultFormatter.format_search_results(
              es_hits, 
              max_score,
2739b281   tangwang   多语言索引调整
1111
              language="en",  # Default language for image search
ca91352a   tangwang   更新文档
1112
1113
              sku_filter_dimension=None  # Image search doesn't support SKU filtering
          )
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1114
  
be52af70   tangwang   first commit
1115
          return SearchResult(
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1116
              results=formatted_results,
be52af70   tangwang   first commit
1117
              total=total_value,
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1118
              max_score=max_score,
be52af70   tangwang   first commit
1119
              took_ms=es_response.get('took', 0),
1f6d15fa   tangwang   重构:SPU级别索引、统一索引架构...
1120
1121
1122
1123
              facets=None,
              query_info={'image_url': image_url, 'search_type': 'image_similarity'},
              suggestions=[],
              related_searches=[]
be52af70   tangwang   first commit
1124
1125
          )
  
b926f678   tangwang   多语言查询
1126
1127
      def get_domain_summary(self) -> Dict[str, Any]:
          """
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
1128
          Get summary of dynamic text retrieval configuration.
b926f678   tangwang   多语言查询
1129
1130
  
          Returns:
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
1131
              Dictionary with language-aware field information
b926f678   tangwang   多语言查询
1132
          """
bd96cead   tangwang   1. 动态多语言字段与统一策略配置
1133
1134
1135
1136
1137
1138
1139
          return {
              "mode": "dynamic_language_fields",
              "multilingual_fields": self.config.query_config.multilingual_fields,
              "shared_fields": self.config.query_config.shared_fields,
              "core_multilingual_fields": self.config.query_config.core_multilingual_fields,
              "field_boosts": self.config.field_boosts,
          }
b926f678   tangwang   多语言查询
1140
  
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1141
      def get_document(self, tenant_id: str, doc_id: str) -> Optional[Dict[str, Any]]:
be52af70   tangwang   first commit
1142
1143
1144
1145
          """
          Get single document by ID.
  
          Args:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1146
              tenant_id: Tenant ID (required to determine which index to query)
be52af70   tangwang   first commit
1147
1148
1149
1150
1151
1152
              doc_id: Document ID
  
          Returns:
              Document or None if not found
          """
          try:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1153
              index_name = get_tenant_index_name(tenant_id)
be52af70   tangwang   first commit
1154
              response = self.es_client.client.get(
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1155
                  index=index_name,
be52af70   tangwang   first commit
1156
1157
1158
1159
                  id=doc_id
              )
              return response.get('_source')
          except Exception as e:
e4a39cc8   tangwang   索引隔离。 不同的tenant_i...
1160
              logger.error(f"Failed to get document {doc_id} from tenant {tenant_id}: {e}", exc_info=True)
be52af70   tangwang   first commit
1161
              return None
6aa246be   tangwang   问题:Pydantic 应该能自动...
1162
1163
1164
1165
  
      def _standardize_facets(
          self,
          es_aggregations: Dict[str, Any],
ff5325fa   tangwang   修复:直接在 Searcher 层...
1166
          facet_configs: Optional[List[Union[str, Any]]],
6aa246be   tangwang   问题:Pydantic 应该能自动...
1167
1168
1169
1170
1171
1172
1173
          current_filters: Optional[Dict[str, Any]]
      ) -> Optional[List[FacetResult]]:
          """
           ES 聚合结果转换为标准化的分面格式(返回 Pydantic 模型)。
          
          Args:
              es_aggregations: ES 原始聚合结果
ff5325fa   tangwang   修复:直接在 Searcher 层...
1174
              facet_configs: 分面配置列表(str  FacetConfig
6aa246be   tangwang   问题:Pydantic 应该能自动...
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
              current_filters: 当前应用的过滤器
          
          Returns:
              标准化的分面结果列表(FacetResult 对象)
          """
          if not es_aggregations or not facet_configs:
              return None
          
          standardized_facets: List[FacetResult] = []
          
          for config in facet_configs:
              # 解析配置
              if isinstance(config, str):
                  field = config
                  facet_type = "terms"
              else:
ff5325fa   tangwang   修复:直接在 Searcher 层...
1191
1192
1193
                  # FacetConfig 对象
                  field = config.field
                  facet_type = config.type
6aa246be   tangwang   问题:Pydantic 应该能自动...
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
              
              agg_name = f"{field}_facet"
              
              if agg_name not in es_aggregations:
                  continue
              
              agg_result = es_aggregations[agg_name]
              
              # 获取当前字段的选中值
              selected_values = set()
              if current_filters and field in current_filters:
                  filter_value = current_filters[field]
                  if isinstance(filter_value, list):
                      selected_values = set(filter_value)
                  else:
                      selected_values = {filter_value}
              
              # 转换 buckets 为 FacetValue 对象
              facet_values: List[FacetValue] = []
              if 'buckets' in agg_result:
                  for bucket in agg_result['buckets']:
                      value = bucket.get('key')
                      count = bucket.get('doc_count', 0)
                      
                      facet_values.append(FacetValue(
                          value=value,
                          label=str(value),
                          count=count,
                          selected=value in selected_values
                      ))
              
              # 构建 FacetResult 对象
              facet_result = FacetResult(
                  field=field,
e7ad2b4a   tangwang   测试页面分页配置
1228
                  label=field,
6aa246be   tangwang   问题:Pydantic 应该能自动...
1229
1230
1231
1232
1233
1234
1235
                  type=facet_type,
                  values=facet_values
              )
              
              standardized_facets.append(facet_result)
          
          return standardized_facets if standardized_facets else None