be52af70
tangwang
first commit
|
1
2
3
|
"""
Main Searcher module - executes search queries against Elasticsearch.
|
bd96cead
tangwang
1. 动态多语言字段与统一策略配置
|
4
|
Handles query parsing, ranking, and result formatting.
|
be52af70
tangwang
first commit
|
5
6
|
"""
|
deccd68a
tangwang
Added the SKU pre...
|
7
|
from typing import Dict, Any, List, Optional, Union, Tuple
|
d1d356f8
tangwang
脚本优化
|
8
|
import os
|
99bea633
tangwang
add logs
|
9
|
import time, json
|
325eec03
tangwang
1. 日志、配置基础设施,使用优化
|
10
|
import logging
|
28e57bb1
tangwang
日志体系优化
|
11
|
import hashlib
|
5f7d7f09
tangwang
性能测试报告.md
|
12
|
from string import Formatter
|
deccd68a
tangwang
Added the SKU pre...
|
13
|
import numpy as np
|
be52af70
tangwang
first commit
|
14
|
|
be52af70
tangwang
first commit
|
15
16
|
from utils.es_client import ESClient
from query import QueryParser, ParsedQuery
|
07cf5a93
tangwang
START_EMBEDDING=...
|
17
|
from embeddings.image_encoder import CLIPImageEncoder
|
be52af70
tangwang
first commit
|
18
|
from .es_query_builder import ESQueryBuilder
|
9f96d6f3
tangwang
短query不用语义搜索
|
19
|
from config import SearchConfig
|
345d960b
tangwang
1. 删除全局 enable_tr...
|
20
|
from config.tenant_config_loader import get_tenant_config_loader
|
ed948666
tangwang
tidy
|
21
|
from context.request_context import RequestContext, RequestContextStage
|
13320ac6
tangwang
分面接口修改:
|
22
|
from api.models import FacetResult, FacetValue, FacetConfig
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
23
|
from api.result_formatter import ResultFormatter
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
24
|
from indexer.mapping_generator import get_tenant_index_name
|
be52af70
tangwang
first commit
|
25
|
|
325eec03
tangwang
1. 日志、配置基础设施,使用优化
|
26
|
logger = logging.getLogger(__name__)
|
28e57bb1
tangwang
日志体系优化
|
27
28
29
30
31
32
33
34
35
|
backend_verbose_logger = logging.getLogger("backend.verbose")
def _log_backend_verbose(payload: Dict[str, Any]) -> None:
if not backend_verbose_logger.handlers:
return
backend_verbose_logger.info(
json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
)
|
325eec03
tangwang
1. 日志、配置基础设施,使用优化
|
36
|
|
be52af70
tangwang
first commit
|
37
38
|
class SearchResult:
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
39
|
"""Container for search results (外部友好格式)."""
|
be52af70
tangwang
first commit
|
40
41
42
|
def __init__(
self,
|
cadc77b6
tangwang
索引字段名、变量名、API数据结构...
|
43
|
results: List[Any], # List[SpuResult]
|
be52af70
tangwang
first commit
|
44
45
46
|
total: int,
max_score: float,
took_ms: int,
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
47
|
facets: Optional[List[FacetResult]] = None,
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
48
|
query_info: Optional[Dict[str, Any]] = None,
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
49
50
|
suggestions: Optional[List[str]] = None,
related_searches: Optional[List[str]] = None,
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
51
|
debug_info: Optional[Dict[str, Any]] = None
|
be52af70
tangwang
first commit
|
52
|
):
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
53
|
self.results = results
|
be52af70
tangwang
first commit
|
54
55
56
|
self.total = total
self.max_score = max_score
self.took_ms = took_ms
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
57
|
self.facets = facets
|
be52af70
tangwang
first commit
|
58
|
self.query_info = query_info or {}
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
59
60
|
self.suggestions = suggestions or []
self.related_searches = related_searches or []
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
61
|
self.debug_info = debug_info
|
43f1139f
tangwang
refactor: ES查询结构重...
|
62
|
|
be52af70
tangwang
first commit
|
63
64
|
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary representation."""
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
65
|
result = {
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
66
|
"results": [r.model_dump() if hasattr(r, 'model_dump') else r for r in self.results],
|
be52af70
tangwang
first commit
|
67
68
69
|
"total": self.total,
"max_score": self.max_score,
"took_ms": self.took_ms,
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
70
|
"facets": [f.model_dump() for f in self.facets] if self.facets else None,
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
71
72
73
|
"query_info": self.query_info,
"suggestions": self.suggestions,
"related_searches": self.related_searches
|
be52af70
tangwang
first commit
|
74
|
}
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
75
76
77
|
if self.debug_info is not None:
result["debug_info"] = self.debug_info
return result
|
be52af70
tangwang
first commit
|
78
79
80
81
82
83
84
85
|
class Searcher:
"""
Main search engine class.
Handles:
- Query parsing and translation
|
bd96cead
tangwang
1. 动态多语言字段与统一策略配置
|
86
|
- Dynamic multi-language text recall planning
|
be52af70
tangwang
first commit
|
87
88
89
90
91
92
|
- ES query building
- Result ranking and formatting
"""
def __init__(
self,
|
be52af70
tangwang
first commit
|
93
|
es_client: ESClient,
|
9f96d6f3
tangwang
短query不用语义搜索
|
94
|
config: SearchConfig,
|
26b910bd
tangwang
refactor service ...
|
95
96
|
query_parser: Optional[QueryParser] = None,
image_encoder: Optional[CLIPImageEncoder] = None,
|
be52af70
tangwang
first commit
|
97
98
99
100
101
|
):
"""
Initialize searcher.
Args:
|
be52af70
tangwang
first commit
|
102
|
es_client: Elasticsearch client
|
9f96d6f3
tangwang
短query不用语义搜索
|
103
|
config: SearchConfig instance
|
be52af70
tangwang
first commit
|
104
|
query_parser: Query parser (created if not provided)
|
26b910bd
tangwang
refactor service ...
|
105
|
image_encoder: Optional pre-initialized image encoder
|
be52af70
tangwang
first commit
|
106
|
"""
|
be52af70
tangwang
first commit
|
107
|
self.es_client = es_client
|
9f96d6f3
tangwang
短query不用语义搜索
|
108
|
self.config = config
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
109
|
# Index name is now generated dynamically per tenant, no longer stored here
|
9f96d6f3
tangwang
短query不用语义搜索
|
110
|
self.query_parser = query_parser or QueryParser(config)
|
9f96d6f3
tangwang
短query不用语义搜索
|
111
|
self.text_embedding_field = config.query_config.text_embedding_field or "title_embedding"
|
26b910bd
tangwang
refactor service ...
|
112
113
114
115
116
117
|
self.image_embedding_field = config.query_config.image_embedding_field
if self.image_embedding_field and image_encoder is None:
self.image_encoder = CLIPImageEncoder()
else:
self.image_encoder = image_encoder
self.source_fields = config.query_config.source_fields
|
be52af70
tangwang
first commit
|
118
|
|
f0d020c3
tangwang
多语言查询改为只支持中英文两种,f...
|
119
|
# Query builder - simplified single-layer architecture
|
f0d020c3
tangwang
多语言查询改为只支持中英文两种,f...
|
120
|
self.query_builder = ESQueryBuilder(
|
bd96cead
tangwang
1. 动态多语言字段与统一策略配置
|
121
122
123
124
125
|
match_fields=[],
field_boosts=self.config.field_boosts,
multilingual_fields=self.config.query_config.multilingual_fields,
shared_fields=self.config.query_config.shared_fields,
core_multilingual_fields=self.config.query_config.core_multilingual_fields,
|
be52af70
tangwang
first commit
|
126
|
text_embedding_field=self.text_embedding_field,
|
13377199
tangwang
接口优化
|
127
|
image_embedding_field=self.image_embedding_field,
|
9f96d6f3
tangwang
短query不用语义搜索
|
128
|
source_fields=self.source_fields,
|
7bc756c5
tangwang
优化 ES 查询构建
|
129
|
function_score_config=self.config.function_score,
|
70dab99f
tangwang
add logs
|
130
|
default_language=self.config.query_config.default_language,
|
bd96cead
tangwang
1. 动态多语言字段与统一策略配置
|
131
132
133
134
135
136
|
knn_boost=self.config.query_config.knn_boost,
base_minimum_should_match=self.config.query_config.base_minimum_should_match,
translation_minimum_should_match=self.config.query_config.translation_minimum_should_match,
translation_boost=self.config.query_config.translation_boost,
translation_boost_when_source_missing=self.config.query_config.translation_boost_when_source_missing,
source_boost_when_missing=self.config.query_config.source_boost_when_missing,
|
bcada818
tangwang
last
|
137
138
139
|
original_query_fallback_boost_when_translation_missing=(
self.config.query_config.original_query_fallback_boost_when_translation_missing
),
|
bd96cead
tangwang
1. 动态多语言字段与统一策略配置
|
140
|
tie_breaker_base_query=self.config.query_config.tie_breaker_base_query,
|
be52af70
tangwang
first commit
|
141
142
|
)
|
26b910bd
tangwang
refactor service ...
|
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
def _apply_source_filter(self, es_query: Dict[str, Any]) -> None:
"""
Apply tri-state _source semantics:
- None: do not set _source (return full source)
- []: _source=false (return no source fields)
- [..]: _source.includes=[..]
"""
if self.source_fields is None:
return
if not isinstance(self.source_fields, list):
raise ValueError("query_config.source_fields must be null or list[str]")
if len(self.source_fields) == 0:
es_query["_source"] = False
return
es_query["_source"] = {"includes": self.source_fields}
|
5f7d7f09
tangwang
性能测试报告.md
|
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
|
def _resolve_rerank_source_filter(self, doc_template: str) -> Dict[str, Any]:
"""
Build a lightweight _source filter for rerank prefetch.
Only fetch fields required by rerank doc template to reduce ES payload.
"""
field_map = {
"title": "title",
"brief": "brief",
"vendor": "vendor",
"description": "description",
"category_path": "category_path",
}
includes: set[str] = set()
template = str(doc_template or "{title}")
for _, field_name, _, _ in Formatter().parse(template):
if not field_name:
continue
key = field_name.split(".", 1)[0].split("!", 1)[0].split(":", 1)[0]
mapped = field_map.get(key)
if mapped:
includes.add(mapped)
# Fallback to title-only to keep rerank docs usable.
if not includes:
includes.add("title")
return {"includes": sorted(includes)}
def _fetch_hits_by_ids(
self,
index_name: str,
doc_ids: List[str],
source_spec: Optional[Any],
) -> tuple[Dict[str, Dict[str, Any]], int]:
"""
Fetch page documents by IDs for final response fill.
Returns:
(hits_by_id, es_took_ms)
"""
if not doc_ids:
return {}, 0
body: Dict[str, Any] = {
"query": {
"ids": {
"values": doc_ids,
}
}
}
if source_spec is not None:
body["_source"] = source_spec
resp = self.es_client.search(
index_name=index_name,
body=body,
size=len(doc_ids),
from_=0,
)
hits = resp.get("hits", {}).get("hits") or []
hits_by_id: Dict[str, Dict[str, Any]] = {}
for hit in hits:
hid = hit.get("_id")
if hid is None:
continue
hits_by_id[str(hid)] = hit
return hits_by_id, int(resp.get("took", 0) or 0)
|
deccd68a
tangwang
Added the SKU pre...
|
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
|
@staticmethod
def _normalize_sku_match_text(value: Optional[str]) -> str:
"""Normalize free text for lightweight SKU option matching."""
if value is None:
return ""
return " ".join(str(value).strip().casefold().split())
def _build_sku_query_texts(self, parsed_query: ParsedQuery) -> List[str]:
"""Collect original and translated query texts for SKU option matching."""
candidates: List[str] = []
for text in (
getattr(parsed_query, "original_query", None),
getattr(parsed_query, "query_normalized", None),
getattr(parsed_query, "rewritten_query", None),
):
normalized = self._normalize_sku_match_text(text)
if normalized:
candidates.append(normalized)
query_text_by_lang = getattr(parsed_query, "query_text_by_lang", {}) or {}
if isinstance(query_text_by_lang, dict):
for text in query_text_by_lang.values():
normalized = self._normalize_sku_match_text(text)
if normalized:
candidates.append(normalized)
translations = getattr(parsed_query, "translations", {}) or {}
if isinstance(translations, dict):
for text in translations.values():
normalized = self._normalize_sku_match_text(text)
if normalized:
candidates.append(normalized)
deduped: List[str] = []
seen = set()
for text in candidates:
if text in seen:
continue
seen.add(text)
deduped.append(text)
return deduped
def _find_query_matching_sku_index(
self,
skus: List[Dict[str, Any]],
query_texts: List[str],
) -> Optional[int]:
"""Return the first SKU whose option1_value appears in query texts."""
if not skus or not query_texts:
return None
for index, sku in enumerate(skus):
option1_value = self._normalize_sku_match_text(sku.get("option1_value"))
if not option1_value:
continue
if any(option1_value in query_text for query_text in query_texts):
return index
return None
def _encode_query_vector_for_sku_matching(
self,
parsed_query: ParsedQuery,
context: Optional[RequestContext] = None,
) -> Optional[np.ndarray]:
"""Best-effort fallback query embedding for final-page SKU matching."""
query_text = (
getattr(parsed_query, "rewritten_query", None)
or getattr(parsed_query, "query_normalized", None)
or getattr(parsed_query, "original_query", None)
)
if not query_text:
return None
text_encoder = getattr(self.query_parser, "text_encoder", None)
if text_encoder is None:
return None
try:
vectors = text_encoder.encode([query_text], priority=1)
except Exception as exc:
logger.warning("Failed to encode query vector for SKU matching: %s", exc, exc_info=True)
if context is not None:
context.add_warning(f"SKU query embedding failed: {exc}")
return None
if vectors is None or len(vectors) == 0:
return None
vector = vectors[0]
if vector is None:
return None
return np.asarray(vector, dtype=np.float32)
def _select_sku_by_embedding(
self,
skus: List[Dict[str, Any]],
option1_vectors: Dict[str, np.ndarray],
query_vector: np.ndarray,
) -> Tuple[Optional[int], Optional[float]]:
"""Select the SKU whose option1_value is most similar to the query."""
best_index: Optional[int] = None
best_score: Optional[float] = None
for index, sku in enumerate(skus):
option1_value_raw = sku.get("option1_value")
if option1_value_raw is None:
continue
option1_value = str(option1_value_raw).strip()
if not option1_value:
continue
option_vector = option1_vectors.get(option1_value)
if option_vector is None:
continue
score = float(np.inner(query_vector, option_vector))
if best_score is None or score > best_score:
best_index = index
best_score = score
return best_index, best_score
@staticmethod
def _promote_matching_sku(source: Dict[str, Any], match_index: int) -> Optional[Dict[str, Any]]:
"""Move the matched SKU to the front and swap the SPU image."""
skus = source.get("skus")
if not isinstance(skus, list) or match_index < 0 or match_index >= len(skus):
return None
matched_sku = skus.pop(match_index)
skus.insert(0, matched_sku)
image_src = matched_sku.get("image_src") or matched_sku.get("imageSrc")
if image_src:
source["image_url"] = image_src
return matched_sku
def _apply_sku_sorting_for_page_hits(
self,
es_hits: List[Dict[str, Any]],
parsed_query: ParsedQuery,
context: Optional[RequestContext] = None,
) -> None:
"""Sort each page hit's SKUs so the best-matching SKU is first."""
if not es_hits:
return
query_texts = self._build_sku_query_texts(parsed_query)
unmatched_hits: List[Dict[str, Any]] = []
option1_values_to_encode: List[str] = []
seen_option1_values = set()
text_matched = 0
embedding_matched = 0
for hit in es_hits:
source = hit.get("_source")
if not isinstance(source, dict):
continue
skus = source.get("skus")
if not isinstance(skus, list) or not skus:
continue
match_index = self._find_query_matching_sku_index(skus, query_texts)
if match_index is not None:
self._promote_matching_sku(source, match_index)
text_matched += 1
continue
unmatched_hits.append(hit)
for sku in skus:
option1_value_raw = sku.get("option1_value")
if option1_value_raw is None:
continue
option1_value = str(option1_value_raw).strip()
if not option1_value or option1_value in seen_option1_values:
continue
seen_option1_values.add(option1_value)
option1_values_to_encode.append(option1_value)
if not unmatched_hits or not option1_values_to_encode:
return
query_vector = getattr(parsed_query, "query_vector", None)
if query_vector is None:
query_vector = self._encode_query_vector_for_sku_matching(parsed_query, context=context)
if query_vector is None:
return
text_encoder = getattr(self.query_parser, "text_encoder", None)
if text_encoder is None:
return
try:
encoded_option_vectors = text_encoder.encode(option1_values_to_encode, priority=1)
except Exception as exc:
logger.warning("Failed to encode SKU option1 values for final-page sorting: %s", exc, exc_info=True)
if context is not None:
context.add_warning(f"SKU option embedding failed: {exc}")
return
option1_vectors: Dict[str, np.ndarray] = {}
for option1_value, vector in zip(option1_values_to_encode, encoded_option_vectors):
if vector is None:
continue
option1_vectors[option1_value] = np.asarray(vector, dtype=np.float32)
query_vector_array = np.asarray(query_vector, dtype=np.float32)
for hit in unmatched_hits:
source = hit.get("_source")
if not isinstance(source, dict):
continue
skus = source.get("skus")
if not isinstance(skus, list) or not skus:
continue
match_index, _ = self._select_sku_by_embedding(skus, option1_vectors, query_vector_array)
if match_index is None:
continue
self._promote_matching_sku(source, match_index)
embedding_matched += 1
if text_matched or embedding_matched:
logger.info(
"Final-page SKU sorting completed | text_matched=%s | embedding_matched=%s",
text_matched,
embedding_matched,
)
|
be52af70
tangwang
first commit
|
453
454
455
|
def search(
self,
query: str,
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
456
|
tenant_id: str,
|
be52af70
tangwang
first commit
|
457
458
459
|
size: int = 10,
from_: int = 0,
filters: Optional[Dict[str, Any]] = None,
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
460
|
range_filters: Optional[Dict[str, Any]] = None,
|
13320ac6
tangwang
分面接口修改:
|
461
|
facets: Optional[List[FacetConfig]] = None,
|
16c42787
tangwang
feat: implement r...
|
462
|
min_score: Optional[float] = None,
|
c86c8237
tangwang
支持聚合。过滤项补充了逻辑,但是有问题
|
463
|
context: Optional[RequestContext] = None,
|
c86c8237
tangwang
支持聚合。过滤项补充了逻辑,但是有问题
|
464
|
sort_by: Optional[str] = None,
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
465
|
sort_order: Optional[str] = "desc",
|
577ec972
tangwang
返回给前端的字段、格式适配。主要包...
|
466
|
debug: bool = False,
|
2739b281
tangwang
多语言索引调整
|
467
|
language: str = "en",
|
a3a5d41b
tangwang
(sku_filter_dimen...
|
468
|
sku_filter_dimension: Optional[List[str]] = None,
|
5f7d7f09
tangwang
性能测试报告.md
|
469
|
enable_rerank: Optional[bool] = None,
|
ff32d894
tangwang
rerank
|
470
471
|
rerank_query_template: Optional[str] = None,
rerank_doc_template: Optional[str] = None,
|
be52af70
tangwang
first commit
|
472
473
|
) -> SearchResult:
"""
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
474
|
Execute search query (外部友好格式).
|
be52af70
tangwang
first commit
|
475
476
477
|
Args:
query: Search query string
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
478
|
tenant_id: Tenant ID (required for filtering)
|
be52af70
tangwang
first commit
|
479
480
|
size: Number of results to return
from_: Offset for pagination
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
481
482
483
|
filters: Exact match filters
range_filters: Range filters for numeric fields
facets: Facet configurations for faceted search
|
be52af70
tangwang
first commit
|
484
|
min_score: Minimum score threshold
|
16c42787
tangwang
feat: implement r...
|
485
|
context: Request context for tracking (created if not provided)
|
c86c8237
tangwang
支持聚合。过滤项补充了逻辑,但是有问题
|
486
487
|
sort_by: Field name for sorting
sort_order: Sort order: 'asc' or 'desc'
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
488
|
debug: Enable debug information output
|
be52af70
tangwang
first commit
|
489
490
|
Returns:
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
491
|
SearchResult object with formatted results
|
be52af70
tangwang
first commit
|
492
|
"""
|
16c42787
tangwang
feat: implement r...
|
493
|
if context is None:
|
ed948666
tangwang
tidy
|
494
|
raise ValueError("context is required")
|
16c42787
tangwang
feat: implement r...
|
495
|
|
345d960b
tangwang
1. 删除全局 enable_tr...
|
496
497
498
|
# 根据租户配置决定翻译开关(离线/在线统一)
tenant_loader = get_tenant_config_loader()
tenant_cfg = tenant_loader.get_tenant_config(tenant_id)
|
038e4e2f
tangwang
refactor(i18n): t...
|
499
500
|
index_langs = tenant_cfg.get("index_languages") or []
enable_translation = len(index_langs) > 0
|
9f96d6f3
tangwang
短query不用语义搜索
|
501
|
enable_embedding = self.config.query_config.enable_text_embedding
|
5f7d7f09
tangwang
性能测试报告.md
|
502
503
504
505
506
507
|
rc = self.config.rerank
effective_query_template = rerank_query_template or rc.rerank_query_template
effective_doc_template = rerank_doc_template or rc.rerank_doc_template
# 重排开关优先级:请求参数显式传值 > 服务端配置(默认开启)
rerank_enabled_by_config = bool(rc.enabled)
do_rerank = rerank_enabled_by_config if enable_rerank is None else bool(enable_rerank)
|
c51d254f
tangwang
性能测试
|
508
|
rerank_window = rc.rerank_window
|
506c39b7
tangwang
feat(search): 统一重...
|
509
|
# 若开启重排且请求范围在窗口内:从 ES 取前 rerank_window 条、重排后再按 from/size 分页;否则不重排,按原 from/size 查 ES
|
ff32d894
tangwang
rerank
|
510
|
in_rerank_window = do_rerank and (from_ + size) <= rerank_window
|
506c39b7
tangwang
feat(search): 统一重...
|
511
512
|
es_fetch_from = 0 if in_rerank_window else from_
es_fetch_size = rerank_window if in_rerank_window else size
|
16c42787
tangwang
feat: implement r...
|
513
514
515
516
517
518
|
# Start timing
context.start_stage(RequestContextStage.TOTAL)
context.logger.info(
f"开始搜索请求 | 查询: '{query}' | 参数: size={size}, from_={from_}, "
|
5f7d7f09
tangwang
性能测试报告.md
|
519
520
521
|
f"enable_rerank(request)={enable_rerank}, enable_rerank(config)={rerank_enabled_by_config}, "
f"enable_rerank(effective)={do_rerank}, in_rerank_window={in_rerank_window}, "
f"es_fetch=({es_fetch_from},{es_fetch_size}) | "
|
506c39b7
tangwang
feat(search): 统一重...
|
522
|
f"enable_translation={enable_translation}, enable_embedding={enable_embedding}, min_score={min_score}",
|
16c42787
tangwang
feat: implement r...
|
523
524
525
526
527
528
529
|
extra={'reqid': context.reqid, 'uid': context.uid}
)
# Store search parameters in context
context.metadata['search_params'] = {
'size': size,
'from_': from_,
|
506c39b7
tangwang
feat(search): 统一重...
|
530
531
532
|
'es_fetch_from': es_fetch_from,
'es_fetch_size': es_fetch_size,
'in_rerank_window': in_rerank_window,
|
5f7d7f09
tangwang
性能测试报告.md
|
533
534
535
536
|
'rerank_enabled_by_config': rerank_enabled_by_config,
'enable_rerank_request': enable_rerank,
'rerank_query_template': effective_query_template,
'rerank_doc_template': effective_doc_template,
|
16c42787
tangwang
feat: implement r...
|
537
|
'filters': filters,
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
538
539
|
'range_filters': range_filters,
'facets': facets,
|
16c42787
tangwang
feat: implement r...
|
540
541
|
'enable_translation': enable_translation,
'enable_embedding': enable_embedding,
|
ff32d894
tangwang
rerank
|
542
|
'enable_rerank': do_rerank,
|
c86c8237
tangwang
支持聚合。过滤项补充了逻辑,但是有问题
|
543
|
'min_score': min_score,
|
c86c8237
tangwang
支持聚合。过滤项补充了逻辑,但是有问题
|
544
545
|
'sort_by': sort_by,
'sort_order': sort_order
|
16c42787
tangwang
feat: implement r...
|
546
|
}
|
be52af70
tangwang
first commit
|
547
|
|
16c42787
tangwang
feat: implement r...
|
548
549
550
|
context.metadata['feature_flags'] = {
'translation_enabled': enable_translation,
'embedding_enabled': enable_embedding,
|
ff32d894
tangwang
rerank
|
551
|
'rerank_enabled': do_rerank
|
16c42787
tangwang
feat: implement r...
|
552
|
}
|
be52af70
tangwang
first commit
|
553
554
|
# Step 1: Parse query
|
16c42787
tangwang
feat: implement r...
|
555
556
557
558
|
context.start_stage(RequestContextStage.QUERY_PARSING)
try:
parsed_query = self.query_parser.parse(
query,
|
345d960b
tangwang
1. 删除全局 enable_tr...
|
559
|
tenant_id=tenant_id,
|
16c42787
tangwang
feat: implement r...
|
560
561
562
563
564
565
|
generate_vector=enable_embedding,
context=context
)
# Store query analysis results in context
context.store_query_analysis(
original_query=parsed_query.original_query,
|
3a5fda00
tangwang
1. ES字段 skus的 ima...
|
566
|
query_normalized=parsed_query.query_normalized,
|
16c42787
tangwang
feat: implement r...
|
567
568
569
570
571
|
rewritten_query=parsed_query.rewritten_query,
detected_language=parsed_query.detected_language,
translations=parsed_query.translations,
query_vector=parsed_query.query_vector.tolist() if parsed_query.query_vector is not None else None,
domain=parsed_query.domain,
|
bd96cead
tangwang
1. 动态多语言字段与统一策略配置
|
572
|
is_simple_query=True
|
16c42787
tangwang
feat: implement r...
|
573
|
)
|
be52af70
tangwang
first commit
|
574
|
|
16c42787
tangwang
feat: implement r...
|
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
|
context.logger.info(
f"查询解析完成 | 原查询: '{parsed_query.original_query}' | "
f"重写后: '{parsed_query.rewritten_query}' | "
f"语言: {parsed_query.detected_language} | "
f"域: {parsed_query.domain} | "
f"向量: {'是' if parsed_query.query_vector is not None else '否'}",
extra={'reqid': context.reqid, 'uid': context.uid}
)
except Exception as e:
context.set_error(e)
context.logger.error(
f"查询解析失败 | 错误: {str(e)}",
extra={'reqid': context.reqid, 'uid': context.uid}
)
raise
finally:
context.end_stage(RequestContextStage.QUERY_PARSING)
|
bd96cead
tangwang
1. 动态多语言字段与统一策略配置
|
593
|
# Step 2: Query building
|
16c42787
tangwang
feat: implement r...
|
594
595
|
context.start_stage(RequestContextStage.QUERY_BUILDING)
try:
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
596
597
|
# Generate tenant-specific index name
index_name = get_tenant_index_name(tenant_id)
|
2739b281
tangwang
多语言索引调整
|
598
|
# index_name = "search_products"
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
599
600
|
# No longer need to add tenant_id to filters since each tenant has its own index
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
601
|
|
f0d020c3
tangwang
多语言查询改为只支持中英文两种,f...
|
602
|
es_query = self.query_builder.build_query(
|
3a5fda00
tangwang
1. ES字段 skus的 ima...
|
603
|
query_text=parsed_query.rewritten_query or parsed_query.query_normalized,
|
16c42787
tangwang
feat: implement r...
|
604
|
query_vector=parsed_query.query_vector if enable_embedding else None,
|
16c42787
tangwang
feat: implement r...
|
605
|
filters=filters,
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
606
|
range_filters=range_filters,
|
c581becd
tangwang
feat: 实现 Multi-Se...
|
607
|
facet_configs=facets,
|
506c39b7
tangwang
feat(search): 统一重...
|
608
609
|
size=es_fetch_size,
from_=es_fetch_from,
|
16c42787
tangwang
feat: implement r...
|
610
|
enable_knn=enable_embedding and parsed_query.query_vector is not None,
|
7bc756c5
tangwang
优化 ES 查询构建
|
611
612
|
min_score=min_score,
parsed_query=parsed_query
|
16c42787
tangwang
feat: implement r...
|
613
|
)
|
be52af70
tangwang
first commit
|
614
|
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
615
616
617
618
619
620
621
|
# Add facets for faceted search
if facets:
facet_aggs = self.query_builder.build_facets(facets)
if facet_aggs:
if "aggs" not in es_query:
es_query["aggs"] = {}
es_query["aggs"].update(facet_aggs)
|
16c42787
tangwang
feat: implement r...
|
622
|
|
c86c8237
tangwang
支持聚合。过滤项补充了逻辑,但是有问题
|
623
624
625
|
# Add sorting if specified
if sort_by:
es_query = self.query_builder.add_sorting(es_query, sort_by, sort_order)
|
a47416ec
tangwang
把融合逻辑改成乘法公式,并把 ES...
|
626
|
es_query["track_scores"] = True
|
c86c8237
tangwang
支持聚合。过滤项补充了逻辑,但是有问题
|
627
|
|
5f7d7f09
tangwang
性能测试报告.md
|
628
629
630
631
632
633
634
635
636
637
638
|
# Keep requested response _source semantics for the final response fill.
response_source_spec = es_query.get("_source")
# In rerank window, first pass only fetches minimal fields required by rerank template.
es_query_for_fetch = es_query
rerank_prefetch_source = None
if in_rerank_window:
rerank_prefetch_source = self._resolve_rerank_source_filter(effective_doc_template)
es_query_for_fetch = dict(es_query)
es_query_for_fetch["_source"] = rerank_prefetch_source
|
16c42787
tangwang
feat: implement r...
|
639
|
# Extract size and from from body for ES client parameters
|
5f7d7f09
tangwang
性能测试报告.md
|
640
|
body_for_es = {k: v for k, v in es_query_for_fetch.items() if k not in ['size', 'from']}
|
16c42787
tangwang
feat: implement r...
|
641
642
643
|
# Store ES query in context
context.store_intermediate_result('es_query', es_query)
|
5f7d7f09
tangwang
性能测试报告.md
|
644
645
|
if in_rerank_window and rerank_prefetch_source is not None:
context.store_intermediate_result('es_query_rerank_prefetch_source', rerank_prefetch_source)
|
16c42787
tangwang
feat: implement r...
|
646
647
|
context.store_intermediate_result('es_body_for_search', body_for_es)
|
28e57bb1
tangwang
日志体系优化
|
648
|
# Serialize ES query to compute a compact size + stable digest for correlation
|
5f7d7f09
tangwang
性能测试报告.md
|
649
|
es_query_compact = json.dumps(es_query_for_fetch, ensure_ascii=False, separators=(",", ":"))
|
28e57bb1
tangwang
日志体系优化
|
650
651
652
|
es_query_digest = hashlib.sha256(es_query_compact.encode("utf-8")).hexdigest()[:16]
knn_enabled = bool(enable_embedding and parsed_query.query_vector is not None)
vector_dims = int(len(parsed_query.query_vector)) if parsed_query.query_vector is not None else 0
|
99bea633
tangwang
add logs
|
653
|
|
16c42787
tangwang
feat: implement r...
|
654
|
context.logger.info(
|
5f7d7f09
tangwang
性能测试报告.md
|
655
|
"ES query built | size: %s chars | digest: %s | KNN: %s | vector_dims: %s | facets: %s | rerank_prefetch_source: %s",
|
28e57bb1
tangwang
日志体系优化
|
656
657
658
659
660
|
len(es_query_compact),
es_query_digest,
"yes" if knn_enabled else "no",
vector_dims,
"yes" if facets else "no",
|
5f7d7f09
tangwang
性能测试报告.md
|
661
|
rerank_prefetch_source,
|
16c42787
tangwang
feat: implement r...
|
662
663
|
extra={'reqid': context.reqid, 'uid': context.uid}
)
|
28e57bb1
tangwang
日志体系优化
|
664
665
666
667
668
669
670
671
672
673
|
_log_backend_verbose({
"event": "es_query_built",
"reqid": context.reqid,
"uid": context.uid,
"tenant_id": tenant_id,
"size_chars": len(es_query_compact),
"sha256_16": es_query_digest,
"knn_enabled": knn_enabled,
"vector_dims": vector_dims,
"has_facets": bool(facets),
|
5f7d7f09
tangwang
性能测试报告.md
|
674
|
"query": es_query_for_fetch,
|
28e57bb1
tangwang
日志体系优化
|
675
|
})
|
16c42787
tangwang
feat: implement r...
|
676
677
678
679
680
681
682
683
684
685
|
except Exception as e:
context.set_error(e)
context.logger.error(
f"ES查询构建失败 | 错误: {str(e)}",
extra={'reqid': context.reqid, 'uid': context.uid}
)
raise
finally:
context.end_stage(RequestContextStage.QUERY_BUILDING)
|
a99e62ba
tangwang
记录各阶段耗时
|
686
687
|
# Step 4: Elasticsearch search (primary recall)
context.start_stage(RequestContextStage.ELASTICSEARCH_SEARCH_PRIMARY)
|
16c42787
tangwang
feat: implement r...
|
688
|
try:
|
506c39b7
tangwang
feat(search): 统一重...
|
689
|
# Use tenant-specific index name(开启重排且在窗口内时已用 es_fetch_size/es_fetch_from)
|
16c42787
tangwang
feat: implement r...
|
690
|
es_response = self.es_client.search(
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
691
|
index_name=index_name,
|
16c42787
tangwang
feat: implement r...
|
692
|
body=body_for_es,
|
506c39b7
tangwang
feat(search): 统一重...
|
693
|
size=es_fetch_size,
|
a47416ec
tangwang
把融合逻辑改成乘法公式,并把 ES...
|
694
695
|
from_=es_fetch_from,
include_named_queries_score=bool(do_rerank and in_rerank_window),
|
be52af70
tangwang
first commit
|
696
697
|
)
|
16c42787
tangwang
feat: implement r...
|
698
699
|
# Store ES response in context
context.store_intermediate_result('es_response', es_response)
|
be52af70
tangwang
first commit
|
700
|
|
16c42787
tangwang
feat: implement r...
|
701
702
703
704
705
|
# Extract timing from ES response
es_took = es_response.get('took', 0)
context.logger.info(
f"ES搜索完成 | 耗时: {es_took}ms | "
f"命中数: {es_response.get('hits', {}).get('total', {}).get('value', 0)} | "
|
25d3e81d
tangwang
fix指定sort项时候的bug
|
706
|
f"最高分: {(es_response.get('hits', {}).get('max_score') or 0):.3f}",
|
16c42787
tangwang
feat: implement r...
|
707
708
709
710
711
712
713
714
715
716
|
extra={'reqid': context.reqid, 'uid': context.uid}
)
except Exception as e:
context.set_error(e)
context.logger.error(
f"ES搜索执行失败 | 错误: {str(e)}",
extra={'reqid': context.reqid, 'uid': context.uid}
)
raise
finally:
|
a99e62ba
tangwang
记录各阶段耗时
|
717
|
context.end_stage(RequestContextStage.ELASTICSEARCH_SEARCH_PRIMARY)
|
16c42787
tangwang
feat: implement r...
|
718
|
|
506c39b7
tangwang
feat(search): 统一重...
|
719
|
# Optional Step 4.5: AI reranking(仅当请求范围在重排窗口内时执行)
|
ff32d894
tangwang
rerank
|
720
|
if do_rerank and in_rerank_window:
|
506c39b7
tangwang
feat(search): 统一重...
|
721
722
723
724
725
|
context.start_stage(RequestContextStage.RERANKING)
try:
from .rerank_client import run_rerank
rerank_query = parsed_query.original_query if parsed_query else query
|
506c39b7
tangwang
feat(search): 统一重...
|
726
727
728
729
|
es_response, rerank_meta, fused_debug = run_rerank(
query=rerank_query,
es_response=es_response,
language=language,
|
506c39b7
tangwang
feat(search): 统一重...
|
730
731
732
|
timeout_sec=rc.timeout_sec,
weight_es=rc.weight_es,
weight_ai=rc.weight_ai,
|
ff32d894
tangwang
rerank
|
733
734
|
rerank_query_template=effective_query_template,
rerank_doc_template=effective_doc_template,
|
d31c7f65
tangwang
补充云服务reranker
|
735
|
top_n=(from_ + size),
|
506c39b7
tangwang
feat(search): 统一重...
|
736
737
738
|
)
if rerank_meta is not None:
|
42e3aea6
tangwang
tidy
|
739
740
|
from config.services_config import get_rerank_service_url
rerank_url = get_rerank_service_url()
|
506c39b7
tangwang
feat(search): 统一重...
|
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
|
context.metadata.setdefault("rerank_info", {})
context.metadata["rerank_info"].update({
"service_url": rerank_url,
"docs": len(es_response.get("hits", {}).get("hits") or []),
"meta": rerank_meta,
})
context.store_intermediate_result("rerank_scores", fused_debug)
context.logger.info(
f"重排完成 | docs={len(fused_debug)} | meta={rerank_meta}",
extra={'reqid': context.reqid, 'uid': context.uid}
)
except Exception as e:
context.add_warning(f"Rerank failed: {e}")
context.logger.warning(
f"调用重排服务失败 | error: {e}",
extra={'reqid': context.reqid, 'uid': context.uid},
exc_info=True,
)
finally:
context.end_stage(RequestContextStage.RERANKING)
# 当本次请求在重排窗口内时:已从 ES 取了 rerank_window 条并可能已重排,需按请求的 from/size 做分页切片
if in_rerank_window:
hits = es_response.get("hits", {}).get("hits") or []
sliced = hits[from_ : from_ + size]
es_response.setdefault("hits", {})["hits"] = sliced
if sliced:
|
af827ce9
tangwang
rerank
|
768
769
770
771
772
|
# 对于启用重排的结果,优先使用 _fused_score 计算 max_score;否则退回原始 _score
slice_max = max(
(h.get("_fused_score", h.get("_score", 0.0)) for h in sliced),
default=0.0,
)
|
506c39b7
tangwang
feat(search): 统一重...
|
773
774
775
776
777
778
|
try:
es_response["hits"]["max_score"] = float(slice_max)
except (TypeError, ValueError):
es_response["hits"]["max_score"] = 0.0
else:
es_response["hits"]["max_score"] = 0.0
|
5f7d7f09
tangwang
性能测试报告.md
|
779
780
781
782
783
784
785
786
787
788
789
|
# Page fill: fetch detailed fields only for final page hits.
if sliced:
if response_source_spec is False:
for hit in sliced:
hit.pop("_source", None)
context.logger.info(
"分页详情回填跳过 | 原查询 _source=false",
extra={'reqid': context.reqid, 'uid': context.uid}
)
else:
|
a99e62ba
tangwang
记录各阶段耗时
|
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
|
context.start_stage(RequestContextStage.ELASTICSEARCH_PAGE_FILL)
try:
page_ids = [str(h.get("_id")) for h in sliced if h.get("_id") is not None]
details_by_id, fill_took = self._fetch_hits_by_ids(
index_name=index_name,
doc_ids=page_ids,
source_spec=response_source_spec,
)
filled = 0
for hit in sliced:
hid = hit.get("_id")
if hid is None:
continue
detail_hit = details_by_id.get(str(hid))
if detail_hit is None:
continue
if "_source" in detail_hit:
hit["_source"] = detail_hit.get("_source") or {}
filled += 1
if fill_took:
es_response["took"] = int((es_response.get("took", 0) or 0) + fill_took)
context.logger.info(
f"分页详情回填 | ids={len(page_ids)} | filled={filled} | took={fill_took}ms",
extra={'reqid': context.reqid, 'uid': context.uid}
)
finally:
context.end_stage(RequestContextStage.ELASTICSEARCH_PAGE_FILL)
|
5f7d7f09
tangwang
性能测试报告.md
|
817
|
|
506c39b7
tangwang
feat(search): 统一重...
|
818
819
820
821
822
|
context.logger.info(
f"重排分页切片 | from={from_}, size={size}, 返回={len(sliced)}条",
extra={'reqid': context.reqid, 'uid': context.uid}
)
|
16c42787
tangwang
feat: implement r...
|
823
824
825
|
# Step 5: Result processing
context.start_stage(RequestContextStage.RESULT_PROCESSING)
try:
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
826
827
|
# Extract ES hits
es_hits = []
|
16c42787
tangwang
feat: implement r...
|
828
|
if 'hits' in es_response and 'hits' in es_response['hits']:
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
829
|
es_hits = es_response['hits']['hits']
|
16c42787
tangwang
feat: implement r...
|
830
831
832
833
834
835
|
# Extract total and max_score
total = es_response.get('hits', {}).get('total', {})
if isinstance(total, dict):
total_value = total.get('value', 0)
else:
total_value = total
|
506c39b7
tangwang
feat(search): 统一重...
|
836
|
# max_score 会在启用 AI 搜索时被更新为融合分数的最大值
|
25d3e81d
tangwang
fix指定sort项时候的bug
|
837
|
max_score = es_response.get('hits', {}).get('max_score') or 0.0
|
be52af70
tangwang
first commit
|
838
|
|
af827ce9
tangwang
rerank
|
839
840
841
842
843
844
845
846
847
848
849
850
|
# 从上下文中取出重排调试信息(若有)
rerank_debug_raw = context.get_intermediate_result('rerank_scores', None)
rerank_debug_by_doc: Dict[str, Dict[str, Any]] = {}
if isinstance(rerank_debug_raw, list):
for item in rerank_debug_raw:
if not isinstance(item, dict):
continue
doc_id = item.get("doc_id")
if doc_id is None:
continue
rerank_debug_by_doc[str(doc_id)] = item
|
deccd68a
tangwang
Added the SKU pre...
|
851
852
|
self._apply_sku_sorting_for_page_hits(es_hits, parsed_query, context=context)
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
853
|
# Format results using ResultFormatter
|
577ec972
tangwang
返回给前端的字段、格式适配。主要包...
|
854
855
856
|
formatted_results = ResultFormatter.format_search_results(
es_hits,
max_score,
|
ca91352a
tangwang
更新文档
|
857
858
|
language=language,
sku_filter_dimension=sku_filter_dimension
|
577ec972
tangwang
返回给前端的字段、格式适配。主要包...
|
859
|
)
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
860
|
|
985752f5
tangwang
1. 前端调试功能
|
861
862
863
864
865
|
# Build per-result debug info (per SPU) when debug mode is enabled
per_result_debug = []
if debug and es_hits and formatted_results:
for hit, spu in zip(es_hits, formatted_results):
source = hit.get("_source", {}) or {}
|
af827ce9
tangwang
rerank
|
866
867
868
869
870
|
doc_id = hit.get("_id")
rerank_debug = None
if doc_id is not None:
rerank_debug = rerank_debug_by_doc.get(str(doc_id))
|
985752f5
tangwang
1. 前端调试功能
|
871
872
873
874
875
876
877
878
879
880
881
882
883
884
|
raw_score = hit.get("_score")
try:
es_score = float(raw_score) if raw_score is not None else 0.0
except (TypeError, ValueError):
es_score = 0.0
try:
normalized = float(es_score) / float(max_score) if max_score else None
except (TypeError, ValueError, ZeroDivisionError):
normalized = None
title_multilingual = source.get("title") if isinstance(source.get("title"), dict) else None
brief_multilingual = source.get("brief") if isinstance(source.get("brief"), dict) else None
vendor_multilingual = source.get("vendor") if isinstance(source.get("vendor"), dict) else None
|
af827ce9
tangwang
rerank
|
885
886
887
888
889
890
891
892
893
894
895
896
897
|
debug_entry: Dict[str, Any] = {
"spu_id": spu.spu_id,
"es_score": es_score,
"es_score_normalized": normalized,
"title_multilingual": title_multilingual,
"brief_multilingual": brief_multilingual,
"vendor_multilingual": vendor_multilingual,
}
# 若存在重排调试信息,则补充 doc 级别的融合分数信息
if rerank_debug:
debug_entry["doc_id"] = rerank_debug.get("doc_id")
# 与 rerank_client 中字段保持一致,便于前端直接使用
|
af827ce9
tangwang
rerank
|
898
|
debug_entry["rerank_score"] = rerank_debug.get("rerank_score")
|
a8261ece
tangwang
检索效果优化
|
899
|
debug_entry["text_score"] = rerank_debug.get("text_score")
|
c90f80ed
tangwang
相关性优化
|
900
901
902
903
904
|
debug_entry["text_source_score"] = rerank_debug.get("text_source_score")
debug_entry["text_translation_score"] = rerank_debug.get("text_translation_score")
debug_entry["text_fallback_score"] = rerank_debug.get("text_fallback_score")
debug_entry["text_primary_score"] = rerank_debug.get("text_primary_score")
debug_entry["text_support_score"] = rerank_debug.get("text_support_score")
|
a8261ece
tangwang
检索效果优化
|
905
|
debug_entry["knn_score"] = rerank_debug.get("knn_score")
|
af827ce9
tangwang
rerank
|
906
|
debug_entry["fused_score"] = rerank_debug.get("fused_score")
|
a8261ece
tangwang
检索效果优化
|
907
|
debug_entry["matched_queries"] = rerank_debug.get("matched_queries")
|
af827ce9
tangwang
rerank
|
908
909
|
per_result_debug.append(debug_entry)
|
985752f5
tangwang
1. 前端调试功能
|
910
|
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
911
912
913
914
915
|
# Format facets
standardized_facets = None
if facets:
standardized_facets = ResultFormatter.format_facets(
es_response.get('aggregations', {}),
|
c581becd
tangwang
feat: 实现 Multi-Se...
|
916
917
|
facets,
filters
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
918
919
920
921
922
923
|
)
# Generate suggestions and related searches
query_text = parsed_query.original_query if parsed_query else query
suggestions = ResultFormatter.generate_suggestions(query_text, formatted_results)
related_searches = ResultFormatter.generate_related_searches(query_text, formatted_results)
|
be52af70
tangwang
first commit
|
924
|
|
16c42787
tangwang
feat: implement r...
|
925
|
context.logger.info(
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
926
|
f"结果处理完成 | 返回: {len(formatted_results)}条 | 总计: {total_value}条",
|
16c42787
tangwang
feat: implement r...
|
927
928
929
930
931
932
933
934
935
936
937
938
|
extra={'reqid': context.reqid, 'uid': context.uid}
)
except Exception as e:
context.set_error(e)
context.logger.error(
f"结果处理失败 | 错误: {str(e)}",
extra={'reqid': context.reqid, 'uid': context.uid}
)
raise
finally:
context.end_stage(RequestContextStage.RESULT_PROCESSING)
|
be52af70
tangwang
first commit
|
939
|
|
16c42787
tangwang
feat: implement r...
|
940
941
942
|
# End total timing and build result
total_duration = context.end_stage(RequestContextStage.TOTAL)
context.performance_metrics.total_duration = total_duration
|
be52af70
tangwang
first commit
|
943
|
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
944
945
946
947
948
949
|
# Collect debug information if requested
debug_info = None
if debug:
debug_info = {
"query_analysis": {
"original_query": context.query_analysis.original_query,
|
3a5fda00
tangwang
1. ES字段 skus的 ima...
|
950
|
"query_normalized": context.query_analysis.query_normalized,
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
951
952
953
|
"rewritten_query": context.query_analysis.rewritten_query,
"detected_language": context.query_analysis.detected_language,
"translations": context.query_analysis.translations,
|
a8261ece
tangwang
检索效果优化
|
954
955
956
|
"query_text_by_lang": context.get_intermediate_result("query_text_by_lang", {}),
"search_langs": context.get_intermediate_result("search_langs", []),
"supplemental_search_langs": context.get_intermediate_result("supplemental_search_langs", []),
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
957
958
|
"has_vector": context.query_analysis.query_vector is not None,
"is_simple_query": context.query_analysis.is_simple_query,
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
|
"domain": context.query_analysis.domain
},
"es_query": context.get_intermediate_result('es_query', {}),
"es_response": {
"took_ms": es_response.get('took', 0),
"total_hits": total_value,
"max_score": max_score,
"shards": es_response.get('_shards', {})
},
"feature_flags": context.metadata.get('feature_flags', {}),
"stage_timings": {
k: round(v, 2) for k, v in context.performance_metrics.stage_timings.items()
},
"search_params": context.metadata.get('search_params', {})
}
|
985752f5
tangwang
1. 前端调试功能
|
974
975
|
if per_result_debug:
debug_info["per_result"] = per_result_debug
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
976
|
|
be52af70
tangwang
first commit
|
977
978
|
# Build result
result = SearchResult(
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
979
|
results=formatted_results,
|
be52af70
tangwang
first commit
|
980
981
|
total=total_value,
max_score=max_score,
|
16c42787
tangwang
feat: implement r...
|
982
|
took_ms=int(total_duration),
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
983
|
facets=standardized_facets,
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
984
|
query_info=parsed_query.to_dict(),
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
985
986
|
suggestions=suggestions,
related_searches=related_searches,
|
1f071951
tangwang
补充调试信息,记录包括各个阶段的 ...
|
987
|
debug_info=debug_info
|
be52af70
tangwang
first commit
|
988
989
|
)
|
16c42787
tangwang
feat: implement r...
|
990
991
|
# Log complete performance summary
context.log_performance_summary()
|
be52af70
tangwang
first commit
|
992
993
994
995
996
997
|
return result
def search_by_image(
self,
image_url: str,
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
998
|
tenant_id: str,
|
be52af70
tangwang
first commit
|
999
|
size: int = 10,
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
1000
1001
|
filters: Optional[Dict[str, Any]] = None,
range_filters: Optional[Dict[str, Any]] = None
|
be52af70
tangwang
first commit
|
1002
1003
|
) -> SearchResult:
"""
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1004
|
Search by image similarity (外部友好格式).
|
be52af70
tangwang
first commit
|
1005
1006
1007
|
Args:
image_url: URL of query image
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1008
|
tenant_id: Tenant ID (required for filtering)
|
be52af70
tangwang
first commit
|
1009
|
size: Number of results
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
1010
1011
|
filters: Exact match filters
range_filters: Range filters for numeric fields
|
be52af70
tangwang
first commit
|
1012
1013
|
Returns:
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1014
|
SearchResult object with formatted results
|
be52af70
tangwang
first commit
|
1015
1016
1017
1018
1019
|
"""
if not self.image_embedding_field:
raise ValueError("Image embedding field not configured")
# Generate image embedding
|
26b910bd
tangwang
refactor service ...
|
1020
1021
|
if self.image_encoder is None:
raise RuntimeError("Image encoder is not initialized at startup")
|
b754fd41
tangwang
图片向量化支持优先级参数
|
1022
|
image_vector = self.image_encoder.encode_image_from_url(image_url, priority=1)
|
be52af70
tangwang
first commit
|
1023
1024
1025
1026
|
if image_vector is None:
raise ValueError(f"Failed to encode image: {image_url}")
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
1027
1028
1029
1030
|
# Generate tenant-specific index name
index_name = get_tenant_index_name(tenant_id)
# No longer need to add tenant_id to filters since each tenant has its own index
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1031
|
|
be52af70
tangwang
first commit
|
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
|
# Build KNN query
es_query = {
"size": size,
"knn": {
"field": self.image_embedding_field,
"query_vector": image_vector.tolist(),
"k": size,
"num_candidates": size * 10
}
}
|
26b910bd
tangwang
refactor service ...
|
1043
1044
|
# Apply source filtering semantics (None / [] / list)
self._apply_source_filter(es_query)
|
13377199
tangwang
接口优化
|
1045
|
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
1046
1047
1048
|
if filters or range_filters:
filter_clauses = self.query_builder._build_filters(filters, range_filters)
if filter_clauses:
|
7fbca0d7
tangwang
启动脚本优化
|
1049
1050
1051
1052
1053
1054
1055
|
if len(filter_clauses) == 1:
es_query["knn"]["filter"] = filter_clauses[0]
else:
es_query["knn"]["filter"] = {
"bool": {
"filter": filter_clauses
}
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
1056
|
}
|
be52af70
tangwang
first commit
|
1057
1058
1059
|
# Execute search
es_response = self.es_client.search(
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
1060
|
index_name=index_name,
|
be52af70
tangwang
first commit
|
1061
1062
1063
1064
|
body=es_query,
size=size
)
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1065
1066
|
# Extract ES hits
es_hits = []
|
be52af70
tangwang
first commit
|
1067
|
if 'hits' in es_response and 'hits' in es_response['hits']:
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1068
|
es_hits = es_response['hits']['hits']
|
be52af70
tangwang
first commit
|
1069
|
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1070
|
# Extract total and max_score
|
be52af70
tangwang
first commit
|
1071
1072
1073
1074
1075
1076
|
total = es_response.get('hits', {}).get('total', {})
if isinstance(total, dict):
total_value = total.get('value', 0)
else:
total_value = total
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1077
1078
1079
|
max_score = es_response.get('hits', {}).get('max_score') or 0.0
# Format results using ResultFormatter
|
ca91352a
tangwang
更新文档
|
1080
1081
1082
|
formatted_results = ResultFormatter.format_search_results(
es_hits,
max_score,
|
2739b281
tangwang
多语言索引调整
|
1083
|
language="en", # Default language for image search
|
ca91352a
tangwang
更新文档
|
1084
1085
|
sku_filter_dimension=None # Image search doesn't support SKU filtering
)
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1086
|
|
be52af70
tangwang
first commit
|
1087
|
return SearchResult(
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1088
|
results=formatted_results,
|
be52af70
tangwang
first commit
|
1089
|
total=total_value,
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1090
|
max_score=max_score,
|
be52af70
tangwang
first commit
|
1091
|
took_ms=es_response.get('took', 0),
|
1f6d15fa
tangwang
重构:SPU级别索引、统一索引架构...
|
1092
1093
1094
1095
|
facets=None,
query_info={'image_url': image_url, 'search_type': 'image_similarity'},
suggestions=[],
related_searches=[]
|
be52af70
tangwang
first commit
|
1096
1097
|
)
|
b926f678
tangwang
多语言查询
|
1098
1099
|
def get_domain_summary(self) -> Dict[str, Any]:
"""
|
bd96cead
tangwang
1. 动态多语言字段与统一策略配置
|
1100
|
Get summary of dynamic text retrieval configuration.
|
b926f678
tangwang
多语言查询
|
1101
1102
|
Returns:
|
bd96cead
tangwang
1. 动态多语言字段与统一策略配置
|
1103
|
Dictionary with language-aware field information
|
b926f678
tangwang
多语言查询
|
1104
|
"""
|
bd96cead
tangwang
1. 动态多语言字段与统一策略配置
|
1105
1106
1107
1108
1109
1110
1111
|
return {
"mode": "dynamic_language_fields",
"multilingual_fields": self.config.query_config.multilingual_fields,
"shared_fields": self.config.query_config.shared_fields,
"core_multilingual_fields": self.config.query_config.core_multilingual_fields,
"field_boosts": self.config.field_boosts,
}
|
b926f678
tangwang
多语言查询
|
1112
|
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
1113
|
def get_document(self, tenant_id: str, doc_id: str) -> Optional[Dict[str, Any]]:
|
be52af70
tangwang
first commit
|
1114
1115
1116
1117
|
"""
Get single document by ID.
Args:
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
1118
|
tenant_id: Tenant ID (required to determine which index to query)
|
be52af70
tangwang
first commit
|
1119
1120
1121
1122
1123
1124
|
doc_id: Document ID
Returns:
Document or None if not found
"""
try:
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
1125
|
index_name = get_tenant_index_name(tenant_id)
|
be52af70
tangwang
first commit
|
1126
|
response = self.es_client.client.get(
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
1127
|
index=index_name,
|
be52af70
tangwang
first commit
|
1128
1129
1130
1131
|
id=doc_id
)
return response.get('_source')
except Exception as e:
|
e4a39cc8
tangwang
索引隔离。 不同的tenant_i...
|
1132
|
logger.error(f"Failed to get document {doc_id} from tenant {tenant_id}: {e}", exc_info=True)
|
be52af70
tangwang
first commit
|
1133
|
return None
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
1134
1135
1136
1137
|
def _standardize_facets(
self,
es_aggregations: Dict[str, Any],
|
ff5325fa
tangwang
修复:直接在 Searcher 层...
|
1138
|
facet_configs: Optional[List[Union[str, Any]]],
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
1139
1140
1141
1142
1143
1144
1145
|
current_filters: Optional[Dict[str, Any]]
) -> Optional[List[FacetResult]]:
"""
将 ES 聚合结果转换为标准化的分面格式(返回 Pydantic 模型)。
Args:
es_aggregations: ES 原始聚合结果
|
ff5325fa
tangwang
修复:直接在 Searcher 层...
|
1146
|
facet_configs: 分面配置列表(str 或 FacetConfig)
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
|
current_filters: 当前应用的过滤器
Returns:
标准化的分面结果列表(FacetResult 对象)
"""
if not es_aggregations or not facet_configs:
return None
standardized_facets: List[FacetResult] = []
for config in facet_configs:
# 解析配置
if isinstance(config, str):
field = config
facet_type = "terms"
else:
|
ff5325fa
tangwang
修复:直接在 Searcher 层...
|
1163
1164
1165
|
# FacetConfig 对象
field = config.field
facet_type = config.type
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
|
agg_name = f"{field}_facet"
if agg_name not in es_aggregations:
continue
agg_result = es_aggregations[agg_name]
# 获取当前字段的选中值
selected_values = set()
if current_filters and field in current_filters:
filter_value = current_filters[field]
if isinstance(filter_value, list):
selected_values = set(filter_value)
else:
selected_values = {filter_value}
# 转换 buckets 为 FacetValue 对象
facet_values: List[FacetValue] = []
if 'buckets' in agg_result:
for bucket in agg_result['buckets']:
value = bucket.get('key')
count = bucket.get('doc_count', 0)
facet_values.append(FacetValue(
value=value,
label=str(value),
count=count,
selected=value in selected_values
))
# 构建 FacetResult 对象
facet_result = FacetResult(
field=field,
|
e7ad2b4a
tangwang
测试页面分页配置
|
1200
|
label=field,
|
6aa246be
tangwang
问题:Pydantic 应该能自动...
|
1201
1202
1203
1204
1205
1206
1207
|
type=facet_type,
values=facet_values
)
standardized_facets.append(facet_result)
return standardized_facets if standardized_facets else None
|