Blame view

search/multilang_query_builder.py 18.7 KB
b926f678   tangwang   多语言查询
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
  """
  Multi-language query builder for handling domain-specific searches.
  
  This module extends the ESQueryBuilder to support multi-language field mappings,
  allowing queries to be routed to appropriate language-specific fields while
  maintaining a unified external interface.
  """
  
  from typing import Dict, Any, List, Optional
  import numpy as np
  
  from config import CustomerConfig, IndexConfig
  from query import ParsedQuery
  from .es_query_builder import ESQueryBuilder
  
  
  class MultiLanguageQueryBuilder(ESQueryBuilder):
      """
      Enhanced query builder with multi-language support.
  
      Handles routing queries to appropriate language-specific fields based on:
      1. Detected query language
      2. Available translations
      3. Domain configuration (language_field_mapping)
      """
  
      def __init__(
          self,
          config: CustomerConfig,
          index_name: str,
          text_embedding_field: Optional[str] = None,
          image_embedding_field: Optional[str] = None
      ):
          """
          Initialize multi-language query builder.
  
          Args:
              config: Customer configuration
              index_name: ES index name
              text_embedding_field: Field name for text embeddings
              image_embedding_field: Field name for image embeddings
          """
          self.config = config
  
          # For default domain, use all fields as fallback
          default_fields = self._get_domain_fields("default")
  
          super().__init__(
              index_name=index_name,
              match_fields=default_fields,
              text_embedding_field=text_embedding_field,
              image_embedding_field=image_embedding_field
          )
  
          # Build domain configurations
          self.domain_configs = self._build_domain_configs()
  
      def _build_domain_configs(self) -> Dict[str, IndexConfig]:
          """Build mapping of domain name to IndexConfig."""
          return {index.name: index for index in self.config.indexes}
  
      def _get_domain_fields(self, domain_name: str) -> List[str]:
          """Get fields for a specific domain with boost notation."""
          for index in self.config.indexes:
              if index.name == domain_name:
                  result = []
                  for field_name in index.fields:
                      field = self._get_field_by_name(field_name)
                      if field and field.boost != 1.0:
                          result.append(f"{field_name}^{field.boost}")
                      else:
                          result.append(field_name)
                  return result
          return []
  
      def _get_field_by_name(self, field_name: str):
          """Get field configuration by name."""
          for field in self.config.fields:
              if field.name == field_name:
                  return field
          return None
  
      def build_multilang_query(
          self,
          parsed_query: ParsedQuery,
          query_vector: Optional[np.ndarray] = None,
f739c5e3   tangwang   fix sch
87
          query_node: Optional[Any] = None,
b926f678   tangwang   多语言查询
88
          filters: Optional[Dict[str, Any]] = None,
6aa246be   tangwang   问题:Pydantic 应该能自动...
89
          range_filters: Optional[Dict[str, Any]] = None,
b926f678   tangwang   多语言查询
90
91
92
93
94
95
96
97
          size: int = 10,
          from_: int = 0,
          enable_knn: bool = True,
          knn_k: int = 50,
          knn_num_candidates: int = 200,
          min_score: Optional[float] = None
      ) -> Dict[str, Any]:
          """
6aa246be   tangwang   问题:Pydantic 应该能自动...
98
          Build ES query with multi-language support (重构版).
b926f678   tangwang   多语言查询
99
100
101
102
  
          Args:
              parsed_query: Parsed query with language info and translations
              query_vector: Query embedding for KNN search
6aa246be   tangwang   问题:Pydantic 应该能自动...
103
104
              filters: Exact match filters
              range_filters: Range filters for numeric fields
b926f678   tangwang   多语言查询
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
              size: Number of results
              from_: Offset for pagination
              enable_knn: Whether to use KNN search
              knn_k: K value for KNN
              knn_num_candidates: Number of candidates for KNN
              min_score: Minimum score threshold
  
          Returns:
              ES query DSL dictionary
          """
          domain = parsed_query.domain
          domain_config = self.domain_configs.get(domain)
  
          if not domain_config:
              # Fallback to default domain
              domain = "default"
              domain_config = self.domain_configs.get("default")
  
          if not domain_config:
              # Use original behavior
              return super().build_query(
                  query_text=parsed_query.rewritten_query,
                  query_vector=query_vector,
                  filters=filters,
6aa246be   tangwang   问题:Pydantic 应该能自动...
129
                  range_filters=range_filters,
b926f678   tangwang   多语言查询
130
131
132
133
134
135
136
137
138
139
140
141
142
                  size=size,
                  from_=from_,
                  enable_knn=enable_knn,
                  knn_k=knn_k,
                  knn_num_candidates=knn_num_candidates,
                  min_score=min_score
              )
  
          print(f"[MultiLangQueryBuilder] Building query for domain: {domain}")
          print(f"[MultiLangQueryBuilder] Detected language: {parsed_query.detected_language}")
          print(f"[MultiLangQueryBuilder] Available translations: {list(parsed_query.translations.keys())}")
  
          # Build query clause with multi-language support
f739c5e3   tangwang   fix sch
143
144
145
146
147
148
149
150
151
152
153
154
          if query_node and isinstance(query_node, tuple) and len(query_node) > 0:
              # Handle boolean query from tuple (AST, score)
              ast_node = query_node[0]
              query_clause = self._build_boolean_query_from_tuple(ast_node)
              print(f"[MultiLangQueryBuilder] Using boolean query: {query_clause}")
          elif query_node and hasattr(query_node, 'operator') and query_node.operator != 'TERM':
              # Handle boolean query using base class method
              query_clause = self._build_boolean_query(query_node)
              print(f"[MultiLangQueryBuilder] Using boolean query: {query_clause}")
          else:
              # Handle text query with multi-language support
              query_clause = self._build_multilang_text_query(parsed_query, domain_config)
b926f678   tangwang   多语言查询
155
  
43f1139f   tangwang   refactor: ES查询结构重...
156
157
          # 构建内层bool: 文本和KNN二选一
          inner_bool_should = [query_clause]
b926f678   tangwang   多语言查询
158
  
43f1139f   tangwang   refactor: ES查询结构重...
159
160
161
162
163
164
165
166
          # 如果启用KNN,添加到should
          if enable_knn and query_vector is not None and self.text_embedding_field:
              knn_query = {
                  "knn": {
                      "field": self.text_embedding_field,
                      "query_vector": query_vector.tolist(),
                      "k": knn_k,
                      "num_candidates": knn_num_candidates
b926f678   tangwang   多语言查询
167
                  }
43f1139f   tangwang   refactor: ES查询结构重...
168
169
              }
              inner_bool_should.append(knn_query)
b926f678   tangwang   多语言查询
170
  
43f1139f   tangwang   refactor: ES查询结构重...
171
172
173
174
175
          # 构建内层bool结构
          inner_bool = {
              "bool": {
                  "should": inner_bool_should,
                  "minimum_should_match": 1
b926f678   tangwang   多语言查询
176
              }
43f1139f   tangwang   refactor: ES查询结构重...
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
          }
  
          # 构建外层bool: 包含filter
          filter_clauses = self._build_filters(filters, range_filters) if (filters or range_filters) else []
  
          outer_bool = {
              "bool": {
                  "must": [inner_bool]
              }
          }
  
          if filter_clauses:
              outer_bool["bool"]["filter"] = filter_clauses
  
          # 包裹function_score
          function_score_query = {
              "function_score": {
                  "query": outer_bool,
                  "functions": self._build_score_functions(),
                  "score_mode": "sum",
                  "boost_mode": "multiply"
              }
          }
  
          es_query = {
              "size": size,
              "from": from_,
              "query": function_score_query
          }
b926f678   tangwang   多语言查询
206
  
b926f678   tangwang   多语言查询
207
208
209
210
211
          if min_score is not None:
              es_query["min_score"] = min_score
  
          return es_query
  
43f1139f   tangwang   refactor: ES查询结构重...
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
      def _build_score_functions(self) -> List[Dict[str, Any]]:
          """
          构建 function_score 的打分函数列表
          
          Returns:
              打分函数列表
          """
          functions = []
          
          # 时效性加权:最近更新的商品得分更高
          functions.append({
              "filter": {
                  "range": {
                      "days_since_last_update": {"lte": 30}
                  }
              },
              "weight": 1.1
          })
          
          # 可以添加更多打分因子
          # functions.append({
          #     "filter": {"term": {"is_video": True}},
          #     "weight": 1.05
          # })
          
          return functions
  
b926f678   tangwang   多语言查询
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
      def _build_multilang_text_query(
          self,
          parsed_query: ParsedQuery,
          domain_config: IndexConfig
      ) -> Dict[str, Any]:
          """
          Build text query with multi-language field routing.
  
          Args:
              parsed_query: Parsed query with language info
              domain_config: Domain configuration
  
          Returns:
              ES query clause
          """
          if not domain_config.language_field_mapping:
              # No multi-language mapping, use all fields with default analyzer
              fields_with_boost = []
              for field_name in domain_config.fields:
                  field = self._get_field_by_name(field_name)
                  if field and field.boost != 1.0:
                      fields_with_boost.append(f"{field_name}^{field.boost}")
                  else:
                      fields_with_boost.append(field_name)
  
              return {
                  "multi_match": {
                      "query": parsed_query.rewritten_query,
                      "fields": fields_with_boost,
                      "minimum_should_match": "67%",
                      "tie_breaker": 0.9,
                      "boost": domain_config.boost,
                      "_name": f"{domain_config.name}_query"
                  }
              }
  
          # Multi-language mapping exists - build targeted queries
          should_clauses = []
          available_languages = set(domain_config.language_field_mapping.keys())
  
          # 1. Query in detected language (if it exists in mapping)
          detected_lang = parsed_query.detected_language
          if detected_lang in available_languages:
              target_fields = domain_config.language_field_mapping[detected_lang]
              fields_with_boost = self._apply_field_boosts(target_fields)
  
              should_clauses.append({
                  "multi_match": {
                      "query": parsed_query.rewritten_query,
                      "fields": fields_with_boost,
                      "minimum_should_match": "67%",
                      "tie_breaker": 0.9,
                      "boost": domain_config.boost * 1.5,  # Higher boost for detected language
                      "_name": f"{domain_config.name}_{detected_lang}_query"
                  }
              })
              print(f"[MultiLangQueryBuilder] Added query for detected language '{detected_lang}' on fields: {target_fields}")
  
          # 2. Query in translated languages (only for languages in mapping)
          for lang, translation in parsed_query.translations.items():
              # Only use translations for languages that exist in the mapping
              if lang in available_languages and translation and translation.strip():
                  target_fields = domain_config.language_field_mapping[lang]
                  fields_with_boost = self._apply_field_boosts(target_fields)
  
                  should_clauses.append({
                      "multi_match": {
                          "query": translation,
                          "fields": fields_with_boost,
                          "minimum_should_match": "67%",
                          "tie_breaker": 0.9,
                          "boost": domain_config.boost,
                          "_name": f"{domain_config.name}_{lang}_translated_query"
                      }
                  })
                  print(f"[MultiLangQueryBuilder] Added translated query for language '{lang}' on fields: {target_fields}")
  
          # 3. Fallback: query all fields in mapping if no language-specific query was built
          if not should_clauses:
              print(f"[MultiLangQueryBuilder] No language mapping matched, using all fields from mapping")
              # Use all fields from all languages in the mapping
              all_mapped_fields = []
              for lang_fields in domain_config.language_field_mapping.values():
                  all_mapped_fields.extend(lang_fields)
              # Remove duplicates while preserving order
              unique_fields = list(dict.fromkeys(all_mapped_fields))
              fields_with_boost = self._apply_field_boosts(unique_fields)
  
              should_clauses.append({
                  "multi_match": {
                      "query": parsed_query.rewritten_query,
                      "fields": fields_with_boost,
                      "minimum_should_match": "67%",
                      "tie_breaker": 0.9,
                      "boost": domain_config.boost * 0.8,  # Lower boost for fallback
                      "_name": f"{domain_config.name}_fallback_query"
                  }
              })
  
          if len(should_clauses) == 1:
              return should_clauses[0]
          else:
              return {
                  "bool": {
                      "should": should_clauses,
                      "minimum_should_match": 1
                  }
              }
  
      def _apply_field_boosts(self, field_names: List[str]) -> List[str]:
          """Apply boost values to field names."""
          result = []
          for field_name in field_names:
              field = self._get_field_by_name(field_name)
              if field and field.boost != 1.0:
                  result.append(f"{field_name}^{field.boost}")
              else:
                  result.append(field_name)
          return result
  
f739c5e3   tangwang   fix sch
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
      def _build_boolean_query_from_tuple(self, node) -> Dict[str, Any]:
          """
          Build query from boolean expression tuple.
  
          Args:
              node: Boolean expression tuple (operator, terms...)
  
          Returns:
              ES query clause
          """
          if not node:
              return {"match_all": {}}
  
          # Handle different node types from boolean parser
          if hasattr(node, 'operator'):
              # QueryNode object
              operator = node.operator
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
376
377
378
379
380
              terms = node.terms if hasattr(node, 'terms') else None
  
              # For TERM nodes, check if there's a value
              if operator == 'TERM' and hasattr(node, 'value') and node.value:
                  terms = node.value
f739c5e3   tangwang   fix sch
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
          elif isinstance(node, tuple) and len(node) > 0:
              # Tuple format from boolean parser
              if hasattr(node[0], 'operator'):
                  # Nested tuple with QueryNode
                  operator = node[0].operator
                  terms = node[0].terms
              elif isinstance(node[0], str):
                  # Simple tuple like ('TERM', 'field:value')
                  operator = node[0]
                  terms = node[1] if len(node) > 1 else ''
              else:
                  # Complex tuple like (OR( TERM(...), TERM(...) ), score)
                  if hasattr(node[0], '__class__') and hasattr(node[0], '__name__'):
                      # Constructor call like OR(...)
                      operator = node[0].__name__
                  elif str(node[0]).startswith('('):
                      # String representation of constructor call
                      import re
                      match = re.match(r'(\w+)\(', str(node[0]))
                      if match:
                          operator = match.group(1)
                      else:
                          return {"match_all": {}}
                  else:
                      operator = str(node[0])
  
                  # Extract terms from nested structure
                  terms = []
                  if len(node) > 1 and isinstance(node[1], tuple):
                      terms = node[1]
          else:
              return {"match_all": {}}
  
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
414
          
f739c5e3   tangwang   fix sch
415
416
417
418
419
420
421
422
423
          if operator == 'TERM':
              # Leaf node - handle field:query format
              if isinstance(terms, str) and ':' in terms:
                  field, value = terms.split(':', 1)
                  return {
                      "term": {
                          field: value
                      }
                  }
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
424
425
426
427
428
429
430
431
432
433
              elif isinstance(terms, str):
                  # Simple text term - create match query
                  return {
                      "multi_match": {
                          "query": terms,
                          "fields": self.match_fields,
                          "type": "best_fields",
                          "operator": "AND"
                      }
                  }
f739c5e3   tangwang   fix sch
434
              else:
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
435
436
437
438
                  # Invalid TERM node - return empty match
                  return {
                      "match_none": {}
                  }
f739c5e3   tangwang   fix sch
439
440
441
442
  
          elif operator == 'OR':
              # Any term must match
              should_clauses = []
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
443
444
445
446
447
448
449
450
451
452
453
454
              if terms:
                  for term in terms:
                      clause = self._build_boolean_query_from_tuple(term)
                      if clause and clause.get("match_none") is None:
                          should_clauses.append(clause)
  
              if should_clauses:
                  return {
                      "bool": {
                          "should": should_clauses,
                          "minimum_should_match": 1
                      }
f739c5e3   tangwang   fix sch
455
                  }
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
456
457
              else:
                  return {"match_none": {}}
f739c5e3   tangwang   fix sch
458
459
460
461
  
          elif operator == 'AND':
              # All terms must match
              must_clauses = []
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
462
463
464
465
466
467
468
469
470
471
472
              if terms:
                  for term in terms:
                      clause = self._build_boolean_query_from_tuple(term)
                      if clause and clause.get("match_none") is None:
                          must_clauses.append(clause)
  
              if must_clauses:
                  return {
                      "bool": {
                          "must": must_clauses
                      }
f739c5e3   tangwang   fix sch
473
                  }
c86c8237   tangwang   支持聚合。过滤项补充了逻辑,但是有问题
474
475
              else:
                  return {"match_none": {}}
f739c5e3   tangwang   fix sch
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
  
          elif operator == 'ANDNOT':
              # First term must match, second must not
              if len(terms) >= 2:
                  return {
                      "bool": {
                          "must": [self._build_boolean_query_from_tuple(terms[0])],
                          "must_not": [self._build_boolean_query_from_tuple(terms[1])]
                      }
                  }
              else:
                  return self._build_boolean_query_from_tuple(terms[0])
  
          elif operator == 'RANK':
              # Like OR but for ranking (all terms contribute to score)
              should_clauses = []
              for term in terms:
                  should_clauses.append(self._build_boolean_query_from_tuple(term))
              return {
                  "bool": {
                      "should": should_clauses
                  }
              }
  
          else:
              # Unknown operator
              return {"match_all": {}}
  
b926f678   tangwang   多语言查询
504
505
506
507
508
509
510
511
512
513
514
515
516
      def get_domain_summary(self) -> Dict[str, Any]:
          """Get summary of all configured domains."""
          summary = {}
          for domain_name, domain_config in self.domain_configs.items():
              summary[domain_name] = {
                  "label": domain_config.label,
                  "fields": domain_config.fields,
                  "analyzer": domain_config.analyzer.value,
                  "boost": domain_config.boost,
                  "has_multilang_mapping": domain_config.language_field_mapping is not None,
                  "supported_languages": list(domain_config.language_field_mapping.keys()) if domain_config.language_field_mapping else []
              }
          return summary