Blame view

query/query_parser.py 12.2 KB
be52af70   tangwang   first commit
1
2
3
4
5
6
7
8
  """
  Query parser - main module for query processing.
  
  Handles query rewriting, translation, and embedding generation.
  """
  
  from typing import Dict, List, Optional, Any
  import numpy as np
325eec03   tangwang   1. 日志、配置基础设施,使用优化
9
  import logging
be52af70   tangwang   first commit
10
  
be52af70   tangwang   first commit
11
  from embeddings import BgeEncoder
9f96d6f3   tangwang   短query不用语义搜索
12
  from config import SearchConfig
be52af70   tangwang   first commit
13
14
15
16
  from .language_detector import LanguageDetector
  from .translator import Translator
  from .query_rewriter import QueryRewriter, QueryNormalizer
  
325eec03   tangwang   1. 日志、配置基础设施,使用优化
17
18
  logger = logging.getLogger(__name__)
  
be52af70   tangwang   first commit
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
  
  class ParsedQuery:
      """Container for parsed query results."""
  
      def __init__(
          self,
          original_query: str,
          normalized_query: str,
          rewritten_query: Optional[str] = None,
          detected_language: str = "unknown",
          translations: Dict[str, str] = None,
          query_vector: Optional[np.ndarray] = None,
          domain: str = "default"
      ):
          self.original_query = original_query
          self.normalized_query = normalized_query
          self.rewritten_query = rewritten_query or normalized_query
          self.detected_language = detected_language
          self.translations = translations or {}
          self.query_vector = query_vector
          self.domain = domain
  
      def to_dict(self) -> Dict[str, Any]:
          """Convert to dictionary representation."""
          result = {
              "original_query": self.original_query,
              "normalized_query": self.normalized_query,
              "rewritten_query": self.rewritten_query,
              "detected_language": self.detected_language,
              "translations": self.translations,
              "domain": self.domain,
              "has_vector": self.query_vector is not None
          }
          return result
  
  
  class QueryParser:
      """
      Main query parser that processes queries through multiple stages:
      1. Normalization
      2. Query rewriting (brand/category mappings, synonyms)
      3. Language detection
      4. Translation to target languages
      5. Text embedding generation (for semantic search)
      """
  
      def __init__(
          self,
9f96d6f3   tangwang   短query不用语义搜索
67
          config: SearchConfig,
be52af70   tangwang   first commit
68
69
70
71
72
73
74
          text_encoder: Optional[BgeEncoder] = None,
          translator: Optional[Translator] = None
      ):
          """
          Initialize query parser.
  
          Args:
9f96d6f3   tangwang   短query不用语义搜索
75
              config: SearchConfig instance
be52af70   tangwang   first commit
76
77
78
              text_encoder: Text embedding encoder (lazy loaded if not provided)
              translator: Translator instance (lazy loaded if not provided)
          """
9f96d6f3   tangwang   短query不用语义搜索
79
          self.config = config
be52af70   tangwang   first commit
80
81
82
83
84
85
          self._text_encoder = text_encoder
          self._translator = translator
  
          # Initialize components
          self.normalizer = QueryNormalizer()
          self.language_detector = LanguageDetector()
9f96d6f3   tangwang   短query不用语义搜索
86
          self.rewriter = QueryRewriter(config.query_config.rewrite_dictionary)
be52af70   tangwang   first commit
87
88
89
90
  
      @property
      def text_encoder(self) -> BgeEncoder:
          """Lazy load text encoder."""
9f96d6f3   tangwang   短query不用语义搜索
91
          if self._text_encoder is None and self.config.query_config.enable_text_embedding:
325eec03   tangwang   1. 日志、配置基础设施,使用优化
92
              logger.info("Initializing text encoder (lazy load)...")
be52af70   tangwang   first commit
93
94
95
96
97
98
              self._text_encoder = BgeEncoder()
          return self._text_encoder
  
      @property
      def translator(self) -> Translator:
          """Lazy load translator."""
9f96d6f3   tangwang   短query不用语义搜索
99
          if self._translator is None and self.config.query_config.enable_translation:
325eec03   tangwang   1. 日志、配置基础设施,使用优化
100
              logger.info("Initializing translator (lazy load)...")
be52af70   tangwang   first commit
101
              self._translator = Translator(
9f96d6f3   tangwang   短query不用语义搜索
102
                  api_key=self.config.query_config.translation_api_key,
522a3964   tangwang   多语言搜索翻译的优化(deepL添...
103
                  use_cache=True,
9f96d6f3   tangwang   短query不用语义搜索
104
105
                  glossary_id=self.config.query_config.translation_glossary_id,
                  translation_context=self.config.query_config.translation_context
be52af70   tangwang   first commit
106
107
108
              )
          return self._translator
  
16c42787   tangwang   feat: implement r...
109
      def parse(self, query: str, generate_vector: bool = True, context: Optional[Any] = None) -> ParsedQuery:
be52af70   tangwang   first commit
110
111
112
113
114
115
          """
          Parse query through all processing stages.
  
          Args:
              query: Raw query string
              generate_vector: Whether to generate query embedding
16c42787   tangwang   feat: implement r...
116
              context: Optional request context for tracking and logging
be52af70   tangwang   first commit
117
118
119
120
  
          Returns:
              ParsedQuery object with all processing results
          """
16c42787   tangwang   feat: implement r...
121
122
123
124
125
126
127
128
          # Initialize logger if context provided
          logger = context.logger if context else None
          if logger:
              logger.info(
                  f"开始查询解析 | 原查询: '{query}' | 生成向量: {generate_vector}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
  
16c42787   tangwang   feat: implement r...
129
          def log_info(msg):
325eec03   tangwang   1. 日志、配置基础设施,使用优化
130
131
              if context and hasattr(context, 'logger'):
                  context.logger.info(msg, extra={'reqid': context.reqid, 'uid': context.uid})
16c42787   tangwang   feat: implement r...
132
              else:
325eec03   tangwang   1. 日志、配置基础设施,使用优化
133
                  logger.info(msg)
16c42787   tangwang   feat: implement r...
134
135
  
          def log_debug(msg):
325eec03   tangwang   1. 日志、配置基础设施,使用优化
136
137
              if context and hasattr(context, 'logger'):
                  context.logger.debug(msg, extra={'reqid': context.reqid, 'uid': context.uid})
16c42787   tangwang   feat: implement r...
138
              else:
325eec03   tangwang   1. 日志、配置基础设施,使用优化
139
                  logger.debug(msg)
be52af70   tangwang   first commit
140
141
142
  
          # Stage 1: Normalize
          normalized = self.normalizer.normalize(query)
16c42787   tangwang   feat: implement r...
143
144
145
          log_debug(f"标准化完成 | '{query}' -> '{normalized}'")
          if context:
              context.store_intermediate_result('normalized_query', normalized)
be52af70   tangwang   first commit
146
147
148
  
          # Extract domain if present (e.g., "brand:Nike" -> domain="brand", query="Nike")
          domain, query_text = self.normalizer.extract_domain_query(normalized)
16c42787   tangwang   feat: implement r...
149
150
151
152
          log_debug(f"域提取 | 域: '{domain}', 查询: '{query_text}'")
          if context:
              context.store_intermediate_result('extracted_domain', domain)
              context.store_intermediate_result('domain_query', query_text)
be52af70   tangwang   first commit
153
154
155
  
          # Stage 2: Query rewriting
          rewritten = None
9f96d6f3   tangwang   短query不用语义搜索
156
          if self.config.query_config.rewrite_dictionary:  # Enable rewrite if dictionary exists
be52af70   tangwang   first commit
157
158
              rewritten = self.rewriter.rewrite(query_text)
              if rewritten != query_text:
16c42787   tangwang   feat: implement r...
159
                  log_info(f"查询重写 | '{query_text}' -> '{rewritten}'")
be52af70   tangwang   first commit
160
                  query_text = rewritten
16c42787   tangwang   feat: implement r...
161
162
163
                  if context:
                      context.store_intermediate_result('rewritten_query', rewritten)
                      context.add_warning(f"查询被重写: {query_text}")
be52af70   tangwang   first commit
164
165
166
  
          # Stage 3: Language detection
          detected_lang = self.language_detector.detect(query_text)
16c42787   tangwang   feat: implement r...
167
168
169
          log_info(f"语言检测 | 检测到语言: {detected_lang}")
          if context:
              context.store_intermediate_result('detected_language', detected_lang)
be52af70   tangwang   first commit
170
171
172
  
          # Stage 4: Translation
          translations = {}
9f96d6f3   tangwang   短query不用语义搜索
173
          if self.config.query_config.enable_translation:
16c42787   tangwang   feat: implement r...
174
175
              try:
                  # Determine target languages for translation
bf89b597   tangwang   feat(search): ada...
176
177
                  # Simplified: always translate to Chinese and English
                  target_langs_for_translation = ['zh', 'en']
16c42787   tangwang   feat: implement r...
178
179
180
181
182
183
184
185
  
                  target_langs = self.translator.get_translation_needs(
                      detected_lang,
                      target_langs_for_translation
                  )
  
                  if target_langs:
                      log_info(f"开始翻译 | 源语言: {detected_lang} | 目标语言: {target_langs}")
522a3964   tangwang   多语言搜索翻译的优化(deepL添...
186
                      # Use e-commerce context for better disambiguation
bf89b597   tangwang   feat(search): ada...
187
                      translation_context = 'e-commerce product search'
16c42787   tangwang   feat: implement r...
188
189
190
                      translations = self.translator.translate_multi(
                          query_text,
                          target_langs,
522a3964   tangwang   多语言搜索翻译的优化(deepL添...
191
192
                          source_lang=detected_lang,
                          context=translation_context
16c42787   tangwang   feat: implement r...
193
194
195
196
197
198
199
200
201
202
203
204
205
                      )
                      log_info(f"翻译完成 | 结果: {translations}")
                      if context:
                          context.store_intermediate_result('translations', translations)
                          for lang, translation in translations.items():
                              if translation:
                                  context.store_intermediate_result(f'translation_{lang}', translation)
  
              except Exception as e:
                  error_msg = f"翻译失败 | 错误: {str(e)}"
                  log_info(error_msg)
                  if context:
                      context.add_warning(error_msg)
be52af70   tangwang   first commit
206
207
208
209
  
          # Stage 5: Text embedding
          query_vector = None
          if (generate_vector and
9f96d6f3   tangwang   短query不用语义搜索
210
              self.config.query_config.enable_text_embedding and
be52af70   tangwang   first commit
211
              domain == "default"):  # Only generate vector for default domain
9f96d6f3   tangwang   短query不用语义搜索
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
              # Get thresholds from config
              chinese_limit = self.config.query_config.embedding_disable_chinese_char_limit
              english_limit = self.config.query_config.embedding_disable_english_word_limit
              
              # Check if embedding should be disabled for short queries
              should_disable_embedding = False
              disable_reason = None
              
              if detected_lang == 'zh':
                  # For Chinese: disable embedding if character count <= threshold
                  char_count = len(query_text.strip())
                  if char_count <= chinese_limit:
                      should_disable_embedding = True
                      disable_reason = f"中文查询字数({char_count}) <= {chinese_limit},禁用向量搜索"
                      log_info(disable_reason)
                      if context:
                          context.store_intermediate_result('embedding_disabled_reason', disable_reason)
              else:
                  # For English: disable embedding if word count <= threshold
                  word_count = len(query_text.strip().split())
                  if word_count <= english_limit:
                      should_disable_embedding = True
                      disable_reason = f"英文查询单词数({word_count}) <= {english_limit},禁用向量搜索"
                      log_info(disable_reason)
                      if context:
                          context.store_intermediate_result('embedding_disabled_reason', disable_reason)
              
              if not should_disable_embedding:
                  try:
                      log_debug("开始生成查询向量")
                      query_vector = self.text_encoder.encode([query_text])[0]
                      log_debug(f"查询向量生成完成 | 形状: {query_vector.shape}")
                      if context:
                          context.store_intermediate_result('query_vector_shape', query_vector.shape)
                  except Exception as e:
                      error_msg = f"查询向量生成失败 | 错误: {str(e)}"
                      log_info(error_msg)
                      if context:
                          context.add_warning(error_msg)
be52af70   tangwang   first commit
251
252
253
254
255
256
257
258
259
260
261
262
  
          # Build result
          result = ParsedQuery(
              original_query=query,
              normalized_query=normalized,
              rewritten_query=rewritten,
              detected_language=detected_lang,
              translations=translations,
              query_vector=query_vector,
              domain=domain
          )
  
325eec03   tangwang   1. 日志、配置基础设施,使用优化
263
264
          if context and hasattr(context, 'logger'):
              context.logger.info(
16c42787   tangwang   feat: implement r...
265
266
267
268
269
270
                  f"查询解析完成 | 原查询: '{query}' | 最终查询: '{rewritten or query_text}' | "
                  f"语言: {detected_lang} | 域: {domain} | "
                  f"翻译数量: {len(translations)} | 向量: {'是' if query_vector is not None else '否'}",
                  extra={'reqid': context.reqid, 'uid': context.uid}
              )
          else:
325eec03   tangwang   1. 日志、配置基础设施,使用优化
271
272
273
274
              logger.info(
                  f"查询解析完成 | 原查询: '{query}' | 最终查询: '{rewritten or query_text}' | "
                  f"语言: {detected_lang} | 域: {domain}"
              )
16c42787   tangwang   feat: implement r...
275
  
be52af70   tangwang   first commit
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
          return result
  
      def get_search_queries(self, parsed_query: ParsedQuery) -> List[str]:
          """
          Get list of queries to search (original + translations).
  
          Args:
              parsed_query: Parsed query object
  
          Returns:
              List of query strings to search
          """
          queries = [parsed_query.rewritten_query]
  
          # Add translations
          for lang, translation in parsed_query.translations.items():
              if translation and translation != parsed_query.rewritten_query:
                  queries.append(translation)
  
          return queries
  
      def update_rewrite_rules(self, rules: Dict[str, str]) -> None:
          """
          Update query rewrite rules.
  
          Args:
              rules: Dictionary of pattern -> replacement mappings
          """
          for pattern, replacement in rules.items():
              self.rewriter.add_rule(pattern, replacement)
  
      def get_rewrite_rules(self) -> Dict[str, str]:
          """Get current rewrite rules."""
          return self.rewriter.get_rules()