""" Query parser - main module for query processing. Handles query rewriting, translation, and embedding generation. """ from typing import Dict, List, Optional, Any, Union import numpy as np import logging import re from concurrent.futures import Future, ThreadPoolExecutor, as_completed from embeddings import TextEmbeddingEncoder from config import SearchConfig from .language_detector import LanguageDetector from providers import create_translation_provider from .query_rewriter import QueryRewriter, QueryNormalizer logger = logging.getLogger(__name__) try: import hanlp # type: ignore except Exception: # pragma: no cover hanlp = None class ParsedQuery: """Container for parsed query results.""" def __init__( self, original_query: str, query_normalized: str, rewritten_query: Optional[str] = None, detected_language: Optional[str] = None, translations: Dict[str, str] = None, query_vector: Optional[np.ndarray] = None, domain: str = "default", keywords: str = "", token_count: int = 0, query_tokens: Optional[List[str]] = None ): self.original_query = original_query self.query_normalized = query_normalized self.rewritten_query = rewritten_query or query_normalized self.detected_language = detected_language self.translations = translations or {} self.query_vector = query_vector self.domain = domain # Query analysis fields self.keywords = keywords self.token_count = token_count self.query_tokens = query_tokens or [] def to_dict(self) -> Dict[str, Any]: """Convert to dictionary representation.""" result = { "original_query": self.original_query, "query_normalized": self.query_normalized, "rewritten_query": self.rewritten_query, "detected_language": self.detected_language, "translations": self.translations, "domain": self.domain } return result class QueryParser: """ Main query parser that processes queries through multiple stages: 1. Normalization 2. Query rewriting (brand/category mappings, synonyms) 3. Language detection 4. Translation to target languages 5. Text embedding generation (for semantic search) """ def __init__( self, config: SearchConfig, text_encoder: Optional[TextEmbeddingEncoder] = None, translator: Optional[Any] = None ): """ Initialize query parser. Args: config: SearchConfig instance text_encoder: Text embedding encoder (lazy loaded if not provided) translator: Translator instance (lazy loaded if not provided) """ self.config = config self._text_encoder = text_encoder self._translator = translator # Initialize components self.normalizer = QueryNormalizer() self.language_detector = LanguageDetector() self.rewriter = QueryRewriter(config.query_config.rewrite_dictionary) # Optional HanLP components (heavy). If unavailable, fall back to a lightweight tokenizer. self._tok = None self._pos_tag = None if hanlp is not None: try: logger.info("Initializing HanLP components...") self._tok = hanlp.load(hanlp.pretrained.tok.CTB9_TOK_ELECTRA_BASE_CRF) self._tok.config.output_spans = True self._pos_tag = hanlp.load(hanlp.pretrained.pos.CTB9_POS_ELECTRA_SMALL) logger.info("HanLP components initialized") except Exception as e: logger.warning(f"HanLP init failed, falling back to simple tokenizer: {e}") self._tok = None self._pos_tag = None else: logger.info("HanLP not installed; using simple tokenizer") @property def text_encoder(self) -> TextEmbeddingEncoder: """Lazy load text encoder.""" if self._text_encoder is None and self.config.query_config.enable_text_embedding: logger.info("Initializing text encoder (lazy load)...") self._text_encoder = TextEmbeddingEncoder() return self._text_encoder @property def translator(self) -> Any: """Lazy load translator.""" if self._translator is None: from config.services_config import get_translation_config cfg = get_translation_config() logger.info("Initializing translator (provider=%s)...", cfg.provider) self._translator = create_translation_provider(self.config.query_config) return self._translator def _simple_tokenize(self, text: str) -> List[str]: """ Lightweight tokenizer fallback. - Groups consecutive CJK chars as a token - Groups consecutive latin/digits/underscore/dash as a token """ if not text: return [] pattern = re.compile(r"[\u4e00-\u9fff]+|[A-Za-z0-9_]+(?:-[A-Za-z0-9_]+)*") return pattern.findall(text) def _extract_keywords(self, query: str) -> str: """Extract keywords (nouns with length > 1) from query.""" if self._tok is not None and self._pos_tag is not None: tok_result = self._tok(query) if not tok_result: return "" words = [x[0] for x in tok_result] pos_tags = self._pos_tag(words) keywords = [] for word, pos in zip(words, pos_tags): if len(word) > 1 and isinstance(pos, str) and pos.startswith("N"): keywords.append(word) return " ".join(keywords) # Fallback: treat tokens with length > 1 as "keywords" tokens = self._simple_tokenize(query) keywords = [t for t in tokens if len(t) > 1] return " ".join(keywords) def _get_token_count(self, query: str) -> int: """Get token count (HanLP if available, otherwise simple).""" if self._tok is not None: tok_result = self._tok(query) return len(tok_result) if tok_result else 0 return len(self._simple_tokenize(query)) def _get_query_tokens(self, query: str) -> List[str]: """Get token list (HanLP if available, otherwise simple).""" if self._tok is not None: tok_result = self._tok(query) return [x[0] for x in tok_result] if tok_result else [] return self._simple_tokenize(query) def parse( self, query: str, tenant_id: Optional[str] = None, generate_vector: bool = True, context: Optional[Any] = None ) -> ParsedQuery: """ Parse query through all processing stages. Args: query: Raw query string generate_vector: Whether to generate query embedding context: Optional request context for tracking and logging Returns: ParsedQuery object with all processing results """ # Initialize logger if context provided active_logger = context.logger if context else logger if context and hasattr(context, "logger"): context.logger.info( f"Starting query parsing | Original query: '{query}' | Generate vector: {generate_vector}", extra={'reqid': context.reqid, 'uid': context.uid} ) def log_info(msg): if context and hasattr(context, 'logger'): context.logger.info(msg, extra={'reqid': context.reqid, 'uid': context.uid}) else: active_logger.info(msg) def log_debug(msg): if context and hasattr(context, 'logger'): context.logger.debug(msg, extra={'reqid': context.reqid, 'uid': context.uid}) else: active_logger.debug(msg) # Stage 1: Normalize normalized = self.normalizer.normalize(query) log_debug(f"Normalization completed | '{query}' -> '{normalized}'") if context: context.store_intermediate_result('query_normalized', normalized) # Extract domain if present (e.g., "brand:Nike" -> domain="brand", query="Nike") domain, query_text = self.normalizer.extract_domain_query(normalized) log_debug(f"Domain extraction | Domain: '{domain}', Query: '{query_text}'") if context: context.store_intermediate_result('extracted_domain', domain) context.store_intermediate_result('domain_query', query_text) # Stage 2: Query rewriting rewritten = None if self.config.query_config.rewrite_dictionary: # Enable rewrite if dictionary exists rewritten = self.rewriter.rewrite(query_text) if rewritten != query_text: log_info(f"Query rewritten | '{query_text}' -> '{rewritten}'") query_text = rewritten if context: context.store_intermediate_result('rewritten_query', rewritten) context.add_warning(f"Query was rewritten: {query_text}") # Stage 3: Language detection detected_lang = self.language_detector.detect(query_text) # Use default language if detection failed (None or "unknown") if not detected_lang or detected_lang == "unknown": detected_lang = self.config.query_config.default_language log_info(f"Language detection | Detected language: {detected_lang}") if context: context.store_intermediate_result('detected_language', detected_lang) # Stage 4: Translation (with async support and conditional waiting) translations = {} translation_futures = {} try: # 根据租户配置的 index_languages 决定翻译目标语言 from config.tenant_config_loader import get_tenant_config_loader tenant_loader = get_tenant_config_loader() tenant_cfg = tenant_loader.get_tenant_config(tenant_id or "default") index_langs = tenant_cfg.get("index_languages") or ["en", "zh"] target_langs_for_translation = [lang for lang in index_langs if lang != detected_lang] if target_langs_for_translation: target_langs = target_langs_for_translation if target_langs: # Use e-commerce context for better disambiguation translation_context = self.config.query_config.translation_context # For query translation, we use a general prompt (not language-specific) query_prompt = self.config.query_config.translation_prompts.get('query_zh') or \ self.config.query_config.translation_prompts.get('default_zh') # Determine if we need to wait for translation results # If detected_lang is not in index_languages, we must wait for translation need_wait_translation = detected_lang not in index_langs if need_wait_translation: # Use async method that returns Futures, so we can wait for results translation_results = self.translator.translate_multi_async( query_text, target_langs, source_lang=detected_lang, context=translation_context, prompt=query_prompt ) # Separate cached results and futures for lang, result in translation_results.items(): if isinstance(result, Future): translation_futures[lang] = result else: translations[lang] = result else: # Use async mode: returns cached translations immediately, missing ones translated in background translations = self.translator.translate_multi( query_text, target_langs, source_lang=detected_lang, context=translation_context, async_mode=True, prompt=query_prompt ) # Filter out None values (missing translations that are being processed async) translations = {k: v for k, v in translations.items() if v is not None} if translations: log_info(f"Translation completed (cache hit) | Query text: '{query_text}' | Results: {translations}") if translation_futures: log_debug(f"Translation in progress, waiting for results... | Query text: '{query_text}' | Languages: {list(translation_futures.keys())}") if context: context.store_intermediate_result('translations', translations) for lang, translation in translations.items(): if translation: context.store_intermediate_result(f'translation_{lang}', translation) except Exception as e: error_msg = f"Translation failed | Error: {str(e)}" log_info(error_msg) if context: context.add_warning(error_msg) # Stage 5: Query analysis (keywords, token count, query_tokens) keywords = self._extract_keywords(query_text) query_tokens = self._get_query_tokens(query_text) token_count = len(query_tokens) log_debug(f"Query analysis | Keywords: {keywords} | Token count: {token_count} | " f"Query tokens: {query_tokens}") if context: context.store_intermediate_result('keywords', keywords) context.store_intermediate_result('token_count', token_count) context.store_intermediate_result('query_tokens', query_tokens) # Stage 6: Text embedding (only for non-short queries) - async execution query_vector = None embedding_future = None should_generate_embedding = ( generate_vector and self.config.query_config.enable_text_embedding and domain == "default" ) encoding_executor = None if should_generate_embedding: try: log_debug("Starting query vector generation (async)") # Submit encoding task to thread pool for async execution encoding_executor = ThreadPoolExecutor(max_workers=1) def _encode_query_vector() -> Optional[np.ndarray]: arr = self.text_encoder.encode([query_text]) if arr is None or len(arr) == 0: return None vec = arr[0] return vec if isinstance(vec, np.ndarray) else None embedding_future = encoding_executor.submit( _encode_query_vector ) except Exception as e: error_msg = f"Query vector generation task submission failed | Error: {str(e)}" log_info(error_msg) if context: context.add_warning(error_msg) encoding_executor = None embedding_future = None # Wait for all async tasks to complete (translation and embedding) if translation_futures or embedding_future: log_debug("Waiting for async tasks to complete...") # Collect all futures with their identifiers all_futures = [] future_to_lang = {} for lang, future in translation_futures.items(): all_futures.append(future) future_to_lang[future] = ('translation', lang) if embedding_future: all_futures.append(embedding_future) future_to_lang[embedding_future] = ('embedding', None) # Wait for all futures to complete for future in as_completed(all_futures): task_type, lang = future_to_lang[future] try: result = future.result() if task_type == 'translation': if result: translations[lang] = result log_info(f"Translation completed | Query text: '{query_text}' | Target language: {lang} | Translation result: '{result}'") if context: context.store_intermediate_result(f'translation_{lang}', result) elif task_type == 'embedding': query_vector = result if query_vector is not None: log_debug(f"Query vector generation completed | Shape: {query_vector.shape}") if context: context.store_intermediate_result('query_vector_shape', query_vector.shape) else: log_info("Query vector generation completed but result is None, will process without vector") except Exception as e: if task_type == 'translation': error_msg = f"Translation failed | Language: {lang} | Error: {str(e)}" else: error_msg = f"Query vector generation failed | Error: {str(e)}" log_info(error_msg) if context: context.add_warning(error_msg) # Clean up encoding executor if encoding_executor: encoding_executor.shutdown(wait=False) # Update translations in context after all are complete if translations and context: context.store_intermediate_result('translations', translations) # Build result result = ParsedQuery( original_query=query, query_normalized=normalized, rewritten_query=rewritten, detected_language=detected_lang, translations=translations, query_vector=query_vector, domain=domain, keywords=keywords, token_count=token_count, query_tokens=query_tokens ) if context and hasattr(context, 'logger'): context.logger.info( f"Query parsing completed | Original query: '{query}' | Final query: '{rewritten or query_text}' | " f"Language: {detected_lang} | Domain: {domain} | " f"Translation count: {len(translations)} | Vector: {'yes' if query_vector is not None else 'no'}", extra={'reqid': context.reqid, 'uid': context.uid} ) else: logger.info( f"Query parsing completed | Original query: '{query}' | Final query: '{rewritten or query_text}' | " f"Language: {detected_lang} | Domain: {domain}" ) return result def get_search_queries(self, parsed_query: ParsedQuery) -> List[str]: """ Get list of queries to search (original + translations). Args: parsed_query: Parsed query object Returns: List of query strings to search """ queries = [parsed_query.rewritten_query] # Add translations for lang, translation in parsed_query.translations.items(): if translation and translation != parsed_query.rewritten_query: queries.append(translation) return queries def update_rewrite_rules(self, rules: Dict[str, str]) -> None: """ Update query rewrite rules. Args: rules: Dictionary of pattern -> replacement mappings """ for pattern, replacement in rules.items(): self.rewriter.add_rule(pattern, replacement) def get_rewrite_rules(self) -> Dict[str, str]: """Get current rewrite rules.""" return self.rewriter.get_rules()