店匠体系数据的搜索:mock data -> mysql, mysql->ES

tangwang
1 parent 41e1f8df
Showing 11 changed files with 41 additions and 52 deletions Show diff stats
api/app.py
config/__init__.py
config/config_loader.py
config/env_config.py
indexer/data_transformer.py
indexer/mapping_generator.py
query/query_parser.py
search/multilang_query_builder.py
search/searcher.py
test_source_fields.py
tests/conftest.py
@@ -39,13 +39,13 @@ limiter = Limiter(key_func=get_remote_address)
 # Add parent directory to path
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  
-from config import ConfigLoader, CustomerConfig
+from config import ConfigLoader, SearchConfig
 from utils import ESClient
 from search import Searcher
 from query import QueryParser
  
 # Global instances
-_config: Optional[CustomerConfig] = None
+_config: Optional[SearchConfig] = None
 _es_client: Optional[ESClient] = None
 _searcher: Optional[Searcher] = None
 _query_parser: Optional[QueryParser] = None
@@ -111,7 +111,7 @@ def init_service(es_host: str = &quot;http://localhost:9200&quot;):
     print("Search service ready!")
  
  
-def get_config() -> CustomerConfig:
+def get_config() -> SearchConfig:
     """Get search engine configuration."""
     if _config is None:
         raise RuntimeError("Service not initialized")
@@ -14,7 +14,7 @@ from .field_types import (
  
 from .config_loader import (
     ConfigLoader,
-    CustomerConfig,
+    SearchConfig,
     IndexConfig,
     RankingConfig,
     QueryConfig,
@@ -38,7 +38,7 @@ __all__ = [
  
     # Config loader
     'ConfigLoader',
-    'CustomerConfig',
+    'SearchConfig',
     'IndexConfig',
     'RankingConfig',
     'QueryConfig',
 """
-Configuration loader and validator for customer-specific search configurations.
+Configuration loader and validator for search engine configurations.
  
 This module handles loading, parsing, and validating YAML configuration files
-that define how each customer's data should be indexed and searched.
+that define how search engine data should be indexed and searched.
 """
  
 import yaml
@@ -85,7 +85,7 @@ class RerankConfig:
  
  
 @dataclass
-class CustomerConfig:
+class SearchConfig:
     """Complete configuration for search engine (multi-tenant)."""
     # Field definitions
     fields: List[FieldConfig]
@@ -161,12 +161,12 @@ class ConfigLoader:
  
         return rewrite_dict
  
-    def load_config(self) -> CustomerConfig:
+    def load_config(self) -> SearchConfig:
         """
         Load unified configuration from YAML file.
  
         Returns:
-            CustomerConfig object
+            SearchConfig object
  
         Raises:
             ConfigurationError: If config file not found or invalid
@@ -182,8 +182,8 @@ class ConfigLoader:
  
         return self._parse_config(config_data)
  
-    def _parse_config(self, config_data: Dict[str, Any]) -> CustomerConfig:
-        """Parse configuration dictionary into CustomerConfig object."""
+    def _parse_config(self, config_data: Dict[str, Any]) -> SearchConfig:
+        """Parse configuration dictionary into SearchConfig object."""
  
         # Parse fields
         fields = []
@@ -243,7 +243,7 @@ class ConfigLoader:
             inner_hits_size=spu_data.get("inner_hits_size", 3)
         )
  
-        return CustomerConfig(
+        return SearchConfig(
             fields=fields,
             indexes=indexes,
             query_config=query_config,
@@ -312,12 +312,12 @@ class ConfigLoader:
             language_field_mapping=language_field_mapping
         )
  
-    def validate_config(self, config: CustomerConfig) -> List[str]:
+    def validate_config(self, config: SearchConfig) -> List[str]:
         """
-        Validate customer configuration.
+        Validate search configuration.
  
         Args:
-            config: Customer configuration to validate
+            config: Search configuration to validate
  
         Returns:
             List of validation error messages (empty if valid)
@@ -407,7 +407,7 @@ class ConfigLoader:
  
         return errors
  
-    def save_config(self, config: CustomerConfig, output_path: Optional[str] = None) -> None:
+    def save_config(self, config: SearchConfig, output_path: Optional[str] = None) -> None:
         """
         Save configuration to YAML file.
  
@@ -31,9 +31,6 @@ REDIS_CONFIG = {
 # DeepL API Key
 DEEPL_AUTH_KEY = os.getenv('DEEPL_AUTH_KEY')
  
-# Customer Configuration
-CUSTOMER_ID = os.getenv('CUSTOMER_ID', 'customer1')
-
 # API Service Configuration
 API_HOST = os.getenv('API_HOST', '0.0.0.0')
 API_PORT = int(os.getenv('API_PORT', 6002))
@@ -74,11 +71,6 @@ def get_deepl_key() -&gt; str:
     return DEEPL_AUTH_KEY
  
  
-def get_customer_id() -> str:
-    """Get default customer ID."""
-    return CUSTOMER_ID
-
-
 def get_db_config() -> Dict[str, Any]:
     """Get MySQL database configuration."""
     return DB_CONFIG.copy()
@@ -103,9 +95,6 @@ def print_config():
     print("\nDeepL:")
     print(f"  API Key: {'*' * 10 if DEEPL_AUTH_KEY else 'None (translation disabled)'}")
  
-    print("\nCustomer:")
-    print(f"  Customer ID: {CUSTOMER_ID}")
-
     print("\nAPI Service:")
     print(f"  Host: {API_HOST}")
     print(f"  Port: {API_PORT}")
@@ -8,7 +8,7 @@ import pandas as pd
 import numpy as np
 import datetime
 from typing import Dict, Any, List, Optional
-from config import CustomerConfig, FieldConfig, FieldType
+from config import SearchConfig, FieldConfig, FieldType
 from embeddings import BgeEncoder, CLIPImageEncoder
 from utils.cache import EmbeddingCache
  
@@ -18,7 +18,7 @@ class DataTransformer:
  
     def __init__(
         self,
-        config: CustomerConfig,
+        config: SearchConfig,
         text_encoder: Optional[BgeEncoder] = None,
         image_encoder: Optional[CLIPImageEncoder] = None,
         use_cache: bool = True
@@ -27,7 +27,7 @@ class DataTransformer:
         Initialize data transformer.
  
         Args:
-            config: Customer configuration
+            config: Search configuration
             text_encoder: Text embedding encoder (lazy loaded if not provided)
             image_encoder: Image embedding encoder (lazy loaded if not provided)
             use_cache: Whether to use embedding cache
 """
 Elasticsearch mapping generator.
  
-Generates Elasticsearch index mappings from customer configuration.
+Generates Elasticsearch index mappings from search configuration.
 """
  
 from typing import Dict, Any
 from config import (
-    CustomerConfig,
+    SearchConfig,
     FieldConfig,
     get_es_mapping_for_field,
     get_default_analyzers,
@@ -15,9 +15,9 @@ from config import (
  
  
 class MappingGenerator:
-    """Generates Elasticsearch mapping from customer configuration."""
+    """Generates Elasticsearch mapping from search configuration."""
  
-    def __init__(self, config: CustomerConfig):
+    def __init__(self, config: SearchConfig):
         self.config = config
  
     def generate_mapping(self) -> Dict[str, Any]:
@@ -7,7 +7,7 @@ Handles query rewriting, translation, and embedding generation.
 from typing import Dict, List, Optional, Any
 import numpy as np
  
-from config import CustomerConfig, QueryConfig
+from config import SearchConfig, QueryConfig
 from embeddings import BgeEncoder
 from .language_detector import LanguageDetector
 from .translator import Translator
@@ -61,7 +61,7 @@ class QueryParser:
  
     def __init__(
         self,
-        config: CustomerConfig,
+        config: SearchConfig,
         text_encoder: Optional[BgeEncoder] = None,
         translator: Optional[Translator] = None
     ):
@@ -69,7 +69,7 @@ class QueryParser:
         Initialize query parser.
  
         Args:
-            config: Customer configuration
+            config: Search configuration
             text_encoder: Text embedding encoder (lazy loaded if not provided)
             translator: Translator instance (lazy loaded if not provided)
         """
@@ -9,7 +9,7 @@ maintaining a unified external interface.
 from typing import Dict, Any, List, Optional
 import numpy as np
  
-from config import CustomerConfig, IndexConfig
+from config import SearchConfig, IndexConfig
 from query import ParsedQuery
 from .es_query_builder import ESQueryBuilder
  
@@ -26,7 +26,7 @@ class MultiLanguageQueryBuilder(ESQueryBuilder):
  
     def __init__(
         self,
-        config: CustomerConfig,
+        config: SearchConfig,
         index_name: str,
         text_embedding_field: Optional[str] = None,
         image_embedding_field: Optional[str] = None,
@@ -7,7 +7,7 @@ Handles query parsing, boolean expressions, ranking, and result formatting.
 from typing import Dict, Any, List, Optional, Union
 import time
  
-from config import CustomerConfig
+from config import SearchConfig
 from utils.es_client import ESClient
 from query import QueryParser, ParsedQuery
 from indexer import MappingGenerator
@@ -75,7 +75,7 @@ class Searcher:
  
     def __init__(
         self,
-        config: CustomerConfig,
+        config: SearchConfig,
         es_client: ESClient,
         query_parser: Optional[QueryParser] = None
     ):
@@ -83,7 +83,7 @@ class Searcher:
         Initialize searcher.
  
         Args:
-            config: Customer configuration
+            config: Search configuration
             es_client: Elasticsearch client
             query_parser: Query parser (created if not provided)
         """
@@ -7,7 +7,7 @@ import sys
 import os
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
  
-from config import ConfigLoader, CustomerConfig
+from config import ConfigLoader, SearchConfig
  
 def test_source_fields_config():
     """测试source_fields配置是否正确加载"""
@@ -15,7 +15,7 @@ from unittest.mock import Mock, MagicMock
 project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, project_root)
  
-from config import CustomerConfig, QueryConfig, IndexConfig, FieldConfig, SPUConfig, RankingConfig, FunctionScoreConfig, RerankConfig
+from config import SearchConfig, QueryConfig, IndexConfig, FieldConfig, SPUConfig, RankingConfig, FunctionScoreConfig, RerankConfig
 from config.field_types import FieldType, AnalyzerType
 from utils.es_client import ESClient
 from search import Searcher
@@ -51,8 +51,8 @@ def sample_index_config() -&gt; IndexConfig:
  
  
 @pytest.fixture
-def sample_customer_config(sample_index_config) -> CustomerConfig:
-    """样例客户配置"""
+def sample_search_config(sample_index_config) -> SearchConfig:
+    """样例搜索配置"""
     query_config = QueryConfig(
         enable_query_rewrite=True,
         enable_translation=True,
@@ -74,7 +74,7 @@ def sample_customer_config(sample_index_config) -&gt; CustomerConfig:
     function_score_config = FunctionScoreConfig()
     rerank_config = RerankConfig()
  
-    return CustomerConfig(
+    return SearchConfig(
         es_index_name="test_products",
         fields=[
             FieldConfig(name="tenant_id", field_type=FieldType.KEYWORD, required=True),
@@ -134,18 +134,18 @@ def mock_es_client() -&gt; Mock:
  
  
 @pytest.fixture
-def test_searcher(sample_customer_config, mock_es_client) -> Searcher:
+def test_searcher(sample_search_config, mock_es_client) -> Searcher:
     """测试用Searcher实例"""
     return Searcher(
-        config=sample_customer_config,
+        config=sample_search_config,
         es_client=mock_es_client
     )
  
  
 @pytest.fixture
-def test_query_parser(sample_customer_config) -> QueryParser:
+def test_query_parser(sample_search_config) -> QueryParser:
     """测试用QueryParser实例"""
-    return QueryParser(sample_customer_config)
+    return QueryParser(sample_search_config)
  
  
 @pytest.fixture