Compare View

switch
from
...
to
 
Commits (3)
Showing 48 changed files   Show diff stats
@@ -86,8 +86,7 @@ limiter = Limiter(key_func=get_remote_address) @@ -86,8 +86,7 @@ limiter = Limiter(key_func=get_remote_address)
86 # Add parent directory to path 86 # Add parent directory to path
87 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 87 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
88 88
89 -from config.env_config import ES_CONFIG, DB_CONFIG  
90 -from config import ConfigLoader 89 +from config import get_app_config
91 from utils import ESClient 90 from utils import ESClient
92 from search import Searcher 91 from search import Searcher
93 from query import QueryParser 92 from query import QueryParser
@@ -99,7 +98,7 @@ _es_client: Optional[ESClient] = None @@ -99,7 +98,7 @@ _es_client: Optional[ESClient] = None
99 _searcher: Optional[Searcher] = None 98 _searcher: Optional[Searcher] = None
100 _query_parser: Optional[QueryParser] = None 99 _query_parser: Optional[QueryParser] = None
101 _suggestion_service: Optional[SuggestionService] = None 100 _suggestion_service: Optional[SuggestionService] = None
102 -_config = None 101 +_app_config = None
103 102
104 103
105 def init_service(es_host: str = "http://localhost:9200"): 104 def init_service(es_host: str = "http://localhost:9200"):
@@ -109,20 +108,20 @@ def init_service(es_host: str = "http://localhost:9200"): @@ -109,20 +108,20 @@ def init_service(es_host: str = "http://localhost:9200"):
109 Args: 108 Args:
110 es_host: Elasticsearch host URL 109 es_host: Elasticsearch host URL
111 """ 110 """
112 - global _es_client, _searcher, _query_parser, _suggestion_service, _config 111 + global _es_client, _searcher, _query_parser, _suggestion_service, _app_config
113 112
114 start_time = time.time() 113 start_time = time.time()
115 logger.info("Initializing search service (multi-tenant)") 114 logger.info("Initializing search service (multi-tenant)")
116 115
117 # Load configuration 116 # Load configuration
118 logger.info("Loading configuration...") 117 logger.info("Loading configuration...")
119 - config_loader = ConfigLoader("config/config.yaml")  
120 - _config = config_loader.load_config() 118 + _app_config = get_app_config()
  119 + search_config = _app_config.search
121 logger.info("Configuration loaded") 120 logger.info("Configuration loaded")
122 121
123 # Get ES credentials 122 # Get ES credentials
124 - es_username = os.getenv('ES_USERNAME') or ES_CONFIG.get('username')  
125 - es_password = os.getenv('ES_PASSWORD') or ES_CONFIG.get('password') 123 + es_username = _app_config.infrastructure.elasticsearch.username
  124 + es_password = _app_config.infrastructure.elasticsearch.password
126 125
127 # Connect to Elasticsearch 126 # Connect to Elasticsearch
128 logger.info(f"Connecting to Elasticsearch at {es_host}...") 127 logger.info(f"Connecting to Elasticsearch at {es_host}...")
@@ -139,15 +138,15 @@ def init_service(es_host: str = "http://localhost:9200"): @@ -139,15 +138,15 @@ def init_service(es_host: str = "http://localhost:9200"):
139 138
140 # Initialize components 139 # Initialize components
141 logger.info("Initializing query parser...") 140 logger.info("Initializing query parser...")
142 - _query_parser = QueryParser(_config) 141 + _query_parser = QueryParser(search_config)
143 142
144 logger.info("Initializing searcher...") 143 logger.info("Initializing searcher...")
145 - _searcher = Searcher(_es_client, _config, _query_parser) 144 + _searcher = Searcher(_es_client, search_config, _query_parser)
146 logger.info("Initializing suggestion service...") 145 logger.info("Initializing suggestion service...")
147 _suggestion_service = SuggestionService(_es_client) 146 _suggestion_service = SuggestionService(_es_client)
148 147
149 elapsed = time.time() - start_time 148 elapsed = time.time() - start_time
150 - logger.info(f"Search service ready! (took {elapsed:.2f}s) | Index: {_config.es_index_name}") 149 + logger.info(f"Search service ready! (took {elapsed:.2f}s) | Index: {search_config.es_index_name}")
151 150
152 151
153 152
@@ -182,9 +181,9 @@ def get_suggestion_service() -> SuggestionService: @@ -182,9 +181,9 @@ def get_suggestion_service() -> SuggestionService:
182 181
183 def get_config(): 182 def get_config():
184 """Get global config instance.""" 183 """Get global config instance."""
185 - if _config is None: 184 + if _app_config is None:
186 raise RuntimeError("Service not initialized") 185 raise RuntimeError("Service not initialized")
187 - return _config 186 + return _app_config
188 187
189 188
190 # Create FastAPI app with enhanced configuration 189 # Create FastAPI app with enhanced configuration
@@ -240,7 +239,7 @@ async def startup_event(): @@ -240,7 +239,7 @@ async def startup_event():
240 except Exception as e: 239 except Exception as e:
241 logger.warning(f"Failed to set thread pool size: {e}, using default") 240 logger.warning(f"Failed to set thread pool size: {e}, using default")
242 241
243 - es_host = os.getenv("ES_HOST", "http://localhost:9200") 242 + es_host = get_app_config().infrastructure.elasticsearch.host
244 logger.info("Starting E-Commerce Search API (Multi-Tenant)") 243 logger.info("Starting E-Commerce Search API (Multi-Tenant)")
245 logger.info(f"Elasticsearch Host: {es_host}") 244 logger.info(f"Elasticsearch Host: {es_host}")
246 245
api/indexer_app.py
@@ -38,8 +38,7 @@ logger = logging.getLogger(__name__) @@ -38,8 +38,7 @@ logger = logging.getLogger(__name__)
38 # Add parent directory to path 38 # Add parent directory to path
39 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 39 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
40 40
41 -from config.env_config import ES_CONFIG # noqa: E402  
42 -from config import ConfigLoader # noqa: E402 41 +from config import get_app_config # noqa: E402
43 from utils import ESClient # noqa: E402 42 from utils import ESClient # noqa: E402
44 from utils.db_connector import create_db_connection # noqa: E402 43 from utils.db_connector import create_db_connection # noqa: E402
45 from indexer.incremental_service import IncrementalIndexerService # noqa: E402 44 from indexer.incremental_service import IncrementalIndexerService # noqa: E402
@@ -55,7 +54,7 @@ from .service_registry import ( @@ -55,7 +54,7 @@ from .service_registry import (
55 54
56 55
57 _es_client: Optional[ESClient] = None 56 _es_client: Optional[ESClient] = None
58 -_config = None 57 +_app_config = None
59 _incremental_service: Optional[IncrementalIndexerService] = None 58 _incremental_service: Optional[IncrementalIndexerService] = None
60 _bulk_indexing_service: Optional[BulkIndexingService] = None 59 _bulk_indexing_service: Optional[BulkIndexingService] = None
61 _suggestion_builder: Optional[SuggestionIndexBuilder] = None 60 _suggestion_builder: Optional[SuggestionIndexBuilder] = None
@@ -68,20 +67,19 @@ def init_indexer_service(es_host: str = "http://localhost:9200"): @@ -68,20 +67,19 @@ def init_indexer_service(es_host: str = "http://localhost:9200"):
68 This mirrors the indexing-related initialization logic in api.app.init_service 67 This mirrors the indexing-related initialization logic in api.app.init_service
69 but without search-related components. 68 but without search-related components.
70 """ 69 """
71 - global _es_client, _config, _incremental_service, _bulk_indexing_service, _suggestion_builder 70 + global _es_client, _app_config, _incremental_service, _bulk_indexing_service, _suggestion_builder
72 71
73 start_time = time.time() 72 start_time = time.time()
74 logger.info("Initializing Indexer service") 73 logger.info("Initializing Indexer service")
75 74
76 # Load configuration (kept for parity/logging; indexer routes don't depend on it) 75 # Load configuration (kept for parity/logging; indexer routes don't depend on it)
77 logger.info("Loading configuration...") 76 logger.info("Loading configuration...")
78 - config_loader = ConfigLoader("config/config.yaml")  
79 - _config = config_loader.load_config() 77 + _app_config = get_app_config()
80 logger.info("Configuration loaded") 78 logger.info("Configuration loaded")
81 79
82 # Get ES credentials 80 # Get ES credentials
83 - es_username = os.getenv("ES_USERNAME") or ES_CONFIG.get("username")  
84 - es_password = os.getenv("ES_PASSWORD") or ES_CONFIG.get("password") 81 + es_username = _app_config.infrastructure.elasticsearch.username
  82 + es_password = _app_config.infrastructure.elasticsearch.password
85 83
86 # Connect to Elasticsearch 84 # Connect to Elasticsearch
87 logger.info(f"Connecting to Elasticsearch at {es_host} for indexer...") 85 logger.info(f"Connecting to Elasticsearch at {es_host} for indexer...")
@@ -97,11 +95,12 @@ def init_indexer_service(es_host: str = "http://localhost:9200"): @@ -97,11 +95,12 @@ def init_indexer_service(es_host: str = "http://localhost:9200"):
97 set_es_client(_es_client) 95 set_es_client(_es_client)
98 96
99 # Initialize indexing services (DB is required here) 97 # Initialize indexing services (DB is required here)
100 - db_host = os.getenv("DB_HOST")  
101 - db_port = int(os.getenv("DB_PORT", 3306))  
102 - db_database = os.getenv("DB_DATABASE")  
103 - db_username = os.getenv("DB_USERNAME")  
104 - db_password = os.getenv("DB_PASSWORD") 98 + db_config = _app_config.infrastructure.database
  99 + db_host = db_config.host
  100 + db_port = db_config.port
  101 + db_database = db_config.database
  102 + db_username = db_config.username
  103 + db_password = db_config.password
105 104
106 if all([db_host, db_database, db_username, db_password]): 105 if all([db_host, db_database, db_username, db_password]):
107 logger.info("Initializing database connection for indexing services...") 106 logger.info("Initializing database connection for indexing services...")
@@ -166,7 +165,7 @@ async def startup_event(): @@ -166,7 +165,7 @@ async def startup_event():
166 except Exception as e: 165 except Exception as e:
167 logger.warning(f"Failed to set thread pool size: {e}, using default") 166 logger.warning(f"Failed to set thread pool size: {e}, using default")
168 167
169 - es_host = os.getenv("ES_HOST", "http://localhost:9200") 168 + es_host = get_app_config().infrastructure.elasticsearch.host
170 logger.info("Starting Indexer API service") 169 logger.info("Starting Indexer API service")
171 logger.info(f"Elasticsearch Host: {es_host}") 170 logger.info(f"Elasticsearch Host: {es_host}")
172 try: 171 try:
@@ -176,14 +175,11 @@ async def startup_event(): @@ -176,14 +175,11 @@ async def startup_event():
176 # Eager warmup: build per-tenant transformer bundles at startup to avoid 175 # Eager warmup: build per-tenant transformer bundles at startup to avoid
177 # first-request latency (config/provider/encoder + transformer wiring). 176 # first-request latency (config/provider/encoder + transformer wiring).
178 try: 177 try:
179 - if _incremental_service is not None and _config is not None: 178 + if _incremental_service is not None and _app_config is not None:
180 tenants = [] 179 tenants = []
181 - # config.tenant_config shape: {"default": {...}, "tenants": {"1": {...}, ...}}  
182 - tc = getattr(_config, "tenant_config", None) or {}  
183 - if isinstance(tc, dict):  
184 - tmap = tc.get("tenants")  
185 - if isinstance(tmap, dict):  
186 - tenants = [str(k) for k in tmap.keys()] 180 + tmap = _app_config.tenants.tenants
  181 + if isinstance(tmap, dict):
  182 + tenants = [str(k) for k in tmap.keys()]
187 # If no explicit tenants configured, skip warmup. 183 # If no explicit tenants configured, skip warmup.
188 if tenants: 184 if tenants:
189 warm = _incremental_service.warmup_transformers(tenants) 185 warm = _incremental_service.warmup_transformers(tenants)
api/routes/admin.py
@@ -3,7 +3,6 @@ Admin API routes for configuration and management. @@ -3,7 +3,6 @@ Admin API routes for configuration and management.
3 """ 3 """
4 4
5 from fastapi import APIRouter, HTTPException, Request 5 from fastapi import APIRouter, HTTPException, Request
6 -from typing import Dict  
7 6
8 from ..models import HealthResponse, ErrorResponse 7 from ..models import HealthResponse, ErrorResponse
9 from indexer.mapping_generator import get_tenant_index_name 8 from indexer.mapping_generator import get_tenant_index_name
@@ -42,22 +41,12 @@ async def health_check(): @@ -42,22 +41,12 @@ async def health_check():
42 @router.get("/config") 41 @router.get("/config")
43 async def get_configuration(): 42 async def get_configuration():
44 """ 43 """
45 - Get current search configuration (sanitized). 44 + Get the effective application configuration (sanitized).
46 """ 45 """
47 try: 46 try:
48 from ..app import get_config 47 from ..app import get_config
49 48
50 - config = get_config()  
51 -  
52 - return {  
53 - "es_index_name": config.es_index_name,  
54 - "num_field_boosts": len(config.field_boosts),  
55 - "multilingual_fields": config.query_config.multilingual_fields,  
56 - "shared_fields": config.query_config.shared_fields,  
57 - "core_multilingual_fields": config.query_config.core_multilingual_fields,  
58 - "supported_languages": config.query_config.supported_languages,  
59 - "spu_enabled": config.spu_config.enabled  
60 - } 49 + return get_config().sanitized_dict()
61 50
62 except HTTPException: 51 except HTTPException:
63 raise 52 raise
@@ -65,45 +54,21 @@ async def get_configuration(): @@ -65,45 +54,21 @@ async def get_configuration():
65 raise HTTPException(status_code=500, detail=str(e)) 54 raise HTTPException(status_code=500, detail=str(e))
66 55
67 56
68 -@router.post("/rewrite-rules")  
69 -async def update_rewrite_rules(rules: Dict[str, str]):  
70 - """  
71 - Update query rewrite rules.  
72 -  
73 - Args:  
74 - rules: Dictionary of pattern -> replacement mappings  
75 - """ 57 +@router.get("/config/meta")
  58 +async def get_configuration_meta():
  59 + """Get configuration metadata for observability."""
76 try: 60 try:
77 - from ..app import get_query_parser  
78 -  
79 - query_parser = get_query_parser()  
80 - query_parser.update_rewrite_rules(rules)  
81 -  
82 - return {  
83 - "status": "success",  
84 - "message": f"Updated {len(rules)} rewrite rules"  
85 - }  
86 -  
87 - except Exception as e:  
88 - raise HTTPException(status_code=500, detail=str(e))  
89 -  
90 -  
91 -@router.get("/rewrite-rules")  
92 -async def get_rewrite_rules():  
93 - """  
94 - Get current query rewrite rules.  
95 - """  
96 - try:  
97 - from ..app import get_query_parser  
98 -  
99 - query_parser = get_query_parser()  
100 - rules = query_parser.get_rewrite_rules() 61 + from ..app import get_config
101 62
  63 + config = get_config()
102 return { 64 return {
103 - "rules": rules,  
104 - "count": len(rules) 65 + "environment": config.runtime.environment,
  66 + "config_hash": config.metadata.config_hash,
  67 + "loaded_files": list(config.metadata.loaded_files),
  68 + "deprecated_keys": list(config.metadata.deprecated_keys),
105 } 69 }
106 - 70 + except HTTPException:
  71 + raise
107 except Exception as e: 72 except Exception as e:
108 raise HTTPException(status_code=500, detail=str(e)) 73 raise HTTPException(status_code=500, detail=str(e))
109 74
config/__init__.py
1 -"""  
2 -Configuration package for search engine. 1 +"""Unified configuration package exports."""
3 2
4 -Provides configuration loading, validation, and utility functions.  
5 -"""  
6 -  
7 -from .config_loader import (  
8 - SearchConfig,  
9 - QueryConfig,  
10 - IndexConfig,  
11 - SPUConfig, 3 +from config.config_loader import ConfigLoader, ConfigurationError
  4 +from config.loader import AppConfigLoader, get_app_config, reload_app_config
  5 +from config.schema import (
  6 + AppConfig,
12 FunctionScoreConfig, 7 FunctionScoreConfig,
  8 + IndexConfig,
  9 + QueryConfig,
13 RerankConfig, 10 RerankConfig,
14 - ConfigLoader,  
15 - ConfigurationError  
16 -)  
17 -  
18 -from .utils import (  
19 - get_match_fields_for_index,  
20 - get_domain_fields 11 + SPUConfig,
  12 + SearchConfig,
  13 + ServicesConfig,
21 ) 14 )
22 -from .services_config import (  
23 - get_translation_config,  
24 - get_embedding_config,  
25 - get_rerank_config, 15 +from config.services_config import (
26 get_embedding_backend_config, 16 get_embedding_backend_config,
27 - get_rerank_backend_config,  
28 - get_translation_base_url,  
29 - get_embedding_text_base_url, 17 + get_embedding_config,
  18 + get_embedding_image_backend_config,
30 get_embedding_image_base_url, 19 get_embedding_image_base_url,
  20 + get_embedding_text_base_url,
  21 + get_rerank_backend_config,
  22 + get_rerank_config,
31 get_rerank_service_url, 23 get_rerank_service_url,
  24 + get_translation_base_url,
32 get_translation_cache_config, 25 get_translation_cache_config,
33 - ServiceConfig, 26 + get_translation_config,
34 ) 27 )
  28 +from config.utils import get_domain_fields, get_match_fields_for_index
35 29
36 __all__ = [ 30 __all__ = [
37 - # Main config classes  
38 - 'SearchConfig',  
39 - 'QueryConfig',  
40 - 'IndexConfig',  
41 - 'SPUConfig',  
42 - 'FunctionScoreConfig',  
43 - 'RerankConfig',  
44 -  
45 - # Loader and utilities  
46 - 'ConfigLoader',  
47 - 'ConfigurationError',  
48 - 'get_match_fields_for_index',  
49 - 'get_domain_fields',  
50 - 'get_translation_config',  
51 - 'get_embedding_config',  
52 - 'get_rerank_config',  
53 - 'get_embedding_backend_config',  
54 - 'get_rerank_backend_config',  
55 - 'get_translation_base_url',  
56 - 'get_embedding_text_base_url',  
57 - 'get_embedding_image_base_url',  
58 - 'get_rerank_service_url',  
59 - 'get_translation_cache_config',  
60 - 'ServiceConfig', 31 + "AppConfig",
  32 + "AppConfigLoader",
  33 + "ConfigLoader",
  34 + "ConfigurationError",
  35 + "FunctionScoreConfig",
  36 + "IndexConfig",
  37 + "QueryConfig",
  38 + "RerankConfig",
  39 + "SPUConfig",
  40 + "SearchConfig",
  41 + "ServicesConfig",
  42 + "get_app_config",
  43 + "reload_app_config",
  44 + "get_domain_fields",
  45 + "get_match_fields_for_index",
  46 + "get_translation_config",
  47 + "get_embedding_config",
  48 + "get_rerank_config",
  49 + "get_embedding_backend_config",
  50 + "get_embedding_image_backend_config",
  51 + "get_rerank_backend_config",
  52 + "get_translation_base_url",
  53 + "get_embedding_text_base_url",
  54 + "get_embedding_image_base_url",
  55 + "get_rerank_service_url",
  56 + "get_translation_cache_config",
61 ] 57 ]
config/config.yaml
@@ -5,6 +5,14 @@ @@ -5,6 +5,14 @@
5 # Elasticsearch Index 5 # Elasticsearch Index
6 es_index_name: "search_products" 6 es_index_name: "search_products"
7 7
  8 +# Config assets
  9 +assets:
  10 + query_rewrite_dictionary_path: "config/dictionaries/query_rewrite.dict"
  11 +
  12 +# Product content understanding (LLM enrich-content) configuration
  13 +product_enrich:
  14 + max_workers: 40
  15 +
8 # ES Index Settings (基础设置) 16 # ES Index Settings (基础设置)
9 es_settings: 17 es_settings:
10 number_of_shards: 1 18 number_of_shards: 1
@@ -211,6 +219,19 @@ services: @@ -211,6 +219,19 @@ services:
211 device: "cuda" 219 device: "cuda"
212 batch_size: 32 220 batch_size: 32
213 normalize_embeddings: true 221 normalize_embeddings: true
  222 + # 服务内图片后端(embedding 进程启动时读取)
  223 + image_backend: "clip_as_service" # clip_as_service | local_cnclip
  224 + image_backends:
  225 + clip_as_service:
  226 + server: "grpc://127.0.0.1:51000"
  227 + model_name: "CN-CLIP/ViT-L-14"
  228 + batch_size: 8
  229 + normalize_embeddings: true
  230 + local_cnclip:
  231 + model_name: "ViT-L-14"
  232 + device: null
  233 + batch_size: 8
  234 + normalize_embeddings: true
214 rerank: 235 rerank:
215 provider: "http" 236 provider: "http"
216 base_url: "http://127.0.0.1:6007" 237 base_url: "http://127.0.0.1:6007"
@@ -218,6 +239,9 @@ services: @@ -218,6 +239,9 @@ services:
218 http: 239 http:
219 base_url: "http://127.0.0.1:6007" 240 base_url: "http://127.0.0.1:6007"
220 service_url: "http://127.0.0.1:6007/rerank" 241 service_url: "http://127.0.0.1:6007/rerank"
  242 + request:
  243 + max_docs: 1000
  244 + normalize: true
221 # 服务内后端(reranker 进程启动时读取) 245 # 服务内后端(reranker 进程启动时读取)
222 backend: "qwen3_vllm" # bge | qwen3_vllm | qwen3_transformers | dashscope_rerank 246 backend: "qwen3_vllm" # bge | qwen3_vllm | qwen3_transformers | dashscope_rerank
223 backends: 247 backends:
config/config_loader.py
1 """ 1 """
2 -Configuration loader and validator for search engine configurations. 2 +Compatibility wrapper for search-behavior config access.
3 3
4 -This module handles loading, parsing, and validating YAML configuration files  
5 -that define how search should be executed (NOT how data should be indexed).  
6 -  
7 -索引结构由 mappings/search_products.json 定义。  
8 -此配置只定义搜索行为:字段权重、搜索域、查询策略等。 4 +The unified loader lives in :mod:`config.loader`. This module now exposes the
  5 +search subtree only, so existing search/indexer code can consume a single
  6 +source-of-truth search config without reparsing YAML separately.
9 """ 7 """
10 8
11 -import yaml  
12 -from typing import Dict, Any, List, Optional  
13 -from dataclasses import dataclass, field  
14 -from pathlib import Path  
15 -  
16 -  
17 -@dataclass  
18 -class IndexConfig:  
19 - """Configuration for an index domain (e.g., default, title, brand)."""  
20 - name: str  
21 - label: str  
22 - fields: List[str] # List of field names to include in this search domain  
23 - boost: float = 1.0  
24 - example: Optional[str] = None  
25 -  
26 -  
27 -@dataclass  
28 -class QueryConfig:  
29 - """Configuration for query processing."""  
30 - supported_languages: List[str] = field(default_factory=lambda: ["zh", "en"])  
31 - default_language: str = "en"  
32 -  
33 - # Feature flags  
34 - enable_text_embedding: bool = True  
35 - enable_query_rewrite: bool = True  
36 -  
37 - # Query rewrite dictionary (loaded from external file)  
38 - rewrite_dictionary: Dict[str, str] = field(default_factory=dict)  
39 -  
40 - # Embedding field names  
41 - text_embedding_field: Optional[str] = "title_embedding"  
42 - image_embedding_field: Optional[str] = None  
43 -  
44 - # Source fields configuration  
45 - source_fields: Optional[List[str]] = None  
46 -  
47 - # KNN boost configuration  
48 - knn_boost: float = 0.25 # Boost value for KNN (embedding recall)  
49 -  
50 - # Dynamic text fields for multi-language retrieval  
51 - multilingual_fields: List[str] = field(  
52 - default_factory=lambda: ["title", "brief", "description", "vendor", "category_path", "category_name_text"]  
53 - )  
54 - shared_fields: List[str] = field(  
55 - default_factory=lambda: ["tags", "option1_values", "option2_values", "option3_values"]  
56 - )  
57 - core_multilingual_fields: List[str] = field(  
58 - default_factory=lambda: ["title", "brief", "vendor", "category_name_text"]  
59 - )  
60 -  
61 - # Unified text strategy tuning  
62 - base_minimum_should_match: str = "75%"  
63 - translation_minimum_should_match: str = "75%"  
64 - translation_boost: float = 0.4  
65 - translation_boost_when_source_missing: float = 1.0  
66 - source_boost_when_missing: float = 0.6  
67 - original_query_fallback_boost_when_translation_missing: float = 0.2  
68 - tie_breaker_base_query: float = 0.9  
69 -  
70 - # Query-time translation model selection (configurable)  
71 - # - zh_to_en_model: model for zh -> en  
72 - # - en_to_zh_model: model for en -> zh  
73 - # - default_translation_model: fallback model for all other language pairs  
74 - zh_to_en_model: str = "opus-mt-zh-en"  
75 - en_to_zh_model: str = "opus-mt-en-zh"  
76 - default_translation_model: str = "nllb-200-distilled-600m"  
77 -  
78 -  
79 -@dataclass  
80 -class SPUConfig:  
81 - """Configuration for SPU aggregation."""  
82 - enabled: bool = False  
83 - spu_field: Optional[str] = None  
84 - inner_hits_size: int = 3  
85 - # 配置哪些option维度参与检索(进索引、以及在线搜索)  
86 - searchable_option_dimensions: List[str] = field(default_factory=lambda: ['option1', 'option2', 'option3'])  
87 -  
88 -  
89 -@dataclass  
90 -class FunctionScoreConfig:  
91 - """Function Score配置(ES层打分规则)"""  
92 - score_mode: str = "sum"  
93 - boost_mode: str = "multiply"  
94 - functions: List[Dict[str, Any]] = field(default_factory=list)  
95 -  
96 -  
97 -@dataclass  
98 -class RerankConfig:  
99 - """重排配置(provider/URL 在 services.rerank)"""  
100 - enabled: bool = True  
101 - rerank_window: int = 384  
102 - timeout_sec: float = 15.0  
103 - weight_es: float = 0.4  
104 - weight_ai: float = 0.6  
105 - rerank_query_template: str = "{query}"  
106 - rerank_doc_template: str = "{title}"  
107 -  
108 -  
109 -@dataclass  
110 -class SearchConfig:  
111 - """Complete configuration for search engine (multi-tenant)."""  
112 -  
113 - # 字段权重配置(用于搜索)  
114 - field_boosts: Dict[str, float]  
115 -  
116 - # Legacy index domains (deprecated; kept for compatibility)  
117 - indexes: List[IndexConfig]  
118 -  
119 - # Query processing  
120 - query_config: QueryConfig  
121 -  
122 - # Function Score configuration (ES层打分)  
123 - function_score: FunctionScoreConfig  
124 -  
125 - # Rerank configuration (本地重排)  
126 - rerank: RerankConfig  
127 -  
128 - # SPU configuration  
129 - spu_config: SPUConfig  
130 -  
131 - # ES index settings  
132 - es_index_name: str  
133 -  
134 - # Tenant configuration  
135 - tenant_config: Dict[str, Any] = field(default_factory=dict)  
136 -  
137 - # ES settings  
138 - es_settings: Dict[str, Any] = field(default_factory=dict)  
139 - # Extensible service/provider registry (translation/embedding/rerank/...)  
140 - services: Dict[str, Any] = field(default_factory=dict) 9 +from __future__ import annotations
141 10
  11 +from dataclasses import asdict
  12 +from pathlib import Path
  13 +from typing import Any, Dict, List, Optional
142 14
143 -class ConfigurationError(Exception):  
144 - """Raised when configuration validation fails."""  
145 - pass 15 +from config.loader import AppConfigLoader, ConfigurationError
  16 +from config.schema import (
  17 + FunctionScoreConfig,
  18 + IndexConfig,
  19 + QueryConfig,
  20 + RerankConfig,
  21 + SPUConfig,
  22 + SearchConfig,
  23 +)
146 24
147 25
148 class ConfigLoader: 26 class ConfigLoader:
149 - """Loads and validates unified search engine configuration from YAML file."""  
150 -  
151 - def __init__(self, config_file: Optional[Path] = None):  
152 - """  
153 - Initialize config loader.  
154 -  
155 - Args:  
156 - config_file: Path to config YAML file (defaults to config/config.yaml)  
157 - """  
158 - if config_file is None:  
159 - config_file = Path(__file__).parent / "config.yaml"  
160 - self.config_file = Path(config_file)  
161 -  
162 - def _load_rewrite_dictionary(self) -> Dict[str, str]:  
163 - """Load query rewrite dictionary from external file."""  
164 - rewrite_file = Path(__file__).parent / "rewrite_dictionary.txt"  
165 - rewrite_dict = {}  
166 -  
167 - if not rewrite_file.exists():  
168 - return rewrite_dict  
169 -  
170 - try:  
171 - with open(rewrite_file, 'r', encoding='utf-8') as f:  
172 - for line in f:  
173 - line = line.strip()  
174 - if not line or line.startswith('#'):  
175 - continue  
176 -  
177 - parts = line.split('\t')  
178 - if len(parts) >= 2:  
179 - original = parts[0].strip()  
180 - replacement = parts[1].strip()  
181 - if original and replacement:  
182 - rewrite_dict[original] = replacement  
183 - except Exception as e:  
184 - print(f"Warning: Failed to load rewrite dictionary: {e}")  
185 -  
186 - return rewrite_dict  
187 -  
188 - def load_config(self, validate: bool = True) -> SearchConfig:  
189 - """  
190 - Load unified configuration from YAML file.  
191 -  
192 - Args:  
193 - validate: Whether to validate configuration after loading  
194 -  
195 - Returns:  
196 - SearchConfig object  
197 -  
198 - Raises:  
199 - ConfigurationError: If config file not found, invalid, or validation fails  
200 - """  
201 - if not self.config_file.exists():  
202 - raise ConfigurationError(f"Configuration file not found: {self.config_file}")  
203 -  
204 - try:  
205 - with open(self.config_file, 'r', encoding='utf-8') as f:  
206 - config_data = yaml.safe_load(f)  
207 - except yaml.YAMLError as e:  
208 - raise ConfigurationError(f"Invalid YAML in {self.config_file}: {e}")  
209 -  
210 - config = self._parse_config(config_data)  
211 -  
212 - # Auto-validate configuration  
213 - if validate:  
214 - errors = self.validate_config(config)  
215 - if errors:  
216 - error_msg = "Configuration validation failed:\n" + "\n".join(f" - {err}" for err in errors)  
217 - raise ConfigurationError(error_msg)  
218 -  
219 - return config  
220 -  
221 - def _parse_config(self, config_data: Dict[str, Any]) -> SearchConfig:  
222 - """Parse configuration dictionary into SearchConfig object."""  
223 -  
224 - # Parse field_boosts  
225 - field_boosts = config_data.get("field_boosts", {})  
226 - if not isinstance(field_boosts, dict):  
227 - raise ConfigurationError("field_boosts must be a dictionary")  
228 -  
229 - # Parse indexes (deprecated; compatibility only)  
230 - indexes = []  
231 - for index_data in config_data.get("indexes", []):  
232 - indexes.append(self._parse_index_config(index_data))  
233 -  
234 - # Parse query config  
235 - query_config_data = config_data.get("query_config", {})  
236 - rewrite_dictionary = self._load_rewrite_dictionary()  
237 - search_fields_cfg = query_config_data.get("search_fields", {})  
238 - text_strategy_cfg = query_config_data.get("text_query_strategy", {}) 27 + """Load the unified app config and return the search subtree."""
239 28
240 - query_config = QueryConfig(  
241 - supported_languages=query_config_data.get("supported_languages") or ["zh", "en"],  
242 - default_language=query_config_data.get("default_language") or "en",  
243 - enable_text_embedding=query_config_data.get("enable_text_embedding", True),  
244 - enable_query_rewrite=query_config_data.get("enable_query_rewrite", True),  
245 - rewrite_dictionary=rewrite_dictionary,  
246 - text_embedding_field=query_config_data.get("text_embedding_field"),  
247 - image_embedding_field=query_config_data.get("image_embedding_field"),  
248 - source_fields=query_config_data.get("source_fields"),  
249 - knn_boost=query_config_data.get("knn_boost", 0.25),  
250 - multilingual_fields=search_fields_cfg.get(  
251 - "multilingual_fields",  
252 - ["title", "brief", "description", "vendor", "category_path", "category_name_text"],  
253 - ),  
254 - shared_fields=search_fields_cfg.get(  
255 - "shared_fields",  
256 - ["tags", "option1_values", "option2_values", "option3_values"],  
257 - ),  
258 - core_multilingual_fields=search_fields_cfg.get(  
259 - "core_multilingual_fields",  
260 - ["title", "brief", "vendor", "category_name_text"],  
261 - ),  
262 - base_minimum_should_match=str(text_strategy_cfg.get("base_minimum_should_match", "75%")),  
263 - translation_minimum_should_match=str(text_strategy_cfg.get("translation_minimum_should_match", "75%")),  
264 - translation_boost=float(text_strategy_cfg.get("translation_boost", 0.4)),  
265 - translation_boost_when_source_missing=float(  
266 - text_strategy_cfg.get("translation_boost_when_source_missing", 1.0)  
267 - ),  
268 - source_boost_when_missing=float(text_strategy_cfg.get("source_boost_when_missing", 0.6)),  
269 - original_query_fallback_boost_when_translation_missing=float(  
270 - text_strategy_cfg.get("original_query_fallback_boost_when_translation_missing", 0.2)  
271 - ),  
272 - tie_breaker_base_query=float(text_strategy_cfg.get("tie_breaker_base_query", 0.9)),  
273 - zh_to_en_model=str(query_config_data.get("zh_to_en_model") or "opus-mt-zh-en"),  
274 - en_to_zh_model=str(query_config_data.get("en_to_zh_model") or "opus-mt-en-zh"),  
275 - default_translation_model=str(  
276 - query_config_data.get("default_translation_model") or "nllb-200-distilled-600m"  
277 - ),  
278 - )  
279 -  
280 - # Parse Function Score configuration  
281 - fs_data = config_data.get("function_score", {})  
282 - function_score = FunctionScoreConfig(  
283 - score_mode=fs_data.get("score_mode") or "sum",  
284 - boost_mode=fs_data.get("boost_mode") or "multiply",  
285 - functions=fs_data.get("functions") or []  
286 - )  
287 -  
288 - # Parse Rerank (provider/URL in services.rerank)  
289 - rerank_data = config_data.get("rerank", {})  
290 - rerank = RerankConfig(  
291 - enabled=bool(rerank_data.get("enabled", True)),  
292 - rerank_window=int(rerank_data.get("rerank_window", 384)),  
293 - timeout_sec=float(rerank_data.get("timeout_sec", 15.0)),  
294 - weight_es=float(rerank_data.get("weight_es", 0.4)),  
295 - weight_ai=float(rerank_data.get("weight_ai", 0.6)),  
296 - rerank_query_template=str(rerank_data.get("rerank_query_template") or "{query}"),  
297 - rerank_doc_template=str(rerank_data.get("rerank_doc_template") or "{title}"),  
298 - )  
299 -  
300 - # Parse SPU config  
301 - spu_data = config_data.get("spu_config", {})  
302 - spu_config = SPUConfig(  
303 - enabled=spu_data.get("enabled", False),  
304 - spu_field=spu_data.get("spu_field"),  
305 - inner_hits_size=spu_data.get("inner_hits_size", 3),  
306 - searchable_option_dimensions=spu_data.get("searchable_option_dimensions", ['option1', 'option2', 'option3'])  
307 - )  
308 -  
309 - # Parse tenant config  
310 - tenant_config_data = config_data.get("tenant_config", {}) 29 + def __init__(self, config_file: Optional[Path] = None) -> None:
  30 + self._loader = AppConfigLoader(config_file=Path(config_file) if config_file is not None else None)
311 31
312 - # Parse extensible services/provider registry  
313 - services_data = config_data.get("services", {}) or {}  
314 - if not isinstance(services_data, dict):  
315 - raise ConfigurationError("services must be a dictionary if provided") 32 + def load_config(self, validate: bool = True) -> SearchConfig:
  33 + return self._loader.load(validate=validate).search
316 34
317 - return SearchConfig(  
318 - field_boosts=field_boosts,  
319 - indexes=indexes,  
320 - query_config=query_config,  
321 - function_score=function_score,  
322 - rerank=rerank,  
323 - spu_config=spu_config,  
324 - tenant_config=tenant_config_data,  
325 - es_index_name=config_data.get("es_index_name", "search_products"),  
326 - es_settings=config_data.get("es_settings", {}),  
327 - services=services_data  
328 - )  
329 -  
330 - def _parse_index_config(self, index_data: Dict[str, Any]) -> IndexConfig:  
331 - """Parse index configuration from dictionary."""  
332 - return IndexConfig(  
333 - name=index_data["name"],  
334 - label=index_data.get("label", index_data["name"]),  
335 - fields=index_data.get("fields", []),  
336 - boost=index_data.get("boost", 1.0),  
337 - example=index_data.get("example")  
338 - )  
339 -  
340 def validate_config(self, config: SearchConfig) -> List[str]: 35 def validate_config(self, config: SearchConfig) -> List[str]:
341 - """  
342 - Validate configuration for common errors.  
343 -  
344 - Args:  
345 - config: SearchConfig to validate  
346 -  
347 - Returns:  
348 - List of error messages (empty if valid)  
349 - """  
350 - errors = []  
351 -  
352 - # Validate es_index_name 36 + errors: List[str] = []
353 if not config.es_index_name: 37 if not config.es_index_name:
354 errors.append("es_index_name is required") 38 errors.append("es_index_name is required")
355 -  
356 - # Validate field_boosts  
357 if not config.field_boosts: 39 if not config.field_boosts:
358 errors.append("field_boosts is empty") 40 errors.append("field_boosts is empty")
359 -  
360 - for field_name, boost in config.field_boosts.items():  
361 - if not isinstance(boost, (int, float)):  
362 - errors.append(f"field_boosts['{field_name}']: boost must be a number, got {type(boost).__name__}")  
363 - elif boost < 0:  
364 - errors.append(f"field_boosts['{field_name}']: boost must be non-negative")  
365 -  
366 - # Validate indexes (deprecated, optional)  
367 - index_names = set()  
368 - for index in config.indexes:  
369 - # Check for duplicate index names  
370 - if index.name in index_names:  
371 - errors.append(f"Duplicate index name: {index.name}")  
372 - index_names.add(index.name)  
373 -  
374 - # Validate fields in index  
375 - if not index.fields:  
376 - errors.append(f"Index '{index.name}': fields list is empty")  
377 -  
378 - # Validate SPU config  
379 - if config.spu_config.enabled:  
380 - if not config.spu_config.spu_field:  
381 - errors.append("SPU aggregation enabled but no spu_field specified")  
382 -  
383 - # Validate query config  
384 - if not config.query_config.supported_languages:  
385 - errors.append("At least one supported language must be specified")  
386 -  
387 if config.query_config.default_language not in config.query_config.supported_languages: 41 if config.query_config.default_language not in config.query_config.supported_languages:
388 - errors.append(  
389 - f"Default language '{config.query_config.default_language}' "  
390 - f"not in supported languages: {config.query_config.supported_languages}"  
391 - )  
392 -  
393 - # Validate dynamic search fields  
394 - def _validate_str_list(name: str, values: List[str]) -> None:  
395 - if not isinstance(values, list) or not values:  
396 - errors.append(f"query_config.{name} must be a non-empty list[str]")  
397 - return  
398 - for i, val in enumerate(values):  
399 - if not isinstance(val, str) or not val.strip():  
400 - errors.append(f"query_config.{name}[{i}] must be a non-empty string")  
401 -  
402 - _validate_str_list("multilingual_fields", config.query_config.multilingual_fields)  
403 - _validate_str_list("shared_fields", config.query_config.shared_fields)  
404 - _validate_str_list("core_multilingual_fields", config.query_config.core_multilingual_fields)  
405 -  
406 - core_set = set(config.query_config.core_multilingual_fields)  
407 - multi_set = set(config.query_config.multilingual_fields)  
408 - if not core_set.issubset(multi_set):  
409 - errors.append("query_config.core_multilingual_fields must be subset of multilingual_fields")  
410 -  
411 - # Validate text query strategy numbers  
412 - for name in (  
413 - "translation_boost",  
414 - "translation_boost_when_source_missing",  
415 - "source_boost_when_missing",  
416 - "original_query_fallback_boost_when_translation_missing",  
417 - "tie_breaker_base_query",  
418 - ):  
419 - value = getattr(config.query_config, name, None)  
420 - if not isinstance(value, (int, float)):  
421 - errors.append(f"query_config.{name} must be a number")  
422 - elif value < 0:  
423 - errors.append(f"query_config.{name} must be non-negative")  
424 -  
425 - # Validate source_fields tri-state semantics  
426 - source_fields = config.query_config.source_fields  
427 - if source_fields is not None:  
428 - if not isinstance(source_fields, list):  
429 - errors.append("query_config.source_fields must be null or list[str]")  
430 - else:  
431 - for idx, field_name in enumerate(source_fields):  
432 - if not isinstance(field_name, str) or not field_name.strip():  
433 - errors.append(  
434 - f"query_config.source_fields[{idx}] must be a non-empty string"  
435 - )  
436 -  
437 - # Validate tenant config shape (default must exist in config)  
438 - tenant_cfg = config.tenant_config  
439 - if not isinstance(tenant_cfg, dict):  
440 - errors.append("tenant_config must be an object")  
441 - else:  
442 - default_cfg = tenant_cfg.get("default")  
443 - if not isinstance(default_cfg, dict):  
444 - errors.append("tenant_config.default must be configured")  
445 - else:  
446 - index_languages = default_cfg.get("index_languages")  
447 - if not isinstance(index_languages, list) or len(index_languages) == 0:  
448 - errors.append("tenant_config.default.index_languages must be a non-empty list")  
449 - 42 + errors.append("default_language must be included in supported_languages")
  43 + if config.spu_config.enabled and not config.spu_config.spu_field:
  44 + errors.append("spu_field is required when SPU is enabled")
450 return errors 45 return errors
451 - 46 +
452 def to_dict(self, config: SearchConfig) -> Dict[str, Any]: 47 def to_dict(self, config: SearchConfig) -> Dict[str, Any]:
453 - """Convert SearchConfig to dictionary representation."""  
454 -  
455 - # Build query_config dict  
456 - query_config_dict = {  
457 - "supported_languages": config.query_config.supported_languages,  
458 - "default_language": config.query_config.default_language,  
459 - "enable_text_embedding": config.query_config.enable_text_embedding,  
460 - "enable_query_rewrite": config.query_config.enable_query_rewrite,  
461 - "text_embedding_field": config.query_config.text_embedding_field,  
462 - "image_embedding_field": config.query_config.image_embedding_field,  
463 - "source_fields": config.query_config.source_fields,  
464 - "search_fields": {  
465 - "multilingual_fields": config.query_config.multilingual_fields,  
466 - "shared_fields": config.query_config.shared_fields,  
467 - "core_multilingual_fields": config.query_config.core_multilingual_fields,  
468 - },  
469 - "text_query_strategy": {  
470 - "base_minimum_should_match": config.query_config.base_minimum_should_match,  
471 - "translation_minimum_should_match": config.query_config.translation_minimum_should_match,  
472 - "translation_boost": config.query_config.translation_boost,  
473 - "translation_boost_when_source_missing": config.query_config.translation_boost_when_source_missing,  
474 - "source_boost_when_missing": config.query_config.source_boost_when_missing,  
475 - "original_query_fallback_boost_when_translation_missing": (  
476 - config.query_config.original_query_fallback_boost_when_translation_missing  
477 - ),  
478 - "tie_breaker_base_query": config.query_config.tie_breaker_base_query,  
479 - }  
480 - }  
481 -  
482 - return {  
483 - "es_index_name": config.es_index_name,  
484 - "es_settings": config.es_settings,  
485 - "field_boosts": config.field_boosts,  
486 - "indexes": [self._index_to_dict(index) for index in config.indexes],  
487 - "query_config": query_config_dict,  
488 - "function_score": {  
489 - "score_mode": config.function_score.score_mode,  
490 - "boost_mode": config.function_score.boost_mode,  
491 - "functions": config.function_score.functions  
492 - },  
493 - "rerank": {  
494 - "enabled": config.rerank.enabled,  
495 - "rerank_window": config.rerank.rerank_window,  
496 - "timeout_sec": config.rerank.timeout_sec,  
497 - "weight_es": config.rerank.weight_es,  
498 - "weight_ai": config.rerank.weight_ai,  
499 - "rerank_query_template": config.rerank.rerank_query_template,  
500 - "rerank_doc_template": config.rerank.rerank_doc_template,  
501 - },  
502 - "spu_config": {  
503 - "enabled": config.spu_config.enabled,  
504 - "spu_field": config.spu_config.spu_field,  
505 - "inner_hits_size": config.spu_config.inner_hits_size,  
506 - "searchable_option_dimensions": config.spu_config.searchable_option_dimensions  
507 - },  
508 - "services": config.services,  
509 - }  
510 -  
511 - def _index_to_dict(self, index: IndexConfig) -> Dict[str, Any]:  
512 - """Convert IndexConfig to dictionary."""  
513 - result = {  
514 - "name": index.name,  
515 - "label": index.label,  
516 - "fields": index.fields,  
517 - "boost": index.boost  
518 - }  
519 -  
520 - if index.example:  
521 - result["example"] = index.example  
522 -  
523 - return result 48 + return asdict(config)
config/dictionaries/query_rewrite.dict 0 → 100644
@@ -0,0 +1,2 @@ @@ -0,0 +1,2 @@
  1 +玩具 category.keyword:玩具 OR default:玩具
  2 +消防 category.keyword:消防 OR default:消防
config/env_config.py
1 """ 1 """
2 -Centralized configuration management for saas-search. 2 +Compatibility accessors for infrastructure/runtime environment settings.
3 3
4 -Loads configuration from environment variables and .env file.  
5 -This module provides a single point for loading .env and setting defaults.  
6 -All configuration variables are exported directly - no need for getter functions. 4 +All values are derived from the unified application config. This module no
  5 +longer owns any independent loading or precedence rules.
7 """ 6 """
8 7
9 -import os  
10 -from pathlib import Path  
11 -from dotenv import load_dotenv  
12 -  
13 -# Load .env file from project root  
14 -PROJECT_ROOT = Path(__file__).parent.parent  
15 -load_dotenv(PROJECT_ROOT / '.env')  
16 -  
17 -  
18 -# Elasticsearch Configuration  
19 -ES_CONFIG = {  
20 - 'host': os.getenv('ES_HOST', 'http://localhost:9200'),  
21 - 'username': os.getenv('ES_USERNAME'),  
22 - 'password': os.getenv('ES_PASSWORD'),  
23 -}  
24 -  
25 -# Runtime environment & index namespace  
26 -# RUNTIME_ENV: 当前运行环境,建议使用 prod / uat / test / dev 等枚举值  
27 -RUNTIME_ENV = os.getenv('RUNTIME_ENV', 'prod')  
28 -# ES_INDEX_NAMESPACE: 用于按环境隔离索引的命名空间前缀,例如 "uat_" / "test_"  
29 -# 为空字符串时表示不加前缀(通常是 prod 环境)  
30 -ES_INDEX_NAMESPACE = os.getenv('ES_INDEX_NAMESPACE')  
31 -if ES_INDEX_NAMESPACE is None:  
32 - # 未显式配置时,非 prod 环境默认加 "<env>_" 前缀,prod 环境默认不加前缀  
33 - ES_INDEX_NAMESPACE = '' if RUNTIME_ENV == 'prod' else f'{RUNTIME_ENV}_'  
34 -  
35 -# Redis Configuration  
36 -REDIS_CONFIG = {  
37 - 'host': os.getenv('REDIS_HOST', 'localhost'),  
38 - 'port': int(os.getenv('REDIS_PORT', 6479)),  
39 - 'snapshot_db': int(os.getenv('REDIS_SNAPSHOT_DB', 0)),  
40 - 'password': os.getenv('REDIS_PASSWORD'),  
41 - 'socket_timeout': int(os.getenv('REDIS_SOCKET_TIMEOUT', 1)),  
42 - 'socket_connect_timeout': int(os.getenv('REDIS_SOCKET_CONNECT_TIMEOUT', 1)),  
43 - 'retry_on_timeout': os.getenv('REDIS_RETRY_ON_TIMEOUT', 'False').lower() == 'true',  
44 - 'cache_expire_days': int(os.getenv('REDIS_CACHE_EXPIRE_DAYS', 360*2)), # 6 months  
45 - # Embedding 缓存 key 前缀,例如 "embedding"  
46 - 'embedding_cache_prefix': os.getenv('REDIS_EMBEDDING_CACHE_PREFIX', 'embedding'),  
47 -}  
48 -  
49 -# DeepL API Key  
50 -DEEPL_AUTH_KEY = os.getenv('DEEPL_AUTH_KEY')  
51 -  
52 -# DashScope API Key (for Qwen models)  
53 -DASHSCOPE_API_KEY = os.getenv('DASHSCOPE_API_KEY')  
54 -  
55 -# API Service Configuration  
56 -API_HOST = os.getenv('API_HOST', '0.0.0.0')  
57 -API_PORT = int(os.getenv('API_PORT', 6002))  
58 -# Indexer service  
59 -INDEXER_HOST = os.getenv('INDEXER_HOST', '0.0.0.0')  
60 -INDEXER_PORT = int(os.getenv('INDEXER_PORT', 6004))  
61 -# Optional dependent services  
62 -# EMBEDDING_HOST / EMBEDDING_PORT are only used by the optional combined embedding mode.  
63 -EMBEDDING_HOST = os.getenv('EMBEDDING_HOST', '127.0.0.1')  
64 -EMBEDDING_PORT = int(os.getenv('EMBEDDING_PORT', 6005))  
65 -EMBEDDING_TEXT_HOST = os.getenv('EMBEDDING_TEXT_HOST', '127.0.0.1')  
66 -EMBEDDING_TEXT_PORT = int(os.getenv('EMBEDDING_TEXT_PORT', 6005))  
67 -EMBEDDING_IMAGE_HOST = os.getenv('EMBEDDING_IMAGE_HOST', '127.0.0.1')  
68 -EMBEDDING_IMAGE_PORT = int(os.getenv('EMBEDDING_IMAGE_PORT', 6008))  
69 -TRANSLATION_HOST = os.getenv('TRANSLATION_HOST', '127.0.0.1')  
70 -TRANSLATION_PORT = int(os.getenv('TRANSLATION_PORT', 6006))  
71 -RERANKER_HOST = os.getenv('RERANKER_HOST', '127.0.0.1')  
72 -RERANKER_PORT = int(os.getenv('RERANKER_PORT', 6007))  
73 -RERANK_PROVIDER = os.getenv('RERANK_PROVIDER', 'http')  
74 -# API_BASE_URL: 如果未设置,根据API_HOST构建(0.0.0.0使用localhost)  
75 -API_BASE_URL = os.getenv('API_BASE_URL')  
76 -if not API_BASE_URL:  
77 - API_BASE_URL = f'http://localhost:{API_PORT}' if API_HOST == '0.0.0.0' else f'http://{API_HOST}:{API_PORT}'  
78 -INDEXER_BASE_URL = os.getenv('INDEXER_BASE_URL') or (  
79 - f'http://localhost:{INDEXER_PORT}' if INDEXER_HOST == '0.0.0.0' else f'http://{INDEXER_HOST}:{INDEXER_PORT}'  
80 -)  
81 -EMBEDDING_TEXT_SERVICE_URL = os.getenv('EMBEDDING_TEXT_SERVICE_URL') or (  
82 - f'http://{EMBEDDING_TEXT_HOST}:{EMBEDDING_TEXT_PORT}'  
83 -)  
84 -EMBEDDING_IMAGE_SERVICE_URL = os.getenv('EMBEDDING_IMAGE_SERVICE_URL') or (  
85 - f'http://{EMBEDDING_IMAGE_HOST}:{EMBEDDING_IMAGE_PORT}'  
86 -)  
87 -RERANKER_SERVICE_URL = os.getenv('RERANKER_SERVICE_URL') or f'http://{RERANKER_HOST}:{RERANKER_PORT}/rerank' 8 +from __future__ import annotations
  9 +
  10 +from typing import Any, Dict
  11 +
  12 +from config.loader import get_app_config
  13 +
  14 +
  15 +def _app():
  16 + return get_app_config()
  17 +
  18 +
  19 +def _runtime():
  20 + return _app().runtime
88 21
89 -# Model IDs / paths  
90 -TEXT_MODEL_DIR = os.getenv('TEXT_MODEL_DIR', os.getenv('TEXT_MODEL_ID', 'Qwen/Qwen3-Embedding-0.6B'))  
91 -IMAGE_MODEL_DIR = os.getenv('IMAGE_MODEL_DIR', '/data/tw/models/cn-clip')  
92 22
93 -# Cache Directory  
94 -CACHE_DIR = os.getenv('CACHE_DIR', '.cache') 23 +def _infra():
  24 + return _app().infrastructure
95 25
96 -# MySQL Database Configuration (Shoplazza)  
97 -DB_CONFIG = {  
98 - 'host': os.getenv('DB_HOST'),  
99 - 'port': int(os.getenv('DB_PORT', 3306)) if os.getenv('DB_PORT') else 3306,  
100 - 'database': os.getenv('DB_DATABASE'),  
101 - 'username': os.getenv('DB_USERNAME'),  
102 - 'password': os.getenv('DB_PASSWORD'),  
103 -}  
104 26
  27 +def _elasticsearch_dict() -> Dict[str, Any]:
  28 + cfg = _infra().elasticsearch
  29 + return {
  30 + "host": cfg.host,
  31 + "username": cfg.username,
  32 + "password": cfg.password,
  33 + }
105 34
106 -def print_config():  
107 - """Print current configuration (with sensitive data masked)."""  
108 - print("=" * 60)  
109 - print("saas-search Configuration")  
110 - print("=" * 60)  
111 35
112 - print("\nElasticsearch:")  
113 - print(f" Host: {ES_CONFIG['host']}")  
114 - print(f" Username: {ES_CONFIG['username']}")  
115 - print(f" Password: {'*' * 10 if ES_CONFIG['password'] else 'None'}") 36 +def _redis_dict() -> Dict[str, Any]:
  37 + cfg = _infra().redis
  38 + return {
  39 + "host": cfg.host,
  40 + "port": cfg.port,
  41 + "snapshot_db": cfg.snapshot_db,
  42 + "password": cfg.password,
  43 + "socket_timeout": cfg.socket_timeout,
  44 + "socket_connect_timeout": cfg.socket_connect_timeout,
  45 + "retry_on_timeout": cfg.retry_on_timeout,
  46 + "cache_expire_days": cfg.cache_expire_days,
  47 + "embedding_cache_prefix": cfg.embedding_cache_prefix,
  48 + "anchor_cache_prefix": cfg.anchor_cache_prefix,
  49 + "anchor_cache_expire_days": cfg.anchor_cache_expire_days,
  50 + }
116 51
117 - print("\nRedis:")  
118 - print(f" Host: {REDIS_CONFIG['host']}")  
119 - print(f" Port: {REDIS_CONFIG['port']}")  
120 - print(f" Password: {'*' * 10 if REDIS_CONFIG['password'] else 'None'}")  
121 52
122 - print("\nDeepL:")  
123 - print(f" API Key: {'*' * 10 if DEEPL_AUTH_KEY else 'None (translation disabled)'}") 53 +def _db_dict() -> Dict[str, Any]:
  54 + cfg = _infra().database
  55 + return {
  56 + "host": cfg.host,
  57 + "port": cfg.port,
  58 + "database": cfg.database,
  59 + "username": cfg.username,
  60 + "password": cfg.password,
  61 + }
  62 +
  63 +
  64 +ES_CONFIG = _elasticsearch_dict()
  65 +REDIS_CONFIG = _redis_dict()
  66 +DB_CONFIG = _db_dict()
  67 +
  68 +RUNTIME_ENV = _runtime().environment
  69 +ES_INDEX_NAMESPACE = _runtime().index_namespace
  70 +
  71 +DEEPL_AUTH_KEY = _infra().secrets.deepl_auth_key
  72 +DASHSCOPE_API_KEY = _infra().secrets.dashscope_api_key
  73 +
  74 +API_HOST = _runtime().api_host
  75 +API_PORT = _runtime().api_port
  76 +INDEXER_HOST = _runtime().indexer_host
  77 +INDEXER_PORT = _runtime().indexer_port
  78 +EMBEDDING_HOST = _runtime().embedding_host
  79 +EMBEDDING_PORT = _runtime().embedding_port
  80 +EMBEDDING_TEXT_HOST = _runtime().embedding_host
  81 +EMBEDDING_TEXT_PORT = _runtime().embedding_text_port
  82 +EMBEDDING_IMAGE_HOST = _runtime().embedding_host
  83 +EMBEDDING_IMAGE_PORT = _runtime().embedding_image_port
  84 +TRANSLATION_HOST = _runtime().translator_host
  85 +TRANSLATION_PORT = _runtime().translator_port
  86 +RERANKER_HOST = _runtime().reranker_host
  87 +RERANKER_PORT = _runtime().reranker_port
  88 +
  89 +API_BASE_URL = f"http://localhost:{API_PORT}" if API_HOST == "0.0.0.0" else f"http://{API_HOST}:{API_PORT}"
  90 +INDEXER_BASE_URL = (
  91 + f"http://localhost:{INDEXER_PORT}" if INDEXER_HOST == "0.0.0.0" else f"http://{INDEXER_HOST}:{INDEXER_PORT}"
  92 +)
  93 +EMBEDDING_TEXT_SERVICE_URL = _app().services.embedding.get_provider_config().get("text_base_url")
  94 +EMBEDDING_IMAGE_SERVICE_URL = _app().services.embedding.get_provider_config().get("image_base_url")
  95 +RERANKER_SERVICE_URL = (
  96 + _app().services.rerank.get_provider_config().get("service_url")
  97 + or _app().services.rerank.get_provider_config().get("base_url")
  98 +)
  99 +
124 100
125 - print("\nAPI Service:")  
126 - print(f" Host: {API_HOST}")  
127 - print(f" Port: {API_PORT}") 101 +def get_es_config() -> Dict[str, Any]:
  102 + return dict(ES_CONFIG)
128 103
129 - print("\nModels:")  
130 - print(f" Text Model: {TEXT_MODEL_DIR}")  
131 - print(f" Image Model: {IMAGE_MODEL_DIR}")  
132 104
133 - print("\nCache:")  
134 - print(f" Cache Directory: {CACHE_DIR}") 105 +def get_db_config() -> Dict[str, Any]:
  106 + return dict(DB_CONFIG)
135 107
136 - print("\nMySQL Database:")  
137 - print(f" Host: {DB_CONFIG['host']}")  
138 - print(f" Port: {DB_CONFIG['port']}")  
139 - print(f" Database: {DB_CONFIG['database']}")  
140 - print(f" Username: {DB_CONFIG['username']}")  
141 - print(f" Password: {'*' * 10 if DB_CONFIG['password'] else 'None'}")  
142 108
143 - print("=" * 60) 109 +def get_redis_config() -> Dict[str, Any]:
  110 + return dict(REDIS_CONFIG)
144 111
145 112
146 -if __name__ == "__main__":  
147 - print_config() 113 +def print_config() -> None:
  114 + config = _app().sanitized_dict()
  115 + print(config)
config/loader.py 0 → 100644
@@ -0,0 +1,589 @@ @@ -0,0 +1,589 @@
  1 +"""
  2 +Unified application configuration loader.
  3 +
  4 +This module is the single source of truth for loading, merging, normalizing,
  5 +and validating application configuration.
  6 +"""
  7 +
  8 +from __future__ import annotations
  9 +
  10 +import hashlib
  11 +import json
  12 +import os
  13 +from copy import deepcopy
  14 +from dataclasses import asdict
  15 +from functools import lru_cache
  16 +from pathlib import Path
  17 +from typing import Any, Dict, Iterable, List, Optional, Tuple
  18 +
  19 +import yaml
  20 +
  21 +try:
  22 + from dotenv import load_dotenv as _load_dotenv # type: ignore
  23 +except Exception: # pragma: no cover
  24 + _load_dotenv = None
  25 +
  26 +from config.schema import (
  27 + AppConfig,
  28 + AssetsConfig,
  29 + ConfigMetadata,
  30 + DatabaseSettings,
  31 + ElasticsearchSettings,
  32 + EmbeddingServiceConfig,
  33 + FunctionScoreConfig,
  34 + IndexConfig,
  35 + InfrastructureConfig,
  36 + QueryConfig,
  37 + ProductEnrichConfig,
  38 + RedisSettings,
  39 + RerankConfig,
  40 + RerankServiceConfig,
  41 + RuntimeConfig,
  42 + SearchConfig,
  43 + SecretsConfig,
  44 + ServicesConfig,
  45 + SPUConfig,
  46 + TenantCatalogConfig,
  47 + TranslationServiceConfig,
  48 +)
  49 +from translation.settings import build_translation_config
  50 +
  51 +
  52 +class ConfigurationError(Exception):
  53 + """Raised when configuration validation fails."""
  54 +
  55 +
  56 +def _deep_merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
  57 + result = deepcopy(base)
  58 + for key, value in (override or {}).items():
  59 + if (
  60 + key in result
  61 + and isinstance(result[key], dict)
  62 + and isinstance(value, dict)
  63 + ):
  64 + result[key] = _deep_merge(result[key], value)
  65 + else:
  66 + result[key] = deepcopy(value)
  67 + return result
  68 +
  69 +
  70 +def _load_yaml(path: Path) -> Dict[str, Any]:
  71 + with open(path, "r", encoding="utf-8") as handle:
  72 + data = yaml.safe_load(handle) or {}
  73 + if not isinstance(data, dict):
  74 + raise ConfigurationError(f"Configuration file root must be a mapping: {path}")
  75 + return data
  76 +
  77 +
  78 +def _read_rewrite_dictionary(path: Path) -> Dict[str, str]:
  79 + rewrite_dict: Dict[str, str] = {}
  80 + if not path.exists():
  81 + return rewrite_dict
  82 +
  83 + with open(path, "r", encoding="utf-8") as handle:
  84 + for raw_line in handle:
  85 + line = raw_line.strip()
  86 + if not line or line.startswith("#"):
  87 + continue
  88 + parts = line.split("\t")
  89 + if len(parts) < 2:
  90 + continue
  91 + original = parts[0].strip()
  92 + replacement = parts[1].strip()
  93 + if original and replacement:
  94 + rewrite_dict[original] = replacement
  95 + return rewrite_dict
  96 +
  97 +
  98 +class AppConfigLoader:
  99 + """Load the unified application configuration."""
  100 +
  101 + def __init__(
  102 + self,
  103 + *,
  104 + config_dir: Optional[Path] = None,
  105 + config_file: Optional[Path] = None,
  106 + env_file: Optional[Path] = None,
  107 + ) -> None:
  108 + self.config_dir = Path(config_dir or Path(__file__).parent)
  109 + self.config_file = Path(config_file) if config_file is not None else None
  110 + self.project_root = self.config_dir.parent
  111 + self.env_file = Path(env_file) if env_file is not None else self.project_root / ".env"
  112 +
  113 + def load(self, validate: bool = True) -> AppConfig:
  114 + self._load_env()
  115 + raw_config, loaded_files = self._load_raw_config()
  116 + app_config = self._build_app_config(raw_config, loaded_files)
  117 + if validate:
  118 + self._validate(app_config)
  119 + return app_config
  120 +
  121 + def _load_env(self) -> None:
  122 + if _load_dotenv is not None:
  123 + _load_dotenv(self.env_file, override=False)
  124 + return
  125 + _load_env_file_fallback(self.env_file)
  126 +
  127 + def _load_raw_config(self) -> Tuple[Dict[str, Any], List[str]]:
  128 + env_name = (os.getenv("APP_ENV") or os.getenv("RUNTIME_ENV") or "prod").strip().lower() or "prod"
  129 + loaded_files: List[str] = []
  130 + raw: Dict[str, Any] = {}
  131 +
  132 + if self.config_file is not None:
  133 + config_path = self.config_file
  134 + if not config_path.exists():
  135 + raise ConfigurationError(f"Configuration file not found: {config_path}")
  136 + raw = _deep_merge(raw, _load_yaml(config_path))
  137 + loaded_files.append(str(config_path))
  138 + else:
  139 + base_path = self.config_dir / "base.yaml"
  140 + legacy_path = self.config_dir / "config.yaml"
  141 + primary_path = base_path if base_path.exists() else legacy_path
  142 + if not primary_path.exists():
  143 + raise ConfigurationError(f"Configuration file not found: {primary_path}")
  144 + raw = _deep_merge(raw, _load_yaml(primary_path))
  145 + loaded_files.append(str(primary_path))
  146 +
  147 + env_path = self.config_dir / "environments" / f"{env_name}.yaml"
  148 + if env_path.exists():
  149 + raw = _deep_merge(raw, _load_yaml(env_path))
  150 + loaded_files.append(str(env_path))
  151 +
  152 + tenant_dir = self.config_dir / "tenants"
  153 + if tenant_dir.is_dir():
  154 + tenant_files = sorted(tenant_dir.glob("*.yaml"))
  155 + if tenant_files:
  156 + tenant_config = {"default": {}, "tenants": {}}
  157 + default_path = tenant_dir / "_default.yaml"
  158 + if default_path.exists():
  159 + tenant_config["default"] = _load_yaml(default_path)
  160 + loaded_files.append(str(default_path))
  161 + for tenant_path in tenant_files:
  162 + if tenant_path.name == "_default.yaml":
  163 + continue
  164 + tenant_config["tenants"][tenant_path.stem] = _load_yaml(tenant_path)
  165 + loaded_files.append(str(tenant_path))
  166 + raw["tenant_config"] = tenant_config
  167 +
  168 + return raw, loaded_files
  169 +
  170 + def _build_app_config(self, raw: Dict[str, Any], loaded_files: List[str]) -> AppConfig:
  171 + assets_cfg = raw.get("assets") if isinstance(raw.get("assets"), dict) else {}
  172 + rewrite_path = (
  173 + assets_cfg.get("query_rewrite_dictionary_path")
  174 + or assets_cfg.get("rewrite_dictionary_path")
  175 + or self.config_dir / "dictionaries" / "query_rewrite.dict"
  176 + )
  177 + rewrite_path = Path(rewrite_path)
  178 + if not rewrite_path.is_absolute():
  179 + rewrite_path = (self.project_root / rewrite_path).resolve()
  180 + if not rewrite_path.exists():
  181 + legacy_rewrite_path = (self.config_dir / "query_rewrite.dict").resolve()
  182 + if legacy_rewrite_path.exists():
  183 + rewrite_path = legacy_rewrite_path
  184 +
  185 + rewrite_dictionary = _read_rewrite_dictionary(rewrite_path)
  186 + search_config = self._build_search_config(raw, rewrite_dictionary)
  187 + services_config = self._build_services_config(raw.get("services") or {})
  188 + tenants_config = self._build_tenants_config(raw.get("tenant_config") or {})
  189 + runtime_config = self._build_runtime_config()
  190 + infrastructure_config = self._build_infrastructure_config(runtime_config.environment)
  191 +
  192 + product_enrich_raw = raw.get("product_enrich") if isinstance(raw.get("product_enrich"), dict) else {}
  193 + product_enrich_config = ProductEnrichConfig(
  194 + max_workers=int(product_enrich_raw.get("max_workers", 40)),
  195 + )
  196 +
  197 + metadata = ConfigMetadata(
  198 + loaded_files=tuple(loaded_files),
  199 + config_hash="",
  200 + deprecated_keys=tuple(self._detect_deprecated_keys(raw)),
  201 + )
  202 +
  203 + app_config = AppConfig(
  204 + runtime=runtime_config,
  205 + infrastructure=infrastructure_config,
  206 + product_enrich=product_enrich_config,
  207 + search=search_config,
  208 + services=services_config,
  209 + tenants=tenants_config,
  210 + assets=AssetsConfig(query_rewrite_dictionary_path=rewrite_path),
  211 + metadata=metadata,
  212 + )
  213 +
  214 + config_hash = self._compute_hash(app_config)
  215 + return AppConfig(
  216 + runtime=app_config.runtime,
  217 + infrastructure=app_config.infrastructure,
  218 + product_enrich=app_config.product_enrich,
  219 + search=app_config.search,
  220 + services=app_config.services,
  221 + tenants=app_config.tenants,
  222 + assets=app_config.assets,
  223 + metadata=ConfigMetadata(
  224 + loaded_files=app_config.metadata.loaded_files,
  225 + config_hash=config_hash,
  226 + deprecated_keys=app_config.metadata.deprecated_keys,
  227 + ),
  228 + )
  229 +
  230 + def _build_search_config(self, raw: Dict[str, Any], rewrite_dictionary: Dict[str, str]) -> SearchConfig:
  231 + field_boosts = raw.get("field_boosts") or {}
  232 + if not isinstance(field_boosts, dict):
  233 + raise ConfigurationError("field_boosts must be a mapping")
  234 +
  235 + indexes: List[IndexConfig] = []
  236 + for item in raw.get("indexes") or []:
  237 + if not isinstance(item, dict):
  238 + raise ConfigurationError("indexes items must be mappings")
  239 + indexes.append(
  240 + IndexConfig(
  241 + name=str(item["name"]),
  242 + label=str(item.get("label") or item["name"]),
  243 + fields=list(item.get("fields") or []),
  244 + boost=float(item.get("boost", 1.0)),
  245 + example=item.get("example"),
  246 + )
  247 + )
  248 +
  249 + query_cfg = raw.get("query_config") if isinstance(raw.get("query_config"), dict) else {}
  250 + search_fields = query_cfg.get("search_fields") if isinstance(query_cfg.get("search_fields"), dict) else {}
  251 + text_strategy = (
  252 + query_cfg.get("text_query_strategy")
  253 + if isinstance(query_cfg.get("text_query_strategy"), dict)
  254 + else {}
  255 + )
  256 + query_config = QueryConfig(
  257 + supported_languages=list(query_cfg.get("supported_languages") or ["zh", "en"]),
  258 + default_language=str(query_cfg.get("default_language") or "en"),
  259 + enable_text_embedding=bool(query_cfg.get("enable_text_embedding", True)),
  260 + enable_query_rewrite=bool(query_cfg.get("enable_query_rewrite", True)),
  261 + rewrite_dictionary=rewrite_dictionary,
  262 + text_embedding_field=query_cfg.get("text_embedding_field"),
  263 + image_embedding_field=query_cfg.get("image_embedding_field"),
  264 + source_fields=query_cfg.get("source_fields"),
  265 + knn_boost=float(query_cfg.get("knn_boost", 0.25)),
  266 + multilingual_fields=list(
  267 + search_fields.get(
  268 + "multilingual_fields",
  269 + ["title", "brief", "description", "vendor", "category_path", "category_name_text"],
  270 + )
  271 + ),
  272 + shared_fields=list(
  273 + search_fields.get(
  274 + "shared_fields",
  275 + ["tags", "option1_values", "option2_values", "option3_values"],
  276 + )
  277 + ),
  278 + core_multilingual_fields=list(
  279 + search_fields.get(
  280 + "core_multilingual_fields",
  281 + ["title", "brief", "vendor", "category_name_text"],
  282 + )
  283 + ),
  284 + base_minimum_should_match=str(text_strategy.get("base_minimum_should_match", "75%")),
  285 + translation_minimum_should_match=str(text_strategy.get("translation_minimum_should_match", "75%")),
  286 + translation_boost=float(text_strategy.get("translation_boost", 0.4)),
  287 + translation_boost_when_source_missing=float(
  288 + text_strategy.get("translation_boost_when_source_missing", 1.0)
  289 + ),
  290 + source_boost_when_missing=float(text_strategy.get("source_boost_when_missing", 0.6)),
  291 + original_query_fallback_boost_when_translation_missing=float(
  292 + text_strategy.get("original_query_fallback_boost_when_translation_missing", 0.2)
  293 + ),
  294 + tie_breaker_base_query=float(text_strategy.get("tie_breaker_base_query", 0.9)),
  295 + zh_to_en_model=str(query_cfg.get("zh_to_en_model") or "opus-mt-zh-en"),
  296 + en_to_zh_model=str(query_cfg.get("en_to_zh_model") or "opus-mt-en-zh"),
  297 + default_translation_model=str(
  298 + query_cfg.get("default_translation_model") or "nllb-200-distilled-600m"
  299 + ),
  300 + )
  301 +
  302 + function_score_cfg = raw.get("function_score") if isinstance(raw.get("function_score"), dict) else {}
  303 + rerank_cfg = raw.get("rerank") if isinstance(raw.get("rerank"), dict) else {}
  304 + spu_cfg = raw.get("spu_config") if isinstance(raw.get("spu_config"), dict) else {}
  305 +
  306 + return SearchConfig(
  307 + field_boosts={str(key): float(value) for key, value in field_boosts.items()},
  308 + indexes=indexes,
  309 + query_config=query_config,
  310 + function_score=FunctionScoreConfig(
  311 + score_mode=str(function_score_cfg.get("score_mode") or "sum"),
  312 + boost_mode=str(function_score_cfg.get("boost_mode") or "multiply"),
  313 + functions=list(function_score_cfg.get("functions") or []),
  314 + ),
  315 + rerank=RerankConfig(
  316 + enabled=bool(rerank_cfg.get("enabled", True)),
  317 + rerank_window=int(rerank_cfg.get("rerank_window", 384)),
  318 + timeout_sec=float(rerank_cfg.get("timeout_sec", 15.0)),
  319 + weight_es=float(rerank_cfg.get("weight_es", 0.4)),
  320 + weight_ai=float(rerank_cfg.get("weight_ai", 0.6)),
  321 + rerank_query_template=str(rerank_cfg.get("rerank_query_template") or "{query}"),
  322 + rerank_doc_template=str(rerank_cfg.get("rerank_doc_template") or "{title}"),
  323 + ),
  324 + spu_config=SPUConfig(
  325 + enabled=bool(spu_cfg.get("enabled", False)),
  326 + spu_field=spu_cfg.get("spu_field"),
  327 + inner_hits_size=int(spu_cfg.get("inner_hits_size", 3)),
  328 + searchable_option_dimensions=list(
  329 + spu_cfg.get("searchable_option_dimensions") or ["option1", "option2", "option3"]
  330 + ),
  331 + ),
  332 + es_index_name=str(raw.get("es_index_name") or "search_products"),
  333 + es_settings=dict(raw.get("es_settings") or {}),
  334 + )
  335 +
  336 + def _build_services_config(self, raw: Dict[str, Any]) -> ServicesConfig:
  337 + if not isinstance(raw, dict):
  338 + raise ConfigurationError("services must be a mapping")
  339 +
  340 + translation_raw = raw.get("translation") if isinstance(raw.get("translation"), dict) else {}
  341 + normalized_translation = build_translation_config(translation_raw)
  342 + translation_config = TranslationServiceConfig(
  343 + endpoint=str(normalized_translation["service_url"]).rstrip("/"),
  344 + timeout_sec=float(normalized_translation["timeout_sec"]),
  345 + default_model=str(normalized_translation["default_model"]),
  346 + default_scene=str(normalized_translation["default_scene"]),
  347 + cache=dict(normalized_translation["cache"]),
  348 + capabilities={str(key): dict(value) for key, value in normalized_translation["capabilities"].items()},
  349 + )
  350 +
  351 + embedding_raw = raw.get("embedding") if isinstance(raw.get("embedding"), dict) else {}
  352 + embedding_provider = str(embedding_raw.get("provider") or "http").strip().lower()
  353 + embedding_providers = dict(embedding_raw.get("providers") or {})
  354 + if embedding_provider not in embedding_providers:
  355 + raise ConfigurationError(f"services.embedding.providers.{embedding_provider} must be configured")
  356 + embedding_backend = str(embedding_raw.get("backend") or "").strip().lower()
  357 + embedding_backends = {
  358 + str(key).strip().lower(): dict(value)
  359 + for key, value in dict(embedding_raw.get("backends") or {}).items()
  360 + }
  361 + if embedding_backend not in embedding_backends:
  362 + raise ConfigurationError(f"services.embedding.backends.{embedding_backend} must be configured")
  363 + image_backend = str(embedding_raw.get("image_backend") or "clip_as_service").strip().lower()
  364 + image_backends = {
  365 + str(key).strip().lower(): dict(value)
  366 + for key, value in dict(embedding_raw.get("image_backends") or {}).items()
  367 + }
  368 + if not image_backends:
  369 + image_backends = {
  370 + "clip_as_service": {
  371 + "server": "grpc://127.0.0.1:51000",
  372 + "model_name": "CN-CLIP/ViT-L-14",
  373 + "batch_size": 8,
  374 + "normalize_embeddings": True,
  375 + },
  376 + "local_cnclip": {
  377 + "model_name": "ViT-L-14",
  378 + "device": None,
  379 + "batch_size": 8,
  380 + "normalize_embeddings": True,
  381 + },
  382 + }
  383 + if image_backend not in image_backends:
  384 + raise ConfigurationError(f"services.embedding.image_backends.{image_backend} must be configured")
  385 +
  386 + embedding_config = EmbeddingServiceConfig(
  387 + provider=embedding_provider,
  388 + providers=embedding_providers,
  389 + backend=embedding_backend,
  390 + backends=embedding_backends,
  391 + image_backend=image_backend,
  392 + image_backends=image_backends,
  393 + )
  394 +
  395 + rerank_raw = raw.get("rerank") if isinstance(raw.get("rerank"), dict) else {}
  396 + rerank_provider = str(rerank_raw.get("provider") or "http").strip().lower()
  397 + rerank_providers = dict(rerank_raw.get("providers") or {})
  398 + if rerank_provider not in rerank_providers:
  399 + raise ConfigurationError(f"services.rerank.providers.{rerank_provider} must be configured")
  400 + rerank_backend = str(rerank_raw.get("backend") or "").strip().lower()
  401 + rerank_backends = {
  402 + str(key).strip().lower(): dict(value)
  403 + for key, value in dict(rerank_raw.get("backends") or {}).items()
  404 + }
  405 + if rerank_backend not in rerank_backends:
  406 + raise ConfigurationError(f"services.rerank.backends.{rerank_backend} must be configured")
  407 + rerank_request = dict(rerank_raw.get("request") or {})
  408 + rerank_request.setdefault("max_docs", 1000)
  409 + rerank_request.setdefault("normalize", True)
  410 +
  411 + rerank_config = RerankServiceConfig(
  412 + provider=rerank_provider,
  413 + providers=rerank_providers,
  414 + backend=rerank_backend,
  415 + backends=rerank_backends,
  416 + request=rerank_request,
  417 + )
  418 +
  419 + return ServicesConfig(
  420 + translation=translation_config,
  421 + embedding=embedding_config,
  422 + rerank=rerank_config,
  423 + )
  424 +
  425 + def _build_tenants_config(self, raw: Dict[str, Any]) -> TenantCatalogConfig:
  426 + if not isinstance(raw, dict):
  427 + raise ConfigurationError("tenant_config must be a mapping")
  428 + default_cfg = raw.get("default") if isinstance(raw.get("default"), dict) else {}
  429 + tenants_cfg = raw.get("tenants") if isinstance(raw.get("tenants"), dict) else {}
  430 + return TenantCatalogConfig(
  431 + default=dict(default_cfg),
  432 + tenants={str(key): dict(value) for key, value in tenants_cfg.items()},
  433 + )
  434 +
  435 + def _build_runtime_config(self) -> RuntimeConfig:
  436 + environment = (os.getenv("APP_ENV") or os.getenv("RUNTIME_ENV") or "prod").strip().lower() or "prod"
  437 + namespace = os.getenv("ES_INDEX_NAMESPACE")
  438 + if namespace is None:
  439 + namespace = "" if environment == "prod" else f"{environment}_"
  440 +
  441 + return RuntimeConfig(
  442 + environment=environment,
  443 + index_namespace=namespace,
  444 + api_host=os.getenv("API_HOST", "0.0.0.0"),
  445 + api_port=int(os.getenv("API_PORT", 6002)),
  446 + indexer_host=os.getenv("INDEXER_HOST", "0.0.0.0"),
  447 + indexer_port=int(os.getenv("INDEXER_PORT", 6004)),
  448 + embedding_host=os.getenv("EMBEDDING_HOST", "127.0.0.1"),
  449 + embedding_port=int(os.getenv("EMBEDDING_PORT", 6005)),
  450 + embedding_text_port=int(os.getenv("EMBEDDING_TEXT_PORT", 6005)),
  451 + embedding_image_port=int(os.getenv("EMBEDDING_IMAGE_PORT", 6008)),
  452 + translator_host=os.getenv("TRANSLATION_HOST", "127.0.0.1"),
  453 + translator_port=int(os.getenv("TRANSLATION_PORT", 6006)),
  454 + reranker_host=os.getenv("RERANKER_HOST", "127.0.0.1"),
  455 + reranker_port=int(os.getenv("RERANKER_PORT", 6007)),
  456 + )
  457 +
  458 + def _build_infrastructure_config(self, environment: str) -> InfrastructureConfig:
  459 + del environment
  460 + return InfrastructureConfig(
  461 + elasticsearch=ElasticsearchSettings(
  462 + host=os.getenv("ES_HOST", "http://localhost:9200"),
  463 + username=os.getenv("ES_USERNAME"),
  464 + password=os.getenv("ES_PASSWORD"),
  465 + ),
  466 + redis=RedisSettings(
  467 + host=os.getenv("REDIS_HOST", "localhost"),
  468 + port=int(os.getenv("REDIS_PORT", 6479)),
  469 + snapshot_db=int(os.getenv("REDIS_SNAPSHOT_DB", 0)),
  470 + password=os.getenv("REDIS_PASSWORD"),
  471 + socket_timeout=int(os.getenv("REDIS_SOCKET_TIMEOUT", 1)),
  472 + socket_connect_timeout=int(os.getenv("REDIS_SOCKET_CONNECT_TIMEOUT", 1)),
  473 + retry_on_timeout=os.getenv("REDIS_RETRY_ON_TIMEOUT", "false").strip().lower() == "true",
  474 + cache_expire_days=int(os.getenv("REDIS_CACHE_EXPIRE_DAYS", 360 * 2)),
  475 + embedding_cache_prefix=os.getenv("REDIS_EMBEDDING_CACHE_PREFIX", "embedding"),
  476 + anchor_cache_prefix=os.getenv("REDIS_ANCHOR_CACHE_PREFIX", "product_anchors"),
  477 + anchor_cache_expire_days=int(os.getenv("REDIS_ANCHOR_CACHE_EXPIRE_DAYS", 30)),
  478 + ),
  479 + database=DatabaseSettings(
  480 + host=os.getenv("DB_HOST"),
  481 + port=int(os.getenv("DB_PORT", 3306)) if os.getenv("DB_PORT") else 3306,
  482 + database=os.getenv("DB_DATABASE"),
  483 + username=os.getenv("DB_USERNAME"),
  484 + password=os.getenv("DB_PASSWORD"),
  485 + ),
  486 + secrets=SecretsConfig(
  487 + dashscope_api_key=os.getenv("DASHSCOPE_API_KEY"),
  488 + deepl_auth_key=os.getenv("DEEPL_AUTH_KEY"),
  489 + ),
  490 + )
  491 +
  492 + def _validate(self, app_config: AppConfig) -> None:
  493 + errors: List[str] = []
  494 +
  495 + if not app_config.search.es_index_name:
  496 + errors.append("search.es_index_name is required")
  497 +
  498 + if not app_config.search.field_boosts:
  499 + errors.append("search.field_boosts cannot be empty")
  500 + else:
  501 + for field_name, boost in app_config.search.field_boosts.items():
  502 + if boost < 0:
  503 + errors.append(f"field_boosts.{field_name} must be non-negative")
  504 +
  505 + query_config = app_config.search.query_config
  506 + if not query_config.supported_languages:
  507 + errors.append("query_config.supported_languages must not be empty")
  508 + if query_config.default_language not in query_config.supported_languages:
  509 + errors.append("query_config.default_language must be included in supported_languages")
  510 + for name, values in (
  511 + ("multilingual_fields", query_config.multilingual_fields),
  512 + ("shared_fields", query_config.shared_fields),
  513 + ("core_multilingual_fields", query_config.core_multilingual_fields),
  514 + ):
  515 + if not values:
  516 + errors.append(f"query_config.{name} must not be empty")
  517 +
  518 + if not set(query_config.core_multilingual_fields).issubset(set(query_config.multilingual_fields)):
  519 + errors.append("query_config.core_multilingual_fields must be a subset of multilingual_fields")
  520 +
  521 + if app_config.search.spu_config.enabled and not app_config.search.spu_config.spu_field:
  522 + errors.append("spu_config.spu_field is required when spu_config.enabled is true")
  523 +
  524 + if not app_config.tenants.default or not app_config.tenants.default.get("index_languages"):
  525 + errors.append("tenant_config.default.index_languages must be configured")
  526 +
  527 + if app_config.metadata.deprecated_keys:
  528 + errors.append(
  529 + "Deprecated tenant config keys are not supported: "
  530 + + ", ".join(app_config.metadata.deprecated_keys)
  531 + )
  532 +
  533 + embedding_provider_cfg = app_config.services.embedding.get_provider_config()
  534 + if not embedding_provider_cfg.get("text_base_url"):
  535 + errors.append("services.embedding.providers.<provider>.text_base_url is required")
  536 + if not embedding_provider_cfg.get("image_base_url"):
  537 + errors.append("services.embedding.providers.<provider>.image_base_url is required")
  538 +
  539 + rerank_provider_cfg = app_config.services.rerank.get_provider_config()
  540 + if not rerank_provider_cfg.get("service_url") and not rerank_provider_cfg.get("base_url"):
  541 + errors.append("services.rerank.providers.<provider>.service_url or base_url is required")
  542 +
  543 + if errors:
  544 + raise ConfigurationError("Configuration validation failed:\n" + "\n".join(f" - {err}" for err in errors))
  545 +
  546 + def _compute_hash(self, app_config: AppConfig) -> str:
  547 + payload = asdict(app_config)
  548 + payload["metadata"]["config_hash"] = ""
  549 + payload["infrastructure"]["elasticsearch"]["password"] = "***" if payload["infrastructure"]["elasticsearch"].get("password") else None
  550 + payload["infrastructure"]["database"]["password"] = "***" if payload["infrastructure"]["database"].get("password") else None
  551 + payload["infrastructure"]["redis"]["password"] = "***" if payload["infrastructure"]["redis"].get("password") else None
  552 + payload["infrastructure"]["secrets"]["dashscope_api_key"] = "***" if payload["infrastructure"]["secrets"].get("dashscope_api_key") else None
  553 + payload["infrastructure"]["secrets"]["deepl_auth_key"] = "***" if payload["infrastructure"]["secrets"].get("deepl_auth_key") else None
  554 + blob = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str)
  555 + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:16]
  556 +
  557 + def _detect_deprecated_keys(self, raw: Dict[str, Any]) -> Iterable[str]:
  558 + # Translation-era legacy flags have been removed; keep the hook for future
  559 + # deprecations, but currently no deprecated keys are detected.
  560 + return ()
  561 +
  562 +
  563 +@lru_cache(maxsize=1)
  564 +def get_app_config() -> AppConfig:
  565 + """Return the process-global application configuration."""
  566 +
  567 + return AppConfigLoader().load()
  568 +
  569 +
  570 +def reload_app_config() -> AppConfig:
  571 + """Clear the cached configuration and reload it."""
  572 +
  573 + get_app_config.cache_clear()
  574 + return get_app_config()
  575 +
  576 +
  577 +def _load_env_file_fallback(path: Path) -> None:
  578 + if not path.exists():
  579 + return
  580 + with open(path, "r", encoding="utf-8") as handle:
  581 + for raw_line in handle:
  582 + line = raw_line.strip()
  583 + if not line or line.startswith("#") or "=" not in line:
  584 + continue
  585 + key, value = line.split("=", 1)
  586 + key = key.strip()
  587 + value = value.strip().strip('"').strip("'")
  588 + if key and key not in os.environ:
  589 + os.environ[key] = value
config/schema.py 0 → 100644
@@ -0,0 +1,315 @@ @@ -0,0 +1,315 @@
  1 +"""
  2 +Typed configuration schema for the unified application configuration.
  3 +
  4 +This module defines the normalized in-memory structure used by all services.
  5 +"""
  6 +
  7 +from __future__ import annotations
  8 +
  9 +from dataclasses import asdict, dataclass, field
  10 +from pathlib import Path
  11 +from typing import Any, Dict, List, Optional, Tuple
  12 +
  13 +
  14 +@dataclass(frozen=True)
  15 +class IndexConfig:
  16 + """Deprecated compatibility shape for legacy diagnostics/tests."""
  17 +
  18 + name: str
  19 + label: str
  20 + fields: List[str]
  21 + boost: float = 1.0
  22 + example: Optional[str] = None
  23 +
  24 +
  25 +@dataclass(frozen=True)
  26 +class QueryConfig:
  27 + """Configuration for query processing."""
  28 +
  29 + supported_languages: List[str] = field(default_factory=lambda: ["zh", "en"])
  30 + default_language: str = "en"
  31 + enable_text_embedding: bool = True
  32 + enable_query_rewrite: bool = True
  33 + rewrite_dictionary: Dict[str, str] = field(default_factory=dict)
  34 + text_embedding_field: Optional[str] = "title_embedding"
  35 + image_embedding_field: Optional[str] = None
  36 + source_fields: Optional[List[str]] = None
  37 + knn_boost: float = 0.25
  38 + multilingual_fields: List[str] = field(
  39 + default_factory=lambda: [
  40 + "title",
  41 + "brief",
  42 + "description",
  43 + "vendor",
  44 + "category_path",
  45 + "category_name_text",
  46 + ]
  47 + )
  48 + shared_fields: List[str] = field(
  49 + default_factory=lambda: ["tags", "option1_values", "option2_values", "option3_values"]
  50 + )
  51 + core_multilingual_fields: List[str] = field(
  52 + default_factory=lambda: ["title", "brief", "vendor", "category_name_text"]
  53 + )
  54 + base_minimum_should_match: str = "75%"
  55 + translation_minimum_should_match: str = "75%"
  56 + translation_boost: float = 0.4
  57 + translation_boost_when_source_missing: float = 1.0
  58 + source_boost_when_missing: float = 0.6
  59 + original_query_fallback_boost_when_translation_missing: float = 0.2
  60 + tie_breaker_base_query: float = 0.9
  61 + zh_to_en_model: str = "opus-mt-zh-en"
  62 + en_to_zh_model: str = "opus-mt-en-zh"
  63 + default_translation_model: str = "nllb-200-distilled-600m"
  64 +
  65 +
  66 +@dataclass(frozen=True)
  67 +class SPUConfig:
  68 + """SPU aggregation/search configuration."""
  69 +
  70 + enabled: bool = False
  71 + spu_field: Optional[str] = None
  72 + inner_hits_size: int = 3
  73 + searchable_option_dimensions: List[str] = field(
  74 + default_factory=lambda: ["option1", "option2", "option3"]
  75 + )
  76 +
  77 +
  78 +@dataclass(frozen=True)
  79 +class FunctionScoreConfig:
  80 + """Function score configuration."""
  81 +
  82 + score_mode: str = "sum"
  83 + boost_mode: str = "multiply"
  84 + functions: List[Dict[str, Any]] = field(default_factory=list)
  85 +
  86 +
  87 +@dataclass(frozen=True)
  88 +class RerankConfig:
  89 + """Search-time rerank configuration."""
  90 +
  91 + enabled: bool = True
  92 + rerank_window: int = 384
  93 + timeout_sec: float = 15.0
  94 + weight_es: float = 0.4
  95 + weight_ai: float = 0.6
  96 + rerank_query_template: str = "{query}"
  97 + rerank_doc_template: str = "{title}"
  98 +
  99 +
  100 +@dataclass(frozen=True)
  101 +class SearchConfig:
  102 + """Search behavior configuration shared by backend and indexer."""
  103 +
  104 + field_boosts: Dict[str, float]
  105 + indexes: List[IndexConfig] = field(default_factory=list)
  106 + query_config: QueryConfig = field(default_factory=QueryConfig)
  107 + function_score: FunctionScoreConfig = field(default_factory=FunctionScoreConfig)
  108 + rerank: RerankConfig = field(default_factory=RerankConfig)
  109 + spu_config: SPUConfig = field(default_factory=SPUConfig)
  110 + es_index_name: str = "search_products"
  111 + es_settings: Dict[str, Any] = field(default_factory=dict)
  112 +
  113 +
  114 +@dataclass(frozen=True)
  115 +class TranslationServiceConfig:
  116 + """Translator service configuration."""
  117 +
  118 + endpoint: str
  119 + timeout_sec: float
  120 + default_model: str
  121 + default_scene: str
  122 + cache: Dict[str, Any]
  123 + capabilities: Dict[str, Dict[str, Any]]
  124 +
  125 + def as_dict(self) -> Dict[str, Any]:
  126 + return {
  127 + "service_url": self.endpoint,
  128 + "timeout_sec": self.timeout_sec,
  129 + "default_model": self.default_model,
  130 + "default_scene": self.default_scene,
  131 + "cache": self.cache,
  132 + "capabilities": self.capabilities,
  133 + }
  134 +
  135 +
  136 +@dataclass(frozen=True)
  137 +class EmbeddingServiceConfig:
  138 + """Embedding service configuration."""
  139 +
  140 + provider: str
  141 + providers: Dict[str, Any]
  142 + backend: str
  143 + backends: Dict[str, Dict[str, Any]]
  144 + image_backend: str
  145 + image_backends: Dict[str, Dict[str, Any]]
  146 +
  147 + def get_provider_config(self) -> Dict[str, Any]:
  148 + return dict(self.providers.get(self.provider, {}) or {})
  149 +
  150 + def get_backend_config(self) -> Dict[str, Any]:
  151 + return dict(self.backends.get(self.backend, {}) or {})
  152 +
  153 + def get_image_backend_config(self) -> Dict[str, Any]:
  154 + return dict(self.image_backends.get(self.image_backend, {}) or {})
  155 +
  156 +
  157 +@dataclass(frozen=True)
  158 +class RerankServiceConfig:
  159 + """Reranker service configuration."""
  160 +
  161 + provider: str
  162 + providers: Dict[str, Any]
  163 + backend: str
  164 + backends: Dict[str, Dict[str, Any]]
  165 + request: Dict[str, Any]
  166 +
  167 + def get_provider_config(self) -> Dict[str, Any]:
  168 + return dict(self.providers.get(self.provider, {}) or {})
  169 +
  170 + def get_backend_config(self) -> Dict[str, Any]:
  171 + return dict(self.backends.get(self.backend, {}) or {})
  172 +
  173 +
  174 +@dataclass(frozen=True)
  175 +class ServicesConfig:
  176 + """All service-level configuration."""
  177 +
  178 + translation: TranslationServiceConfig
  179 + embedding: EmbeddingServiceConfig
  180 + rerank: RerankServiceConfig
  181 +
  182 +
  183 +@dataclass(frozen=True)
  184 +class TenantCatalogConfig:
  185 + """Tenant catalog configuration."""
  186 +
  187 + default: Dict[str, Any]
  188 + tenants: Dict[str, Dict[str, Any]]
  189 +
  190 + def get_raw(self) -> Dict[str, Any]:
  191 + return {
  192 + "default": dict(self.default),
  193 + "tenants": {str(key): dict(value) for key, value in self.tenants.items()},
  194 + }
  195 +
  196 +
  197 +@dataclass(frozen=True)
  198 +class ElasticsearchSettings:
  199 + host: str = "http://localhost:9200"
  200 + username: Optional[str] = None
  201 + password: Optional[str] = None
  202 +
  203 +
  204 +@dataclass(frozen=True)
  205 +class RedisSettings:
  206 + host: str = "localhost"
  207 + port: int = 6479
  208 + snapshot_db: int = 0
  209 + password: Optional[str] = None
  210 + socket_timeout: int = 1
  211 + socket_connect_timeout: int = 1
  212 + retry_on_timeout: bool = False
  213 + cache_expire_days: int = 720
  214 + embedding_cache_prefix: str = "embedding"
  215 + anchor_cache_prefix: str = "product_anchors"
  216 + anchor_cache_expire_days: int = 30
  217 +
  218 +
  219 +@dataclass(frozen=True)
  220 +class DatabaseSettings:
  221 + host: Optional[str] = None
  222 + port: int = 3306
  223 + database: Optional[str] = None
  224 + username: Optional[str] = None
  225 + password: Optional[str] = None
  226 +
  227 +
  228 +@dataclass(frozen=True)
  229 +class SecretsConfig:
  230 + dashscope_api_key: Optional[str] = None
  231 + deepl_auth_key: Optional[str] = None
  232 +
  233 +
  234 +@dataclass(frozen=True)
  235 +class InfrastructureConfig:
  236 + elasticsearch: ElasticsearchSettings
  237 + redis: RedisSettings
  238 + database: DatabaseSettings
  239 + secrets: SecretsConfig
  240 +
  241 +
  242 +@dataclass(frozen=True)
  243 +class ProductEnrichConfig:
  244 + """Configuration for LLM-based product content understanding (enrich-content)."""
  245 +
  246 + max_workers: int = 40
  247 +
  248 +
  249 +@dataclass(frozen=True)
  250 +class RuntimeConfig:
  251 + environment: str = "prod"
  252 + index_namespace: str = ""
  253 + api_host: str = "0.0.0.0"
  254 + api_port: int = 6002
  255 + indexer_host: str = "0.0.0.0"
  256 + indexer_port: int = 6004
  257 + embedding_host: str = "127.0.0.1"
  258 + embedding_port: int = 6005
  259 + embedding_text_port: int = 6005
  260 + embedding_image_port: int = 6008
  261 + translator_host: str = "127.0.0.1"
  262 + translator_port: int = 6006
  263 + reranker_host: str = "127.0.0.1"
  264 + reranker_port: int = 6007
  265 +
  266 +
  267 +@dataclass(frozen=True)
  268 +class AssetsConfig:
  269 + query_rewrite_dictionary_path: Path
  270 +
  271 +
  272 +@dataclass(frozen=True)
  273 +class ConfigMetadata:
  274 + loaded_files: Tuple[str, ...]
  275 + config_hash: str
  276 + deprecated_keys: Tuple[str, ...] = field(default_factory=tuple)
  277 +
  278 +
  279 +@dataclass(frozen=True)
  280 +class AppConfig:
  281 + """Root application configuration."""
  282 +
  283 + runtime: RuntimeConfig
  284 + infrastructure: InfrastructureConfig
  285 + product_enrich: ProductEnrichConfig
  286 + search: SearchConfig
  287 + services: ServicesConfig
  288 + tenants: TenantCatalogConfig
  289 + assets: AssetsConfig
  290 + metadata: ConfigMetadata
  291 +
  292 + def sanitized_dict(self) -> Dict[str, Any]:
  293 + data = asdict(self)
  294 + data["infrastructure"]["elasticsearch"]["password"] = _mask_secret(
  295 + data["infrastructure"]["elasticsearch"].get("password")
  296 + )
  297 + data["infrastructure"]["database"]["password"] = _mask_secret(
  298 + data["infrastructure"]["database"].get("password")
  299 + )
  300 + data["infrastructure"]["redis"]["password"] = _mask_secret(
  301 + data["infrastructure"]["redis"].get("password")
  302 + )
  303 + data["infrastructure"]["secrets"]["dashscope_api_key"] = _mask_secret(
  304 + data["infrastructure"]["secrets"].get("dashscope_api_key")
  305 + )
  306 + data["infrastructure"]["secrets"]["deepl_auth_key"] = _mask_secret(
  307 + data["infrastructure"]["secrets"].get("deepl_auth_key")
  308 + )
  309 + return data
  310 +
  311 +
  312 +def _mask_secret(value: Optional[str]) -> Optional[str]:
  313 + if not value:
  314 + return value
  315 + return "***"
config/services_config.py
1 """ 1 """
2 -Services configuration - single source for translation, embedding, rerank. 2 +Unified service configuration accessors.
3 3
4 -Translation is modeled as:  
5 -- one translator service endpoint used by business callers  
6 -- multiple translation capabilities loaded inside the translator service 4 +This module is now a thin adapter over ``config.loader.get_app_config`` and
  5 +contains no independent parsing or precedence logic.
7 """ 6 """
8 7
9 from __future__ import annotations 8 from __future__ import annotations
10 9
11 -import os  
12 -from dataclasses import dataclass, field  
13 -from functools import lru_cache  
14 -from pathlib import Path  
15 -from typing import Any, Dict, List, Optional  
16 -  
17 -import yaml  
18 -from translation.settings import TranslationConfig, build_translation_config, get_translation_cache  
19 -  
20 -  
21 -@dataclass  
22 -class ServiceConfig:  
23 - """Config for one capability (embedding/rerank)."""  
24 -  
25 - provider: str  
26 - providers: Dict[str, Any] = field(default_factory=dict)  
27 -  
28 - def get_provider_cfg(self) -> Dict[str, Any]:  
29 - p = (self.provider or "").strip().lower()  
30 - return self.providers.get(p, {}) if isinstance(self.providers, dict) else {}  
31 -  
32 -  
33 -def _load_services_raw(config_path: Optional[Path] = None) -> Dict[str, Any]:  
34 - if config_path is None:  
35 - config_path = Path(__file__).parent / "config.yaml"  
36 - path = Path(config_path)  
37 - if not path.exists():  
38 - raise FileNotFoundError(f"services config file not found: {path}")  
39 - try:  
40 - with open(path, "r", encoding="utf-8") as f:  
41 - data = yaml.safe_load(f)  
42 - except Exception as exc:  
43 - raise RuntimeError(f"failed to parse services config from {path}: {exc}") from exc  
44 - if not isinstance(data, dict):  
45 - raise RuntimeError(f"invalid config format in {path}: expected mapping root")  
46 - services = data.get("services")  
47 - if not isinstance(services, dict):  
48 - raise RuntimeError("config.yaml must contain a valid 'services' mapping")  
49 - return services  
50 -  
51 -  
52 -def _resolve_provider_name(env_name: str, config_provider: Any, capability: str) -> str:  
53 - provider = os.getenv(env_name) or config_provider  
54 - if not provider:  
55 - raise ValueError(  
56 - f"services.{capability}.provider is required "  
57 - f"(or set env override {env_name})"  
58 - )  
59 - return str(provider).strip().lower()  
60 -  
61 -  
62 -def _resolve_translation() -> TranslationConfig:  
63 - raw = _load_services_raw()  
64 - cfg = raw.get("translation", {}) if isinstance(raw.get("translation"), dict) else {}  
65 - return build_translation_config(cfg)  
66 -  
67 -  
68 -def _resolve_embedding() -> ServiceConfig:  
69 - raw = _load_services_raw()  
70 - cfg = raw.get("embedding", {}) if isinstance(raw.get("embedding"), dict) else {}  
71 - providers = cfg.get("providers", {}) if isinstance(cfg.get("providers"), dict) else {}  
72 -  
73 - provider = _resolve_provider_name(  
74 - env_name="EMBEDDING_PROVIDER",  
75 - config_provider=cfg.get("provider"),  
76 - capability="embedding",  
77 - )  
78 - if provider != "http":  
79 - raise ValueError(f"Unsupported embedding provider: {provider}")  
80 -  
81 - env_text_url = os.getenv("EMBEDDING_TEXT_SERVICE_URL")  
82 - env_image_url = os.getenv("EMBEDDING_IMAGE_SERVICE_URL")  
83 - if provider == "http":  
84 - providers = dict(providers)  
85 - http_cfg = dict(providers.get("http", {}))  
86 - if env_text_url:  
87 - http_cfg["text_base_url"] = env_text_url.rstrip("/")  
88 - if env_image_url:  
89 - http_cfg["image_base_url"] = env_image_url.rstrip("/")  
90 - if not http_cfg.get("text_base_url"):  
91 - raise ValueError("services.embedding.providers.http.text_base_url is required")  
92 - if not http_cfg.get("image_base_url"):  
93 - raise ValueError("services.embedding.providers.http.image_base_url is required")  
94 - providers["http"] = http_cfg  
95 -  
96 - return ServiceConfig(provider=provider, providers=providers)  
97 -  
98 -  
99 -def _resolve_rerank() -> ServiceConfig:  
100 - raw = _load_services_raw()  
101 - cfg = raw.get("rerank", {}) if isinstance(raw.get("rerank"), dict) else {}  
102 - providers = cfg.get("providers", {}) if isinstance(cfg.get("providers"), dict) else {}  
103 -  
104 - provider = _resolve_provider_name(  
105 - env_name="RERANK_PROVIDER",  
106 - config_provider=cfg.get("provider"),  
107 - capability="rerank",  
108 - )  
109 - if provider != "http":  
110 - raise ValueError(f"Unsupported rerank provider: {provider}")  
111 -  
112 - env_url = os.getenv("RERANKER_SERVICE_URL")  
113 - if env_url:  
114 - url = env_url.rstrip("/")  
115 - if not url.endswith("/rerank"):  
116 - url = f"{url}/rerank" if "/rerank" not in url else url  
117 - providers = dict(providers)  
118 - providers["http"] = dict(providers.get("http", {}))  
119 - providers["http"]["base_url"] = url.replace("/rerank", "")  
120 - providers["http"]["service_url"] = url  
121 -  
122 - return ServiceConfig(provider=provider, providers=providers)  
123 -  
124 -  
125 -def get_rerank_backend_config() -> tuple[str, dict]:  
126 - raw = _load_services_raw()  
127 - cfg = raw.get("rerank", {}) if isinstance(raw.get("rerank"), dict) else {}  
128 - backends = cfg.get("backends", {}) if isinstance(cfg.get("backends"), dict) else {}  
129 - name = os.getenv("RERANK_BACKEND") or cfg.get("backend")  
130 - if not name:  
131 - raise ValueError("services.rerank.backend is required (or env RERANK_BACKEND)")  
132 - name = str(name).strip().lower()  
133 - backend_cfg = backends.get(name, {}) if isinstance(backends.get(name), dict) else {}  
134 - if not backend_cfg:  
135 - raise ValueError(f"services.rerank.backends.{name} is required")  
136 - return name, backend_cfg  
137 -  
138 -  
139 -def get_embedding_backend_config() -> tuple[str, dict]:  
140 - raw = _load_services_raw()  
141 - cfg = raw.get("embedding", {}) if isinstance(raw.get("embedding"), dict) else {}  
142 - backends = cfg.get("backends", {}) if isinstance(cfg.get("backends"), dict) else {}  
143 - name = os.getenv("EMBEDDING_BACKEND") or cfg.get("backend")  
144 - if not name:  
145 - raise ValueError("services.embedding.backend is required (or env EMBEDDING_BACKEND)")  
146 - name = str(name).strip().lower()  
147 - backend_cfg = backends.get(name, {}) if isinstance(backends.get(name), dict) else {}  
148 - if not backend_cfg:  
149 - raise ValueError(f"services.embedding.backends.{name} is required")  
150 - return name, backend_cfg  
151 -  
152 -  
153 -@lru_cache(maxsize=1)  
154 -def get_translation_config() -> TranslationConfig:  
155 - return _resolve_translation()  
156 -  
157 -  
158 -@lru_cache(maxsize=1)  
159 -def get_embedding_config() -> ServiceConfig:  
160 - return _resolve_embedding()  
161 -  
162 -  
163 -@lru_cache(maxsize=1)  
164 -def get_rerank_config() -> ServiceConfig:  
165 - return _resolve_rerank() 10 +from typing import Any, Dict, Tuple
  11 +
  12 +from config.loader import get_app_config
  13 +from config.schema import EmbeddingServiceConfig, RerankServiceConfig, TranslationServiceConfig
  14 +
  15 +
  16 +def get_translation_config() -> Dict[str, Any]:
  17 + return get_app_config().services.translation.as_dict()
  18 +
  19 +
  20 +def get_embedding_config() -> EmbeddingServiceConfig:
  21 + return get_app_config().services.embedding
  22 +
  23 +
  24 +def get_rerank_config() -> RerankServiceConfig:
  25 + return get_app_config().services.rerank
166 26
167 27
168 def get_translation_base_url() -> str: 28 def get_translation_base_url() -> str:
169 - return str(get_translation_config()["service_url"]) 29 + return get_app_config().services.translation.endpoint
170 30
171 31
172 def get_translation_cache_config() -> Dict[str, Any]: 32 def get_translation_cache_config() -> Dict[str, Any]:
173 - return get_translation_cache(get_translation_config()) 33 + return dict(get_app_config().services.translation.cache)
174 34
175 35
176 def get_embedding_text_base_url() -> str: 36 def get_embedding_text_base_url() -> str:
177 - provider_cfg = get_embedding_config().providers.get("http", {})  
178 - base = os.getenv("EMBEDDING_TEXT_SERVICE_URL") or provider_cfg.get("text_base_url") 37 + provider_cfg = get_app_config().services.embedding.get_provider_config()
  38 + base = provider_cfg.get("text_base_url")
179 if not base: 39 if not base:
180 - raise ValueError("Embedding text HTTP base_url is not configured") 40 + raise ValueError("Embedding text base_url is not configured")
181 return str(base).rstrip("/") 41 return str(base).rstrip("/")
182 42
183 43
184 def get_embedding_image_base_url() -> str: 44 def get_embedding_image_base_url() -> str:
185 - provider_cfg = get_embedding_config().providers.get("http", {})  
186 - base = os.getenv("EMBEDDING_IMAGE_SERVICE_URL") or provider_cfg.get("image_base_url") 45 + provider_cfg = get_app_config().services.embedding.get_provider_config()
  46 + base = provider_cfg.get("image_base_url")
187 if not base: 47 if not base:
188 - raise ValueError("Embedding image HTTP base_url is not configured") 48 + raise ValueError("Embedding image base_url is not configured")
189 return str(base).rstrip("/") 49 return str(base).rstrip("/")
190 50
191 51
  52 +def get_embedding_backend_config() -> Tuple[str, Dict[str, Any]]:
  53 + cfg = get_app_config().services.embedding
  54 + return cfg.backend, cfg.get_backend_config()
  55 +
  56 +
  57 +def get_embedding_image_backend_config() -> Tuple[str, Dict[str, Any]]:
  58 + cfg = get_app_config().services.embedding
  59 + return cfg.image_backend, cfg.get_image_backend_config()
  60 +
  61 +
  62 +def get_rerank_backend_config() -> Tuple[str, Dict[str, Any]]:
  63 + cfg = get_app_config().services.rerank
  64 + return cfg.backend, cfg.get_backend_config()
  65 +
  66 +
192 def get_rerank_base_url() -> str: 67 def get_rerank_base_url() -> str:
193 - base = (  
194 - os.getenv("RERANKER_SERVICE_URL")  
195 - or get_rerank_config().providers.get("http", {}).get("service_url")  
196 - or get_rerank_config().providers.get("http", {}).get("base_url")  
197 - ) 68 + provider_cfg = get_app_config().services.rerank.get_provider_config()
  69 + base = provider_cfg.get("service_url") or provider_cfg.get("base_url")
198 if not base: 70 if not base:
199 - raise ValueError("Rerank HTTP base_url is not configured") 71 + raise ValueError("Rerank service URL is not configured")
200 return str(base).rstrip("/") 72 return str(base).rstrip("/")
201 73
202 74
203 def get_rerank_service_url() -> str: 75 def get_rerank_service_url() -> str:
204 - """Backward-compatible alias."""  
205 return get_rerank_base_url() 76 return get_rerank_base_url()
config/tenant_config_loader.py
@@ -2,12 +2,13 @@ @@ -2,12 +2,13 @@
2 租户配置加载器。 2 租户配置加载器。
3 3
4 从统一配置文件(config.yaml)加载租户配置,包括主语言和索引语言(index_languages)。 4 从统一配置文件(config.yaml)加载租户配置,包括主语言和索引语言(index_languages)。
5 -支持旧配置 translate_to_en / translate_to_zh 的兼容解析。  
6 """ 5 """
7 6
8 import logging 7 import logging
9 from typing import Dict, Any, Optional, List 8 from typing import Dict, Any, Optional, List
10 9
  10 +from config.loader import get_app_config
  11 +
11 logger = logging.getLogger(__name__) 12 logger = logging.getLogger(__name__)
12 13
13 # 支持的索引语言:code -> display name(供商家勾选主市场语言等场景使用) 14 # 支持的索引语言:code -> display name(供商家勾选主市场语言等场景使用)
@@ -83,25 +84,13 @@ def resolve_index_languages( @@ -83,25 +84,13 @@ def resolve_index_languages(
83 ) -> List[str]: 84 ) -> List[str]:
84 """ 85 """
85 从租户配置解析 index_languages。 86 从租户配置解析 index_languages。
86 - 若存在 index_languages 则用之;否则按旧配置 translate_to_en / translate_to_zh 推导 87 + 若配置缺失或非法,则回退到默认配置
87 """ 88 """
88 - if "index_languages" in tenant_config:  
89 - normalized = normalize_index_languages(  
90 - tenant_config["index_languages"],  
91 - tenant_config.get("primary_language") or "en",  
92 - )  
93 - return normalized if normalized else list(default_index_languages)  
94 - primary = (tenant_config.get("primary_language") or "en").strip().lower()  
95 - to_en = bool(tenant_config.get("translate_to_en"))  
96 - to_zh = bool(tenant_config.get("translate_to_zh"))  
97 - langs: List[str] = []  
98 - if primary and primary in SOURCE_LANG_CODE_MAP:  
99 - langs.append(primary)  
100 - for code in ("en", "zh"):  
101 - if code not in langs and ((code == "en" and to_en) or (code == "zh" and to_zh)):  
102 - if code in SOURCE_LANG_CODE_MAP:  
103 - langs.append(code)  
104 - return langs if langs else list(default_index_languages) 89 + normalized = normalize_index_languages(
  90 + tenant_config.get("index_languages"),
  91 + tenant_config.get("primary_language") or "en",
  92 + )
  93 + return normalized if normalized else list(default_index_languages)
105 94
106 95
107 class TenantConfigLoader: 96 class TenantConfigLoader:
@@ -122,15 +111,8 @@ class TenantConfigLoader: @@ -122,15 +111,8 @@ class TenantConfigLoader:
122 return self._config 111 return self._config
123 112
124 try: 113 try:
125 - from config import ConfigLoader  
126 -  
127 - config_loader = ConfigLoader()  
128 - search_config = config_loader.load_config()  
129 - tenant_cfg = search_config.tenant_config  
130 - if not isinstance(tenant_cfg, dict):  
131 - raise RuntimeError("tenant_config must be an object")  
132 -  
133 - default_cfg = tenant_cfg.get("default") 114 + tenant_cfg = get_app_config().tenants
  115 + default_cfg = tenant_cfg.default
134 if not isinstance(default_cfg, dict): 116 if not isinstance(default_cfg, dict):
135 raise RuntimeError("tenant_config.default must be configured in config.yaml") 117 raise RuntimeError("tenant_config.default must be configured in config.yaml")
136 default_primary = (default_cfg.get("primary_language") or "en").strip().lower() 118 default_primary = (default_cfg.get("primary_language") or "en").strip().lower()
@@ -143,7 +125,7 @@ class TenantConfigLoader: @@ -143,7 +125,7 @@ class TenantConfigLoader:
143 "tenant_config.default.index_languages must include at least one supported language" 125 "tenant_config.default.index_languages must include at least one supported language"
144 ) 126 )
145 127
146 - tenants_cfg = tenant_cfg.get("tenants", {}) 128 + tenants_cfg = tenant_cfg.tenants
147 if not isinstance(tenants_cfg, dict): 129 if not isinstance(tenants_cfg, dict):
148 raise RuntimeError("tenant_config.tenants must be an object") 130 raise RuntimeError("tenant_config.tenants must be an object")
149 131
1 """Configuration helper functions for dynamic multi-language search fields.""" 1 """Configuration helper functions for dynamic multi-language search fields."""
2 2
3 from typing import Dict, List 3 from typing import Dict, List
4 -from .config_loader import SearchConfig 4 +
  5 +from config.schema import SearchConfig
5 6
6 7
7 def _format_field_with_boost(field_name: str, boost: float) -> str: 8 def _format_field_with_boost(field_name: str, boost: float) -> str:
docs/config-system-review-and-redesign.md 0 → 100644
@@ -0,0 +1,738 @@ @@ -0,0 +1,738 @@
  1 +# Configuration System Review And Redesign
  2 +
  3 +## 1. Goal
  4 +
  5 +This document reviews the current configuration system and proposes a practical redesign for long-term maintainability.
  6 +
  7 +The target is a configuration system that is:
  8 +
  9 +- unified in loading and ownership
  10 +- clear in boundaries and precedence
  11 +- visible in effective behavior
  12 +- easy to evolve across development, deployment, and operations
  13 +
  14 +This review is based on the current implementation, not only on the intended architecture in docs.
  15 +
  16 +## 2. Project Context
  17 +
  18 +The repo already defines the right architectural direction:
  19 +
  20 +- `config/config.yaml` should be the main configuration source for search behavior and service wiring
  21 +- `.env` should mainly carry deployment-specific values and secrets
  22 +- provider/backend expansion should stay centralized instead of spreading through business code
  23 +
  24 +That direction is described in:
  25 +
  26 +- [`README.md`](/data/saas-search/README.md)
  27 +- [`docs/DEVELOPER_GUIDE.md`](/data/saas-search/docs/DEVELOPER_GUIDE.md)
  28 +- [`docs/QUICKSTART.md`](/data/saas-search/docs/QUICKSTART.md)
  29 +- [`translation/README.md`](/data/saas-search/translation/README.md)
  30 +
  31 +The problem is not the architectural intent. The problem is that the current implementation only partially follows it.
  32 +
  33 +## 3. Current-State Review
  34 +
  35 +### 3.1 What exists today
  36 +
  37 +The current system effectively has several configuration channels:
  38 +
  39 +- `config/config.yaml`
  40 + - search behavior
  41 + - rerank behavior
  42 + - services registry
  43 + - tenant config
  44 +- `config/config_loader.py`
  45 + - parses search behavior and tenant config into `SearchConfig`
  46 + - also injects some defaults from code
  47 +- `config/services_config.py`
  48 + - reparses `config/config.yaml` again, independently
  49 + - resolves translation, embedding, rerank service config
  50 + - also applies env overrides
  51 +- `config/env_config.py`
  52 + - loads `.env`
  53 + - defines ES, Redis, DB, host/port, service URLs, namespace, model path defaults
  54 +- service-local config modules
  55 + - [`embeddings/config.py`](/data/saas-search/embeddings/config.py)
  56 + - [`reranker/config.py`](/data/saas-search/reranker/config.py)
  57 +- startup scripts
  58 + - derive defaults from shell env, Python config, and YAML in different combinations
  59 +- inline fallbacks in business logic
  60 + - query parsing
  61 + - indexing
  62 + - service startup
  63 +
  64 +### 3.2 Main findings
  65 +
  66 +#### Finding A: there is no single loader for the full effective configuration
  67 +
  68 +`ConfigLoader` and `services_config` both parse `config/config.yaml`, but they do so separately and with different responsibilities.
  69 +
  70 +- [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L148)
  71 +- [`config/services_config.py`](/data/saas-search/config/services_config.py#L33)
  72 +
  73 +Impact:
  74 +
  75 +- the same file is loaded twice through different code paths
  76 +- search config and services config can drift in interpretation
  77 +- alternative config paths are hard to support cleanly
  78 +- tests and tools cannot ask one place for the full effective config tree
  79 +
  80 +#### Finding B: precedence is not explicit, stable, or globally enforced
  81 +
  82 +Current precedence differs by subsystem:
  83 +
  84 +- search behavior mostly comes from YAML plus code defaults
  85 +- embedding and rerank allow env overrides for provider/backend/url
  86 +- translation intentionally blocks some env overrides
  87 +- startup scripts still choose host/port and mode via env
  88 +- some values are reconstructed from other env vars
  89 +
  90 +Examples:
  91 +
  92 +- env override for embedding provider/url/backend:
  93 + - [`config/services_config.py`](/data/saas-search/config/services_config.py#L52)
  94 + - [`config/services_config.py`](/data/saas-search/config/services_config.py#L68)
  95 + - [`config/services_config.py`](/data/saas-search/config/services_config.py#L139)
  96 +- host/port and service URL reconstruction:
  97 + - [`config/env_config.py`](/data/saas-search/config/env_config.py#L55)
  98 + - [`config/env_config.py`](/data/saas-search/config/env_config.py#L75)
  99 +- translator host/port still driven by startup env:
  100 + - [`scripts/start_translator.sh`](/data/saas-search/scripts/start_translator.sh#L28)
  101 +
  102 +Impact:
  103 +
  104 +- operators cannot reliably predict the effective configuration by reading one file
  105 +- the same setting category behaves differently across services
  106 +- incidents become harder to debug because source-of-truth depends on the code path
  107 +
  108 +#### Finding C: defaults are duplicated across YAML and code
  109 +
  110 +There are several layers of default values:
  111 +
  112 +- dataclass defaults in `QueryConfig`
  113 +- fallback defaults in `ConfigLoader._parse_config`
  114 +- defaults in `config.yaml`
  115 +- defaults in `env_config.py`
  116 +- defaults in `embeddings/config.py`
  117 +- defaults in `reranker/config.py`
  118 +- defaults in startup scripts
  119 +
  120 +Examples:
  121 +
  122 +- query defaults duplicated in dataclass and parser:
  123 + - [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L24)
  124 + - [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L240)
  125 +- embedding defaults duplicated in YAML, `services_config`, `embeddings/config.py`, and startup script:
  126 + - [`config/config.yaml`](/data/saas-search/config/config.yaml#L196)
  127 + - [`embeddings/config.py`](/data/saas-search/embeddings/config.py#L14)
  128 + - [`scripts/start_embedding_service.sh`](/data/saas-search/scripts/start_embedding_service.sh#L29)
  129 +- reranker defaults duplicated in YAML and `reranker/config.py`:
  130 + - [`config/config.yaml`](/data/saas-search/config/config.yaml#L214)
  131 + - [`reranker/config.py`](/data/saas-search/reranker/config.py#L6)
  132 +
  133 +Impact:
  134 +
  135 +- changing a default is risky because there may be multiple hidden copies
  136 +- code review cannot easily tell whether a value is authoritative or dead legacy
  137 +- “same config” may behave differently across processes
  138 +
  139 +#### Finding D: config is still embedded in runtime logic
  140 +
  141 +Some important behavior remains encoded as inline fallback logic rather than declared config.
  142 +
  143 +Examples:
  144 +
  145 +- query-time translation target languages fallback to `["en", "zh"]`:
  146 + - [`query/query_parser.py`](/data/saas-search/query/query_parser.py#L339)
  147 +- indexer text handling and LLM enrichment also fallback to `["en", "zh"]`:
  148 + - [`indexer/document_transformer.py`](/data/saas-search/indexer/document_transformer.py#L216)
  149 + - [`indexer/document_transformer.py`](/data/saas-search/indexer/document_transformer.py#L310)
  150 + - [`indexer/document_transformer.py`](/data/saas-search/indexer/document_transformer.py#L649)
  151 +
  152 +Impact:
  153 +
  154 +- configuration is not fully visible in config files
  155 +- behavior can silently change when tenant config is missing or malformed
  156 +- “default behavior” is spread across business modules
  157 +
  158 +#### Finding E: some configuration assets are not managed as first-class config
  159 +
  160 +Query rewrite is configured through an external file, but the file path is hardcoded and currently inconsistent with the repository content.
  161 +
  162 +- loader expects:
  163 + - [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L162)
  164 +- repo currently contains:
  165 + - [`config/query_rewrite.dict`](/data/saas-search/config/query_rewrite.dict)
  166 +
  167 +There is also an admin API that mutates rewrite rules in memory only:
  168 +
  169 +- [`api/routes/admin.py`](/data/saas-search/api/routes/admin.py#L68)
  170 +- [`query/query_parser.py`](/data/saas-search/query/query_parser.py#L622)
  171 +
  172 +Impact:
  173 +
  174 +- rewrite rules are neither cleanly file-backed nor fully runtime-managed
  175 +- restart behavior is unclear
  176 +- configuration visibility and persistence are weak
  177 +
  178 +#### Finding F: visibility is limited
  179 +
  180 +The system exposes only a small sanitized subset at `/admin/config`.
  181 +
  182 +- [`api/routes/admin.py`](/data/saas-search/api/routes/admin.py#L42)
  183 +
  184 +At the same time, the true effective config includes:
  185 +
  186 +- tenant overlays
  187 +- env overrides
  188 +- service backend selections
  189 +- script-selected modes
  190 +- hidden defaults in code
  191 +
  192 +Impact:
  193 +
  194 +- there is no authoritative “effective config” view
  195 +- debugging configuration mismatches requires source reading
  196 +- operators cannot easily verify what each process actually started with
  197 +
  198 +#### Finding G: the indexer does not really consume the unified config as a first-class dependency
  199 +
  200 +Indexer startup explicitly says config is loaded only for parity/logging and routes do not depend on it.
  201 +
  202 +- [`api/indexer_app.py`](/data/saas-search/api/indexer_app.py#L76)
  203 +
  204 +Impact:
  205 +
  206 +- configuration is not truly system-wide
  207 +- search-side and indexer-side behavior can drift
  208 +- the current “unified config” is only partially unified
  209 +
  210 +#### Finding H: docs still carry legacy and mixed mental models
  211 +
  212 +Most high-level docs describe the desired centralized model, but some implementation/docs still expose legacy concepts such as `translate_to_en` and `translate_to_zh`.
  213 +
  214 +- desired model:
  215 + - [`README.md`](/data/saas-search/README.md#L78)
  216 + - [`docs/DEVELOPER_GUIDE.md`](/data/saas-search/docs/DEVELOPER_GUIDE.md#L207)
  217 + - [`translation/README.md`](/data/saas-search/translation/README.md#L161)
  218 +- legacy tenant translation flags still documented:
  219 + - [`indexer/README.md`](/data/saas-search/indexer/README.md#L39)
  220 +
  221 +Impact:
  222 +
  223 +- new developers may follow old mental models
  224 +- cleanup work keeps getting deferred because old and new systems appear both “supported”
  225 +
  226 +## 4. Design Principles For The Redesign
  227 +
  228 +The redesign should follow these rules.
  229 +
  230 +### 4.1 One logical configuration system
  231 +
  232 +It is acceptable to have multiple files, but not multiple loaders with overlapping ownership.
  233 +
  234 +There must be one loader pipeline that produces one typed `AppConfig`.
  235 +
  236 +### 4.2 Configuration files declare, parser code interprets, env provides runtime injection
  237 +
  238 +Responsibilities should be:
  239 +
  240 +- configuration files
  241 + - declare non-secret desired behavior and non-secret deployable settings
  242 +- parsing logic
  243 + - load, merge, validate, normalize, and expose typed config
  244 + - never invent hidden business behavior
  245 +- environment variables
  246 + - carry secrets and a small set of runtime/process values
  247 + - do not redefine business behavior casually
  248 +
  249 +### 4.3 One precedence rule for the whole system
  250 +
  251 +Every config category should follow the same merge model unless explicitly exempted.
  252 +
  253 +### 4.4 No silent implicit fallback for business behavior
  254 +
  255 +Fail fast at startup when required config is missing or invalid.
  256 +
  257 +Do not silently fall back to legacy behavior such as hardcoded language lists.
  258 +
  259 +### 4.5 Effective configuration must be observable
  260 +
  261 +Every service should be able to show:
  262 +
  263 +- config version or hash
  264 +- source files loaded
  265 +- environment name
  266 +- sanitized effective configuration
  267 +
  268 +## 5. Recommended Target Design
  269 +
  270 +## 5.1 Boundary model
  271 +
  272 +Use three clear layers.
  273 +
  274 +### Layer 1: repository-managed static config
  275 +
  276 +Purpose:
  277 +
  278 +- search behavior
  279 +- tenant behavior
  280 +- provider/backend registry
  281 +- non-secret service topology defaults
  282 +- feature switches
  283 +
  284 +Examples:
  285 +
  286 +- field boosts
  287 +- query strategy
  288 +- rerank fusion parameters
  289 +- tenant language plans
  290 +- translation capability registry
  291 +- embedding backend selection default
  292 +
  293 +### Layer 2: environment-specific overlays
  294 +
  295 +Purpose:
  296 +
  297 +- per-environment non-secret differences
  298 +- service endpoints by environment
  299 +- resource sizing defaults by environment
  300 +- dev/test/prod operational differences
  301 +
  302 +Examples:
  303 +
  304 +- local embedding URL vs production URL
  305 +- dev rerank backend vs prod rerank backend
  306 +- lower concurrency in local development
  307 +
  308 +### Layer 3: environment variables
  309 +
  310 +Purpose:
  311 +
  312 +- secrets
  313 +- bind host/port
  314 +- external infrastructure credentials
  315 +- container-orchestrator last-mile injection
  316 +
  317 +Examples:
  318 +
  319 +- `ES_HOST`, `ES_USERNAME`, `ES_PASSWORD`
  320 +- `DB_HOST`, `DB_USERNAME`, `DB_PASSWORD`
  321 +- `REDIS_HOST`, `REDIS_PASSWORD`
  322 +- `DASHSCOPE_API_KEY`, `DEEPL_AUTH_KEY`
  323 +- `API_HOST`, `API_PORT`, `INDEXER_PORT`, `TRANSLATION_PORT`
  324 +
  325 +Rule:
  326 +
  327 +- environment variables should not be the normal path for choosing business behavior such as translation model, embedding backend, or tenant language policy
  328 +- if an env override is allowed for a non-secret field, it must be explicitly listed and documented as an operational override, not a hidden convention
  329 +
  330 +## 5.2 Unified precedence
  331 +
  332 +Recommended precedence:
  333 +
  334 +1. schema defaults in code
  335 +2. `config/base.yaml`
  336 +3. `config/environments/<env>.yaml`
  337 +4. tenant overlay from `config/tenants/`
  338 +5. environment variables for the explicitly allowed runtime keys
  339 +6. CLI flags for the current process only
  340 +
  341 +Important rule:
  342 +
  343 +- only one module may implement this merge logic
  344 +- no business module may call `os.getenv()` directly for configuration
  345 +
  346 +## 5.3 Recommended directory structure
  347 +
  348 +```text
  349 +config/
  350 + schema.py
  351 + loader.py
  352 + sources.py
  353 + base.yaml
  354 + environments/
  355 + dev.yaml
  356 + test.yaml
  357 + prod.yaml
  358 + tenants/
  359 + _default.yaml
  360 + 1.yaml
  361 + 162.yaml
  362 + 170.yaml
  363 + dictionaries/
  364 + query_rewrite.dict
  365 + README.md
  366 +.env.example
  367 +```
  368 +
  369 +Notes:
  370 +
  371 +- `base.yaml` contains shared defaults and feature behavior
  372 +- `environments/*.yaml` contains environment-specific non-secret overrides
  373 +- `tenants/*.yaml` contains tenant-specific overrides only
  374 +- `dictionaries/` stores first-class config assets such as rewrite dictionaries
  375 +- `schema.py` defines the typed config model
  376 +- `loader.py` is the only entry point that loads and merges config
  377 +
  378 +If the team prefers fewer files, `tenants.yaml` is also acceptable. The key requirement is not “one file”, but “one loading model with clear ownership”.
  379 +
  380 +## 5.4 Typed configuration model
  381 +
  382 +Introduce one root object, for example:
  383 +
  384 +```python
  385 +class AppConfig(BaseModel):
  386 + runtime: RuntimeConfig
  387 + infrastructure: InfrastructureConfig
  388 + search: SearchConfig
  389 + services: ServicesConfig
  390 + tenants: TenantCatalogConfig
  391 + assets: ConfigAssets
  392 +```
  393 +
  394 +Suggested subtrees:
  395 +
  396 +- `runtime`
  397 + - environment name
  398 + - config revision/hash
  399 + - bind addresses/ports
  400 +- `infrastructure`
  401 + - ES
  402 + - DB
  403 + - Redis
  404 + - index namespace
  405 +- `search`
  406 + - field boosts
  407 + - query config
  408 + - function score
  409 + - rerank behavior
  410 + - spu config
  411 +- `services`
  412 + - translation
  413 + - embedding
  414 + - rerank
  415 +- `tenants`
  416 + - default tenant config
  417 + - tenant overrides
  418 +- `assets`
  419 + - rewrite dictionary path
  420 +
  421 +Benefits:
  422 +
  423 +- one validated object shared by backend, indexer, translator, embedding, reranker
  424 +- one place for defaults
  425 +- one place for schema evolution
  426 +
  427 +## 5.5 Loading flow
  428 +
  429 +Recommended loading flow:
  430 +
  431 +1. determine `APP_ENV` or `RUNTIME_ENV`
  432 +2. load schema defaults
  433 +3. load `config/base.yaml`
  434 +4. load `config/environments/<env>.yaml` if present
  435 +5. load tenant files
  436 +6. inject first-class assets such as rewrite dictionary
  437 +7. apply allowed env overrides
  438 +8. validate the final `AppConfig`
  439 +9. freeze and cache the config object
  440 +10. expose a sanitized effective-config view
  441 +
  442 +Important:
  443 +
  444 +- every process should call the same loader
  445 +- services should receive a resolved `AppConfig`, not re-open YAML independently
  446 +
  447 +## 5.6 Clear responsibility split
  448 +
  449 +### Configuration files are responsible for
  450 +
  451 +- what the system should do
  452 +- what providers/backends are available
  453 +- which features are enabled
  454 +- tenant language/index policies
  455 +- non-secret service topology
  456 +
  457 +### Parser/loader code is responsible for
  458 +
  459 +- locating sources
  460 +- merge precedence
  461 +- type validation
  462 +- normalization
  463 +- deprecation warnings
  464 +- producing the final immutable config object
  465 +
  466 +### Environment variables are responsible for
  467 +
  468 +- secrets
  469 +- bind addresses/ports
  470 +- infrastructure endpoints when the deployment platform injects them
  471 +- a very small set of documented operational overrides
  472 +
  473 +### Business code is not responsible for
  474 +
  475 +- inventing defaults for missing config
  476 +- loading YAML directly
  477 +- calling `os.getenv()` for normal application behavior
  478 +
  479 +## 5.7 How to handle service config
  480 +
  481 +Unify all service-facing config under one structure:
  482 +
  483 +```yaml
  484 +services:
  485 + translation:
  486 + endpoint: "http://translator:6006"
  487 + timeout_sec: 10
  488 + default_model: "llm"
  489 + default_scene: "general"
  490 + capabilities: ...
  491 + embedding:
  492 + endpoint:
  493 + text: "http://embedding:6005"
  494 + image: "http://embedding-image:6008"
  495 + backend: "tei"
  496 + backends: ...
  497 + rerank:
  498 + endpoint: "http://reranker:6007/rerank"
  499 + backend: "qwen3_vllm"
  500 + backends: ...
  501 +```
  502 +
  503 +Rules:
  504 +
  505 +- `endpoint` is how callers reach the service
  506 +- `backend` is how the service itself is implemented
  507 +- only the service process cares about `backend`
  508 +- only callers care about `endpoint`
  509 +- both still belong to the same config tree, because they are part of one system
  510 +
  511 +## 5.8 How to handle tenant config
  512 +
  513 +Tenant config should become explicit policy, not translation-era leftovers.
  514 +
  515 +Recommended tenant fields:
  516 +
  517 +- `primary_language`
  518 +- `index_languages`
  519 +- `search_languages`
  520 +- `translation_policy`
  521 +- `facet_policy`
  522 +- optional tenant-specific ranking overrides
  523 +
  524 +Avoid keeping `translate_to_en` and `translate_to_zh` as active concepts in the long-term model.
  525 +
  526 +If compatibility is needed, support them only in the loader as deprecated aliases and emit warnings.
  527 +
  528 +## 5.9 How to handle rewrite rules and similar assets
  529 +
  530 +Treat them as declared config assets.
  531 +
  532 +Recommended rules:
  533 +
  534 +- file path declared in config
  535 +- one canonical location under `config/dictionaries/`
  536 +- loader validates presence and format
  537 +- admin runtime updates either:
  538 + - are removed, or
  539 + - write back through a controlled persistence path
  540 +
  541 +Do not keep a hybrid model where startup loads one file and admin mutates only in memory.
  542 +
  543 +## 5.10 Observability improvements
  544 +
  545 +Add the following:
  546 +
  547 +- `config dump` CLI that prints sanitized effective config
  548 +- startup log with config hash, environment, and config file list
  549 +- `/admin/config/effective` endpoint returning sanitized effective config
  550 +- `/admin/config/meta` endpoint returning:
  551 + - environment
  552 + - config hash
  553 + - loaded source files
  554 + - deprecated keys in use
  555 +
  556 +This is important for operations and for multi-service debugging.
  557 +
  558 +## 6. Practical Refactor Plan
  559 +
  560 +The refactor should be incremental.
  561 +
  562 +### Phase 1: establish the new config core without changing behavior
  563 +
  564 +- create `config/schema.py`
  565 +- create `config/loader.py`
  566 +- move all current defaults into schema models
  567 +- make loader read current `config/config.yaml`
  568 +- make loader read `.env` only for approved keys
  569 +- expose one `get_app_config()`
  570 +
  571 +Result:
  572 +
  573 +- same behavior, but one typed root config becomes available
  574 +
  575 +### Phase 2: remove duplicate readers
  576 +
  577 +- make `services_config.py` a thin adapter over `get_app_config()`
  578 +- make `tenant_config_loader.py` read from `get_app_config()`
  579 +- stop reparsing YAML in `services_config.py`
  580 +- stop service modules from depending on legacy local config modules for behavior
  581 +
  582 +Result:
  583 +
  584 +- one parsing path
  585 +- fewer divergence risks
  586 +
  587 +### Phase 3: move hidden defaults out of business logic
  588 +
  589 +- remove hardcoded fallback language lists from query/indexer modules
  590 +- require tenant defaults to come from config schema only
  591 +- remove duplicate behavior defaults from service code
  592 +
  593 +Result:
  594 +
  595 +- behavior becomes visible and reviewable
  596 +
  597 +### Phase 4: clean service startup configuration
  598 +
  599 +- make startup scripts ask the unified loader for resolved values
  600 +- keep only bind host/port and secret injection in shell env
  601 +- retire or reduce `embeddings/config.py` and `reranker/config.py`
  602 +
  603 +Result:
  604 +
  605 +- startup behavior matches runtime config model
  606 +
  607 +### Phase 5: split config files by responsibility
  608 +
  609 +- keep a single root loader
  610 +- split current giant `config.yaml` into:
  611 + - `base.yaml`
  612 + - `environments/<env>.yaml`
  613 + - `tenants/*.yaml`
  614 + - `dictionaries/query_rewrite.dict`
  615 +
  616 +Result:
  617 +
  618 +- config remains unified logically, but is easier to read and maintain physically
  619 +
  620 +### Phase 6: deprecate legacy compatibility
  621 +
  622 +- deprecate `translate_to_en` and `translate_to_zh`
  623 +- deprecate env-based backend/provider selection except for explicitly approved keys
  624 +- remove old code paths after one or two release cycles
  625 +
  626 +Result:
  627 +
  628 +- the system becomes simpler instead of carrying two generations forever
  629 +
  630 +## 7. Concrete Rules To Adopt
  631 +
  632 +These rules should be documented and enforced in code review.
  633 +
  634 +### Rule 1
  635 +
  636 +Only `config/loader.py` may load config files or `.env`.
  637 +
  638 +### Rule 2
  639 +
  640 +Only `config/loader.py` may read `os.getenv()` for application config.
  641 +
  642 +### Rule 3
  643 +
  644 +Business modules receive typed config objects and do not read files or env directly.
  645 +
  646 +### Rule 4
  647 +
  648 +Each config key has one owner.
  649 +
  650 +Examples:
  651 +
  652 +- `search.query.knn_boost` belongs to search behavior config
  653 +- `services.embedding.backend` belongs to service implementation config
  654 +- `infrastructure.redis.password` belongs to env/secrets
  655 +
  656 +### Rule 5
  657 +
  658 +Every fallback must be either:
  659 +
  660 +- declared in schema defaults, or
  661 +- rejected at startup
  662 +
  663 +No hidden fallback in runtime logic.
  664 +
  665 +### Rule 6
  666 +
  667 +Every configuration asset must be visible in one of these places only:
  668 +
  669 +- config file
  670 +- env var
  671 +- generated runtime metadata
  672 +
  673 +Not inside parser code as an implicit constant.
  674 +
  675 +## 8. Recommended Naming Conventions
  676 +
  677 +Suggested conventions:
  678 +
  679 +- config keys use noun-based hierarchical names
  680 +- avoid mixing transport and implementation concepts in one field
  681 +- use `endpoint` for caller-facing addresses
  682 +- use `backend` for service-internal implementation choice
  683 +- use `enabled` only for true feature toggles
  684 +- use `default_*` only when a real selection happens at runtime
  685 +
  686 +Examples:
  687 +
  688 +- good: `services.rerank.endpoint`
  689 +- good: `services.rerank.backend`
  690 +- good: `tenants.default.index_languages`
  691 +- avoid: `service_url`, `base_url`, `provider`, `backend`, and script env all meaning slightly different things without a common model
  692 +
  693 +## 9. Highest-Priority Cleanup Items
  694 +
  695 +If the team wants the shortest path to improvement, start here:
  696 +
  697 +1. build one root `AppConfig`
  698 +2. make `services_config.py` stop reparsing YAML
  699 +3. declare rewrite dictionary path explicitly and fix the current mismatch
  700 +4. remove hardcoded `["en", "zh"]` fallbacks from query/indexer logic
  701 +5. replace `/admin/config` with an effective-config endpoint
  702 +6. retire `embeddings/config.py` and `reranker/config.py` as behavior sources
  703 +7. deprecate legacy tenant translation flags
  704 +
  705 +## 10. Expected Outcome
  706 +
  707 +After the redesign:
  708 +
  709 +- developers can answer “where does this setting come from?” in one step
  710 +- operators can see effective config without reading source code
  711 +- backend, indexer, translator, embedding, and reranker all share one model
  712 +- tenant behavior is explicit instead of partially implicit
  713 +- migration becomes safer because defaults and precedence are centralized
  714 +- adding a new provider/backend becomes configuration extension, not configuration archaeology
  715 +
  716 +## 11. Summary
  717 +
  718 +The current system has the right intent but not yet the right implementation shape.
  719 +
  720 +Today the main problems are:
  721 +
  722 +- duplicate config loaders
  723 +- inconsistent precedence
  724 +- duplicated defaults
  725 +- config hidden in runtime logic
  726 +- weak effective-config visibility
  727 +- leftover legacy concepts
  728 +
  729 +The recommended direction is:
  730 +
  731 +- one root typed config
  732 +- one loader pipeline
  733 +- explicit layered sources
  734 +- narrow env responsibility
  735 +- no hidden business fallbacks
  736 +- observable effective config
  737 +
  738 +That design is practical to implement incrementally in this repository and aligns well with the project's multi-tenant, multi-service, provider/backend-based architecture.
docs/搜索API对接指南-00-总览与快速开始.md 0 → 100644
@@ -0,0 +1,110 @@ @@ -0,0 +1,110 @@
  1 +# 搜索API对接指南-00-总览与快速开始
  2 +
  3 +本文档旨在为搜索服务的使用方提供完整的API对接指南,包括接口说明、请求参数、响应格式和使用示例。
  4 +拆分目录:
  5 +- `-01-搜索接口(POST /search/ 与响应)`
  6 +- `-02-搜索建议与即时搜索`
  7 +- `-03-获取文档(GET /search/{doc_id})`
  8 +- `-05-索引接口(Indexer)`
  9 +- `-06-管理接口(Admin)`
  10 +- `-07-微服务接口(Embedding/Reranker/Translation)`
  11 +- `-08-数据模型与字段速查`
  12 +- `-10-接口级压测脚本`
  13 +
  14 +## 快速开始
  15 +
  16 +### 1.1 基础信息
  17 +
  18 +- **Base URL**: `http://43.166.252.75:6002`
  19 +- **协议**: HTTP/HTTPS
  20 +- **数据格式**: JSON
  21 +- **字符编码**: UTF-8
  22 +- **请求方法**: POST(搜索接口)
  23 +
  24 +**重要提示**: `tenant_id` 通过 HTTP Header `X-Tenant-ID` 传递,不在请求体中。
  25 +
  26 +**环境与凭证**:MySQL、Redis、Elasticsearch 等外部服务的 AI 生产地址与凭证见 [QUICKSTART.md §1.6](./QUICKSTART.md#16-外部服务与-env含生产凭证)。
  27 +
  28 +### 1.2 最简单的搜索请求
  29 +
  30 +```bash
  31 +curl -X POST "http://43.166.252.75:6002/search/" \
  32 + -H "Content-Type: application/json" \
  33 + -H "X-Tenant-ID: 162" \
  34 + -d '{"query": "芭比娃娃"}'
  35 +```
  36 +
  37 +### 1.3 带过滤与分页的搜索
  38 +
  39 +```bash
  40 +curl -X POST "http://43.166.252.75:6002/search/" \
  41 + -H "Content-Type: application/json" \
  42 + -H "X-Tenant-ID: 162" \
  43 + -d '{
  44 + "query": "芭比娃娃",
  45 + "size": 5,
  46 + "from": 10,
  47 + "range_filters": {
  48 + "min_price": {
  49 + "gte": 50,
  50 + "lte": 200
  51 + },
  52 + "create_time": {
  53 + "gte": "2020-01-01T00:00:00Z"
  54 + }
  55 + },
  56 + "sort_by": "price",
  57 + "sort_order": "asc"
  58 + }'
  59 +```
  60 +
  61 +### 1.4 开启分面的搜索
  62 +
  63 +```bash
  64 +curl -X POST "http://43.166.252.75:6002/search/" \
  65 + -H "Content-Type: application/json" \
  66 + -H "X-Tenant-ID: 162" \
  67 + -d '{
  68 + "query": "芭比娃娃",
  69 + "facets": [
  70 + {"field": "category1_name", "size": 10, "type": "terms"},
  71 + {"field": "specifications.color", "size": 10, "type": "terms"},
  72 + {"field": "specifications.size", "size": 10, "type": "terms"}
  73 + ],
  74 + "min_score": 0.2
  75 + }'
  76 +```
  77 +
  78 +---
  79 +
  80 +## 接口概览
  81 +
  82 +| 接口 | HTTP Method | Endpoint | 说明 |
  83 +|------|------|------|------|
  84 +| 搜索 | POST | `/search/` | 执行搜索查询 |
  85 +| 搜索建议 | GET | `/search/suggestions` | 搜索建议(自动补全/热词,多语言) |
  86 +| 即时搜索 | GET | `/search/instant` | 即时搜索预留接口(当前返回 `501 Not Implemented`) |
  87 +| 获取文档 | GET | `/search/{doc_id}` | 获取单个文档 |
  88 +| 全量索引 | POST | `/indexer/reindex` | 全量索引接口(导入数据,不删除索引,仅推荐自测使用) |
  89 +| 增量索引 | POST | `/indexer/index` | 增量索引接口(指定SPU ID列表进行索引,支持自动检测删除和显式删除,仅推荐自测使用) |
  90 +| 查询文档 | POST | `/indexer/documents` | 查询SPU文档数据(不写入ES) |
  91 +| 构建ES文档(正式对接) | POST | `/indexer/build-docs` | 基于上游提供的 MySQL 行数据构建 ES doc,不写入 ES,供 Java 等调用后自行写入 |
  92 +| 构建ES文档(测试用) | POST | `/indexer/build-docs-from-db` | 仅在测试/调试时使用,根据 `tenant_id + spu_ids` 内部查库并构建 ES doc |
  93 +| 内容理解字段生成 | POST | `/indexer/enrich-content` | 根据商品标题批量生成 qanchors、semantic_attributes、tags,供微服务组合方式使用 |
  94 +| 索引健康检查 | GET | `/indexer/health` | 检查索引服务状态 |
  95 +| 健康检查 | GET | `/admin/health` | 服务健康检查 |
  96 +| 获取配置 | GET | `/admin/config` | 获取租户配置 |
  97 +| 索引统计 | GET | `/admin/stats` | 获取租户索引统计信息(需 tenant_id) |
  98 +
  99 +**微服务(独立端口或 Indexer 内,外部可直连)**:
  100 +
  101 +| 服务 | 端口 | 接口 | 说明 |
  102 +|------|------|------|------|
  103 +| 向量服务(文本) | 6005 | `POST /embed/text` | 文本向量化 |
  104 +| 向量服务(图片) | 6008 | `POST /embed/image` | 图片向量化 |
  105 +| 翻译服务 | 6006 | `POST /translate` | 文本翻译(支持 qwen-mt / llm / deepl / 本地模型) |
  106 +| 重排服务 | 6007 | `POST /rerank` | 检索结果重排 |
  107 +| 内容理解(Indexer 内) | 6004 | `POST /indexer/enrich-content` | 根据商品标题生成 qanchors、tags 等,供 indexer 微服务组合方式使用 |
  108 +
  109 +---
  110 +
docs/搜索API对接指南-01-搜索接口.md 0 → 100644
@@ -0,0 +1,903 @@ @@ -0,0 +1,903 @@
  1 +# 搜索API对接指南-01-搜索接口(POST /search/ 与响应)
  2 +
  3 +本篇以 `POST /search/` 为主线,包含:
  4 +- 请求参数:`3.2`、过滤器:`3.3`、分面:`3.4`、SKU筛选维度:`3.5`
  5 +- 响应格式:第 `4` 章(4.1~4.5)
  6 +- 常见场景示例:第 `8` 章(示例整体并入本篇,避免散落)
  7 +
  8 +## 搜索接口
  9 +
  10 +### 3.1 接口信息
  11 +
  12 +- **端点**: `POST /search/`
  13 +- **描述**: 执行文本搜索查询,支持多语言、过滤器和分面搜索
  14 +- **租户标识**:`tenant_id` 通过 HTTP 请求头 **`X-Tenant-ID`** 传递(推荐);也可通过 URL query 参数 **`tenant_id`** 传递。**不要放在请求体中。**
  15 +
  16 +**请求示例(推荐)**:
  17 +
  18 +```python
  19 +url = f"{base_url.rstrip('/')}/search/"
  20 +headers = {
  21 + "Content-Type": "application/json",
  22 + "X-Tenant-ID": "162", # 租户ID,必填
  23 +}
  24 +response = requests.post(url, headers=headers, json={"query": "芭比娃娃"})
  25 +```
  26 +
  27 +### 3.2 请求参数
  28 +
  29 +#### 完整请求体结构
  30 +
  31 +```json
  32 +{
  33 + "query": "string (required)",
  34 + "size": 10,
  35 + "from": 0,
  36 + "language": "zh",
  37 + "filters": {},
  38 + "range_filters": {},
  39 + "facets": [],
  40 + "sort_by": "string",
  41 + "sort_order": "desc",
  42 + "min_score": 0.0,
  43 + "sku_filter_dimension": ["string"],
  44 + "debug": false,
  45 + "enable_rerank": null,
  46 + "rerank_query_template": "{query}",
  47 + "rerank_doc_template": "{title}",
  48 + "user_id": "string",
  49 + "session_id": "string"
  50 +}
  51 +```
  52 +
  53 +#### 参数详细说明
  54 +
  55 +| 参数 | 类型 | 必填 | 默认值 | 说明 |
  56 +|------|------|------|--------|------|
  57 +| `query` | string | Y | - | 搜索查询字符串(统一文本检索策略) |
  58 +| `size` | integer | N | 10 | 返回结果数量(1-100) |
  59 +| `from` | integer | N | 0 | 分页偏移量(用于分页) |
  60 +| `language` | string | N | "zh" | 返回语言:`zh`(中文)或 `en`(英文)。后端会根据此参数选择对应的中英文字段返回 |
  61 +| `filters` | object | N | null | 精确匹配过滤器(见[过滤器详解](#33-过滤器详解)) |
  62 +| `range_filters` | object | N | null | 数值范围过滤器(见[过滤器详解](#33-过滤器详解)) |
  63 +| `facets` | array | N | null | 分面配置(见[分面配置](#34-分面配置)) |
  64 +| `sort_by` | string | N | null | 排序字段名。支持:`price`(价格)、`sales`(销量)、`create_time`(创建时间)、`update_time`(更新时间)。默认按相关性排序 |
  65 +| `sort_order` | string | N | "desc" | 排序方向:`asc`(升序)或 `desc`(降序)。注意:`price`+`asc`=价格从低到高,`price`+`desc`=价格从高到低(后端自动映射为min_price或max_price) |
  66 +| `min_score` | float | N | null | 最小相关性分数阈值 |
  67 +| `sku_filter_dimension` | array[string] | N | null | 子SKU筛选维度列表(见[SKU筛选维度](#35-sku筛选维度)) |
  68 +| `debug` | boolean | N | false | 是否返回调试信息 |
  69 +| `enable_rerank` | boolean/null | N | null | 是否开启重排(调用外部重排服务对 ES 结果进行二次排序)。不传/传 null 使用服务端 `rerank.enabled`(默认开启)。开启后会先对 ES TopN(`rerank_window`)重排,再按分页截取;若 `from+size>1000`,则不重排,直接按分页从 ES 返回 |
  70 +| `rerank_query_template` | string | N | null | 重排 query 模板(可选)。支持 `{query}` 占位符;不传则使用服务端配置 |
  71 +| `rerank_doc_template` | string | N | null | 重排 doc 模板(可选)。支持 `{title} {brief} {vendor} {description} {category_path}`;不传则使用服务端配置 |
  72 +| `user_id` | string | N | null | 用户ID(用于个性化,预留) |
  73 +| `session_id` | string | N | null | 会话ID(用于分析,预留) |
  74 +
  75 +### 3.3 过滤器详解
  76 +
  77 +#### 3.3.1 精确匹配过滤器 (filters)
  78 +
  79 +用于精确匹配或多值匹配。对于普通字段,数组表示 OR 逻辑(匹配任意一个值);对于 specifications 字段,按维度分组处理。**任意字段名加 `_all` 后缀**表示多值 AND 逻辑(必须同时匹配所有值)。
  80 +
  81 +**格式**:
  82 +
  83 +```json
  84 +{
  85 + "filters": {
  86 + "category_name": "手机", // 可以为单值 或者 数组 匹配数组中任意一个(OR)
  87 + "category1_name": "服装", // 可以为单值 或者 数组 匹配数组中任意一个(OR)
  88 + "category2_name": "男装", // 可以为单值 或者 数组 匹配数组中任意一个(OR)
  89 + "category3_name": "衬衫", // 可以为单值 或者 数组 匹配数组中任意一个(OR)
  90 + "vendor.zh.keyword": ["奇乐", "品牌A"], // 可以为单值 或者 数组 匹配数组中任意一个(OR)
  91 + "tags": "手机", // 可以为单值 或者 数组 匹配数组中任意一个(OR)
  92 + "tags_all": ["手机", "促销", "新品"], // *_all:多值为 AND,必须同时包含所有标签
  93 + "category1_name_all": ["服装", "男装"], // 同上,适用于任意可过滤字段
  94 + // specifications 嵌套过滤(特殊格式)
  95 + "specifications": {
  96 + "name": "color",
  97 + "value": "white"
  98 + }
  99 + }
  100 +}
  101 +```
  102 +
  103 +**支持的值类型**:
  104 +- 字符串:精确匹配
  105 +- 整数:精确匹配
  106 +- 布尔值:精确匹配
  107 +- 数组:匹配任意值(OR 逻辑);若字段名以 `_all` 结尾,则数组表示 AND 逻辑(必须同时匹配所有值)
  108 +- 对象:specifications 嵌套过滤(见下文)
  109 +
  110 +**`*_all` 语义(多值 AND)**:
  111 +- 任意过滤字段均可使用 `_all` 后缀,对应 ES 字段名为去掉 `_all` 后的名称。
  112 +- 例如:`tags_all: ["A", "B"]` 表示文档的 `tags` 必须**同时包含** A 和 B;`vendor.zh.keyword_all: ["奇乐", "品牌A"]` 表示同时匹配两个品牌(通常用于 keyword 多值场景)。
  113 +- `specifications_all`:传列表 `[{"name":"color","value":"white"},{"name":"size","value":"256GB"}]` 时,表示所有列出的规格条件都要满足(与 `specifications` 多维度时的 AND 一致;若同维度多值则要求文档同时满足多个值,一般用于嵌套多值场景)。
  114 +
  115 +**Specifications 嵌套过滤**:
  116 +
  117 +`specifications` 是嵌套字段,支持按规格名称和值进行过滤。
  118 +
  119 +**单个规格过滤**:
  120 +
  121 +```json
  122 +{
  123 + "filters": {
  124 + "specifications": {
  125 + "name": "color",
  126 + "value": "white"
  127 + }
  128 + }
  129 +}
  130 +```
  131 +
  132 +查询规格名称为"color"且值为"white"的商品。
  133 +
  134 +**多个规格过滤(按维度分组)**:
  135 +
  136 +```json
  137 +{
  138 + "filters": {
  139 + "specifications": [
  140 + {"name": "color", "value": "white"},
  141 + {"name": "size", "value": "256GB"}
  142 + ]
  143 + }
  144 +}
  145 +```
  146 +
  147 +查询同时满足所有规格的商品(color=white **且** size=256GB)。
  148 +
  149 +**相同维度的多个值(OR 逻辑)**:
  150 +
  151 +```json
  152 +{
  153 + "filters": {
  154 + "specifications": [
  155 + {"name": "size", "value": "3"},
  156 + {"name": "size", "value": "4"},
  157 + {"name": "size", "value": "5"},
  158 + {"name": "color", "value": "green"}
  159 + ]
  160 + }
  161 +}
  162 +```
  163 +
  164 +查询满足 (size=3 **或** size=4 **或** size=5) **且** color=green 的商品。
  165 +
  166 +**过滤逻辑说明**:
  167 +- **不同维度**(不同的 `name`)之间是 **AND** 关系(求交集)
  168 +- **相同维度**(相同的 `name`)的多个值之间是 **OR** 关系(求并集)
  169 +
  170 +**常用过滤字段**(详见[常用字段列表](./搜索API对接指南-08-数据模型与字段速查.md#93-常用字段列表)):
  171 +- `category_name`: 类目名称
  172 +- `category1_name`, `category2_name`, `category3_name`: 多级类目
  173 +- `category_id`: 类目ID
  174 +- `vendor.zh.keyword`, `vendor.en.keyword`: 供应商/品牌(使用keyword子字段)
  175 +- `tags`: 标签(keyword类型,支持数组)
  176 +- `option1_name`, `option2_name`, `option3_name`: 选项名称
  177 +- `specifications`: 规格过滤(嵌套字段,格式见上文)
  178 +- 以上任意字段均可加 `_all` 后缀表示多值 AND,如 `tags_all`、`category1_name_all`。
  179 +
  180 +#### 3.3.2 范围过滤器 (range_filters)
  181 +
  182 +用于数值字段的范围过滤。
  183 +
  184 +**格式**:
  185 +
  186 +```json
  187 +{
  188 + "range_filters": {
  189 + "min_price": {
  190 + "gte": 50, // 大于等于
  191 + "lte": 200 // 小于等于
  192 + },
  193 + "max_price": {
  194 + "gt": 100 // 大于
  195 + },
  196 + "create_time": {
  197 + "gte": "2024-01-01T00:00:00Z" // 日期时间字符串
  198 + }
  199 + }
  200 +}
  201 +```
  202 +
  203 +**支持的操作符**:
  204 +- `gte`: 大于等于 (>=)
  205 +- `gt`: 大于 (>)
  206 +- `lte`: 小于等于 (<=)
  207 +- `lt`: 小于 (<)
  208 +
  209 +**注意**: 至少需要指定一个操作符。
  210 +
  211 +**常用范围字段**(详见[常用字段列表](./搜索API对接指南-08-数据模型与字段速查.md#93-常用字段列表)):
  212 +- `min_price`: 最低价格
  213 +- `max_price`: 最高价格
  214 +- `compare_at_price`: 原价
  215 +- `create_time`: 创建时间
  216 +- `update_time`: 更新时间
  217 +
  218 +### 3.4 分面配置
  219 +
  220 +用于生成分面统计(分组聚合),常用于构建筛选器UI。
  221 +
  222 +#### 3.4.1 配置格式
  223 +
  224 +```json
  225 +{
  226 + "facets": [
  227 + {
  228 + "field": "category1_name",
  229 + "size": 15,
  230 + "type": "terms",
  231 + "disjunctive": false
  232 + },
  233 + {
  234 + "field": "brand_name",
  235 + "size": 10,
  236 + "type": "terms",
  237 + "disjunctive": true
  238 + },
  239 + {
  240 + "field": "specifications.color",
  241 + "size": 20,
  242 + "type": "terms",
  243 + "disjunctive": true
  244 + },
  245 + {
  246 + "field": "min_price",
  247 + "type": "range",
  248 + "ranges": [
  249 + {"key": "0-50", "to": 50},
  250 + {"key": "50-100", "from": 50, "to": 100},
  251 + {"key": "100-200", "from": 100, "to": 200},
  252 + {"key": "200+", "from": 200}
  253 + ]
  254 + }
  255 + ]
  256 +}
  257 +```
  258 +
  259 +#### 3.4.2 Facet 字段说明
  260 +
  261 +| 字段 | 类型 | 必填 | 默认值 | 说明 |
  262 +|------|------|------|--------|------|
  263 +| `field` | string | 是 | - | 分面字段名 |
  264 +| `size` | int | 否 | 10 | 返回的分面值数量(1-100) |
  265 +| `type` | string | 否 | "terms" | 分面类型:`terms`(词条聚合)或 `range`(范围聚合) |
  266 +| `disjunctive` | bool | 否 | false | 是否支持多选(disjunctive faceting)。启用后,选中该分面的过滤器时,仍会显示其他可选项 |
  267 +| `ranges` | array | 否 | null | 范围配置(仅 `type="range"` 时需要) |
  268 +
  269 +#### 3.4.3 disjunctive字段说明
  270 +
  271 +**重要特性**: `disjunctive` 字段控制分面的行为模式。启用后,选中该分面的过滤器时,仍会显示其他可选项
  272 +
  273 +**标准模式 (disjunctive: false)**:
  274 +- **行为**: 选中某个分面值后,该分面只显示选中的值
  275 +- **适用场景**: 层级类目、互斥选择
  276 +- **示例**: 类目下钻(玩具 > 娃娃 > 芭比)
  277 +
  278 +**Multi-Select 模式 (disjunctive: true)** ⭐:
  279 +- **行为**: 选中某个分面值后,该分面仍显示所有可选项
  280 +- **适用场景**: 颜色、品牌、尺码等可切换属性
  281 +- **示例**: 选择了"红色"后,仍能看到"蓝色"、"绿色"等选项
  282 +
  283 +**推荐配置**:
  284 +
  285 +| 分面类型 | disjunctive | 原因 |
  286 +|---------|-------------|------|
  287 +| 颜色 | `true` | 用户需要切换颜色 |
  288 +| 品牌 | `true` | 用户需要比较品牌 |
  289 +| 尺码 | `true` | 用户需要查看其他尺码 |
  290 +| 类目 | `false` | 层级下钻 |
  291 +| 价格区间 | `false` | 互斥选择 |
  292 +
  293 +#### 3.4.4 规格分面说明
  294 +
  295 +`specifications` 是嵌套字段,支持两种分面模式:
  296 +
  297 +**模式1:所有规格名称的分面**:
  298 +
  299 +```json
  300 +{
  301 + "facets": [
  302 + {
  303 + "field": "specifications",
  304 + "size": 10,
  305 + "type": "terms"
  306 + }
  307 + ]
  308 +}
  309 +```
  310 +
  311 +返回所有规格名称(name)及其对应的值(value)列表。每个 name 会生成一个独立的分面结果。
  312 +
  313 +**模式2:指定规格名称的分面**:
  314 +
  315 +```json
  316 +{
  317 + "facets": [
  318 + {
  319 + "field": "specifications.color",
  320 + "size": 20,
  321 + "type": "terms",
  322 + "disjunctive": true
  323 + },
  324 + {
  325 + "field": "specifications.size",
  326 + "size": 15,
  327 + "type": "terms",
  328 + "disjunctive": true
  329 + }
  330 + ]
  331 +}
  332 +```
  333 +
  334 +只返回指定规格名称的值列表。格式:`specifications.{name}`,其中 `{name}` 是规格名称(如"color"、"size"、"material")。
  335 +
  336 +**返回格式示例**:
  337 +
  338 +```json
  339 +{
  340 + "facets": [
  341 + {
  342 + "field": "specifications.color",
  343 + "label": "color",
  344 + "type": "terms",
  345 + "values": [
  346 + {"value": "white", "count": 50, "selected": true}, // ✓ selected 字段由后端标记
  347 + {"value": "black", "count": 30, "selected": false},
  348 + {"value": "red", "count": 20, "selected": false}
  349 + ]
  350 + },
  351 + {
  352 + "field": "specifications.size",
  353 + "label": "size",
  354 + "type": "terms",
  355 + "values": [
  356 + {"value": "256GB", "count": 40, "selected": false},
  357 + {"value": "512GB", "count": 20, "selected": false}
  358 + ]
  359 + }
  360 + ]
  361 +}
  362 +```
  363 +
  364 +### 3.5 SKU筛选维度
  365 +
  366 +**功能说明**:
  367 +`sku_filter_dimension` 用于控制搜索列表页中 **每个 SPU 下方可切换的子款式(子 SKU)维度**,为字符串列表。
  368 +在店铺的 **主题装修配置** 中,商家可以为店铺设置一个或多个子款式筛选维度(例如 `color`、`size`),前端列表页会在每个 SPU 下展示这些维度对应的子 SKU 列表,用户可以通过点击不同维度值(如不同颜色)来切换展示的子款式。
  369 +当指定 `sku_filter_dimension` 后,后端会根据店铺的这项配置,从所有 SKU 中筛选出这些维度组合对应的子 SKU 数据:系统会按指定维度**组合**对 SKU 进行分组,每个维度组合只返回第一个 SKU(从简实现,选择该组合下的第一款),其余不在这些维度组合中的子 SKU 将不返回。
  370 +
  371 +**支持的维度值**:
  372 +1. **直接选项字段**: `option1`、`option2`、`option3`
  373 + - 直接使用对应的 `option1_value`、`option2_value`、`option3_value` 字段进行分组
  374 +
  375 +2. **规格/选项名称**: 通过 `option1_name`、`option2_name`、`option3_name` 匹配
  376 + - 例如:如果 `option1_name` 为 `"color"`,则可以使用 `sku_filter_dimension: ["color"]` 来按颜色分组
  377 +
  378 +**示例**:
  379 +
  380 +**按颜色筛选(假设 option1_name = "color")**:
  381 +
  382 +```json
  383 +{
  384 + "query": "芭比娃娃",
  385 + "sku_filter_dimension": ["color"]
  386 +}
  387 +```
  388 +
  389 +**按选项1筛选**:
  390 +
  391 +```json
  392 +{
  393 + "query": "芭比娃娃",
  394 + "sku_filter_dimension": ["option1"]
  395 +}
  396 +```
  397 +
  398 +**按颜色 + 尺寸组合筛选(假设 option1_name = "color", option2_name = "size")**:
  399 +
  400 +```json
  401 +{
  402 + "query": "芭比娃娃",
  403 + "sku_filter_dimension": ["color", "size"]
  404 +}
  405 +```
  406 +
  407 +## 响应格式说明
  408 +
  409 +### 4.1 标准响应结构
  410 +
  411 +```json
  412 +{
  413 + "results": [
  414 + {
  415 + "spu_id": "12345",
  416 + "title": "芭比时尚娃娃",
  417 + "brief": "高品质芭比娃娃",
  418 + "description": "详细描述...",
  419 + "vendor": "美泰",
  420 + "category": "玩具",
  421 + "category_path": "玩具/娃娃/时尚",
  422 + "category_name": "时尚",
  423 + "category_id": "cat_001",
  424 + "category_level": 3,
  425 + "category1_name": "玩具",
  426 + "category2_name": "娃娃",
  427 + "category3_name": "时尚",
  428 + "tags": ["娃娃", "玩具", "女孩"],
  429 + "price": 89.99,
  430 + "compare_at_price": 129.99,
  431 + "currency": "USD",
  432 + "image_url": "https://example.com/image.jpg",
  433 + "in_stock": true,
  434 + "sku_prices": [89.99, 99.99, 109.99],
  435 + "sku_weights": [100, 150, 200],
  436 + "sku_weight_units": ["g", "g", "g"],
  437 + "total_inventory": 500,
  438 + "option1_name": "color",
  439 + "option2_name": "size",
  440 + "option3_name": null,
  441 + "specifications": [
  442 + {"sku_id": "sku_001", "name": "color", "value": "pink"},
  443 + {"sku_id": "sku_001", "name": "size", "value": "standard"}
  444 + ],
  445 + "skus": [
  446 + {
  447 + "sku_id": "67890",
  448 + "price": 89.99,
  449 + "compare_at_price": 129.99,
  450 + "sku": "BARBIE-001",
  451 + "stock": 100,
  452 + "weight": 0.1,
  453 + "weight_unit": "kg",
  454 + "option1_value": "pink",
  455 + "option2_value": "standard",
  456 + "option3_value": null,
  457 + "image_src": "https://example.com/sku1.jpg"
  458 + }
  459 + ],
  460 + "relevance_score": 8.5
  461 + }
  462 + ],
  463 + "total": 118,
  464 + "max_score": 8.5,
  465 + "facets": [
  466 + {
  467 + "field": "category1_name",
  468 + "label": "category1_name",
  469 + "type": "terms",
  470 + "values": [
  471 + {
  472 + "value": "玩具",
  473 + "label": "玩具",
  474 + "count": 85,
  475 + "selected": false
  476 + }
  477 + ]
  478 + },
  479 + {
  480 + "field": "specifications.color",
  481 + "label": "color",
  482 + "type": "terms",
  483 + "values": [
  484 + {
  485 + "value": "pink",
  486 + "label": "pink",
  487 + "count": 30,
  488 + "selected": false
  489 + }
  490 + ]
  491 + }
  492 + ],
  493 + "query_info": {
  494 + "original_query": "芭比娃娃",
  495 + "query_normalized": "芭比娃娃",
  496 + "rewritten_query": "芭比娃娃",
  497 + "detected_language": "zh",
  498 + "translations": {
  499 + "en": "barbie doll"
  500 + },
  501 + "domain": "default"
  502 + },
  503 + "suggestions": [],
  504 + "related_searches": [],
  505 + "took_ms": 45,
  506 + "performance_info": null,
  507 + "debug_info": null
  508 +}
  509 +```
  510 +
  511 +### 4.2 响应字段说明
  512 +
  513 +| 字段 | 类型 | 说明 |
  514 +|------|------|------|
  515 +| `results` | array | 搜索结果列表(SpuResult对象数组) |
  516 +| `results[].spu_id` | string | SPU ID |
  517 +| `results[].title` | string | 商品标题 |
  518 +| `results[].price` | float | 价格(min_price) |
  519 +| `results[].skus` | array | SKU列表(如果指定了`sku_filter_dimension`,则按维度过滤后的SKU) |
  520 +| `results[].relevance_score` | float | 相关性分数 |
  521 +| `total` | integer | 匹配的总文档数 |
  522 +| `max_score` | float | 最高相关性分数 |
  523 +| `facets` | array | 分面统计结果 |
  524 +| `query_info` | object | query处理信息 |
  525 +| `took_ms` | integer | 搜索耗时(毫秒) |
  526 +| `debug_info` | object/null | 调试信息,仅当请求传 `debug=true` 时返回 |
  527 +
  528 +#### 4.2.1 query_info 说明
  529 +
  530 +`query_info` 包含本次搜索的查询解析与处理结果:
  531 +
  532 +| 子字段 | 类型 | 说明 |
  533 +|--------|------|------|
  534 +| `original_query` | string | 用户原始查询 |
  535 +| `query_normalized` | string | 归一化后的查询(去空白、大小写等预处理,用于后续解析与改写) |
  536 +| `rewritten_query` | string | 重写后的查询(同义词/词典扩展等) |
  537 +| `detected_language` | string | 检测到的查询语言(如 `zh`、`en`) |
  538 +| `translations` | object | 翻译结果,键为语言代码,值为翻译文本 |
  539 +| `domain` | string | 查询域(如 `default`、`title`、`brand` 等) |
  540 +
  541 +#### 4.2.2 debug_info 说明
  542 +
  543 +`debug_info` 主要用于检索效果评估、融合打分分析与 bad case 排查。
  544 +
  545 +`debug_info.query_analysis` 常见字段:
  546 +
  547 +| 子字段 | 类型 | 说明 |
  548 +|--------|------|------|
  549 +| `original_query` | string | 原始查询 |
  550 +| `query_normalized` | string | 归一化后的查询 |
  551 +| `rewritten_query` | string | 重写后的查询 |
  552 +| `detected_language` | string | 检测到的语言 |
  553 +| `translations` | object | 翻译结果 |
  554 +| `query_text_by_lang` | object | 实际参与检索的多语言 query 文本 |
  555 +| `search_langs` | array[string] | 实际参与检索的语言列表 |
  556 +| `supplemental_search_langs` | array[string] | 因 mixed query 补入的附加语言列表 |
  557 +| `has_vector` | boolean | 是否生成了向量 |
  558 +
  559 +`debug_info.per_result[]` 常见字段:
  560 +
  561 +| 子字段 | 类型 | 说明 |
  562 +|--------|------|------|
  563 +| `spu_id` | string | 结果 SPU ID |
  564 +| `es_score` | float | ES 原始 `_score` |
  565 +| `rerank_score` | float | 重排分数 |
  566 +| `text_score` | float | 文本相关性大分(由 `base_query` / `base_query_trans_*` / `fallback_original_query_*` 聚合而来) |
  567 +| `text_source_score` | float | `base_query` 分数 |
  568 +| `text_translation_score` | float | `base_query_trans_*` 里的最大分数 |
  569 +| `text_fallback_score` | float | `fallback_original_query_*` 里的最大分数 |
  570 +| `text_primary_score` | float | 文本大分中的主证据部分 |
  571 +| `text_support_score` | float | 文本大分中的辅助证据部分 |
  572 +| `knn_score` | float | `knn_query` 分数 |
  573 +| `fused_score` | float | 最终融合分数 |
  574 +| `matched_queries` | object/array | ES named queries 命中详情 |
  575 +
  576 +### 4.3 SpuResult字段说明
  577 +
  578 +| 字段 | 类型 | 说明 |
  579 +|------|------|------|
  580 +| `spu_id` | string | SPU ID |
  581 +| `title` | string | 商品标题(根据language参数自动选择 `title.zh` 或 `title.en`) |
  582 +| `brief` | string | 商品短描述(根据language参数自动选择) |
  583 +| `description` | string | 商品详细描述(根据language参数自动选择) |
  584 +| `vendor` | string | 供应商/品牌(根据language参数自动选择) |
  585 +| `category` | string | 类目(兼容字段,等同于category_name) |
  586 +| `category_path` | string | 类目路径(多级,用于面包屑,根据language参数自动选择) |
  587 +| `category_name` | string | 类目名称(展示用,根据language参数自动选择) |
  588 +| `category_id` | string | 类目ID |
  589 +| `category_level` | integer | 类目层级(1/2/3) |
  590 +| `category1_name` | string | 一级类目名称 |
  591 +| `category2_name` | string | 二级类目名称 |
  592 +| `category3_name` | string | 三级类目名称 |
  593 +| `tags` | array[string] | 标签列表 |
  594 +| `price` | float | 价格(min_price) |
  595 +| `compare_at_price` | float | 原价 |
  596 +| `currency` | string | 货币单位(默认USD) |
  597 +| `image_url` | string | 主图URL |
  598 +| `in_stock` | boolean | 是否有库存(任意SKU有库存即为true) |
  599 +| `sku_prices` | array[float] | 所有SKU价格列表 |
  600 +| `sku_weights` | array[integer] | 所有SKU重量列表 |
  601 +| `sku_weight_units` | array[string] | 所有SKU重量单位列表 |
  602 +| `total_inventory` | integer | 总库存 |
  603 +| `sales` | integer | 销量(展示销量) |
  604 +| `option1_name` | string | 选项1名称(如"color") |
  605 +| `option2_name` | string | 选项2名称(如"size") |
  606 +| `option3_name` | string | 选项3名称 |
  607 +| `specifications` | array[object] | 规格列表(与ES specifications字段对应) |
  608 +| `skus` | array | SKU 列表 |
  609 +| `relevance_score` | float | 相关性分数(默认为 ES 原始分数;当开启 AI 搜索时为融合后的最终分数) |
  610 +
  611 +### 4.4 SkuResult字段说明
  612 +
  613 +| 字段 | 类型 | 说明 |
  614 +|------|------|------|
  615 +| `sku_id` | string | SKU ID |
  616 +| `price` | float | 价格 |
  617 +| `compare_at_price` | float | 原价 |
  618 +| `sku` | string | SKU编码(sku_code) |
  619 +| `stock` | integer | 库存数量 |
  620 +| `weight` | float | 重量 |
  621 +| `weight_unit` | string | 重量单位 |
  622 +| `option1_value` | string | 选项1取值(如color值) |
  623 +| `option2_value` | string | 选项2取值(如size值) |
  624 +| `option3_value` | string | 选项3取值 |
  625 +| `image_src` | string | SKU图片地址 |
  626 +
  627 +### 4.5 多语言字段说明
  628 +
  629 +- `title`, `brief`, `description`, `vendor`, `category_path`, `category_name` 会根据请求的 `language` 参数自动选择对应的中英文字段
  630 +- `language="zh"`: 优先返回 `*_zh` 字段,如果为空则回退到 `*_en` 字段
  631 +- `language="en"`: 优先返回 `*_en` 字段,如果为空则回退到 `*_zh` 字段
  632 +
  633 +---
  634 +
  635 +## 8. 常见场景示例
  636 +
  637 +以下示例仅展示**请求体**(body);实际调用时请加上请求头 `X-Tenant-ID: <租户ID>`(或 URL 参数 `tenant_id`),参见 [3.1 接口信息](#31-接口信息)。
  638 +
  639 +### 8.1 基础搜索与排序
  640 +
  641 +**按价格从低到高排序**:
  642 +
  643 +```json
  644 +{
  645 + "query": "玩具",
  646 + "size": 20,
  647 + "from": 0,
  648 + "sort_by": "price",
  649 + "sort_order": "asc"
  650 +}
  651 +```
  652 +
  653 +**按价格从高到低排序**:
  654 +
  655 +```json
  656 +{
  657 + "query": "玩具",
  658 + "size": 20,
  659 + "from": 0,
  660 + "sort_by": "price",
  661 + "sort_order": "desc"
  662 +}
  663 +```
  664 +
  665 +**按销量从高到低排序**:
  666 +
  667 +```json
  668 +{
  669 + "query": "玩具",
  670 + "size": 20,
  671 + "from": 0,
  672 + "sort_by": "sales",
  673 + "sort_order": "desc"
  674 +}
  675 +```
  676 +
  677 +**按默认(相关性)排序**:
  678 +
  679 +```json
  680 +{
  681 + "query": "玩具",
  682 + "size": 20,
  683 + "from": 0
  684 +}
  685 +```
  686 +
  687 +### 8.2 过滤搜索
  688 +
  689 +**需求**: 搜索"玩具",筛选类目为"益智玩具",价格在50-200之间
  690 +
  691 +```json
  692 +{
  693 + "query": "玩具",
  694 + "size": 20,
  695 + "language": "zh",
  696 + "filters": {
  697 + "category_name": "益智玩具"
  698 + },
  699 + "range_filters": {
  700 + "min_price": {
  701 + "gte": 50,
  702 + "lte": 200
  703 + }
  704 + }
  705 +}
  706 +```
  707 +
  708 +**需求**: 搜索"手机",筛选多个品牌,价格范围
  709 +
  710 +```json
  711 +{
  712 + "query": "手机",
  713 + "size": 20,
  714 + "language": "zh",
  715 + "filters": {
  716 + "vendor.zh.keyword": ["品牌A", "品牌B"]
  717 + },
  718 + "range_filters": {
  719 + "min_price": {
  720 + "gte": 50,
  721 + "lte": 200
  722 + }
  723 + }
  724 +}
  725 +```
  726 +
  727 +### 8.3 分面搜索
  728 +
  729 +**需求**: 搜索"玩具",获取类目和规格的分面统计,用于构建筛选器
  730 +
  731 +```json
  732 +{
  733 + "query": "玩具",
  734 + "size": 20,
  735 + "language": "zh",
  736 + "facets": [
  737 + {"field": "category1_name", "size": 15, "type": "terms"},
  738 + {"field": "category2_name", "size": 10, "type": "terms"},
  739 + {"field": "specifications", "size": 10, "type": "terms"}
  740 + ]
  741 +}
  742 +```
  743 +
  744 +**需求**: 搜索"手机",获取价格区间和规格的分面统计
  745 +
  746 +```json
  747 +{
  748 + "query": "手机",
  749 + "size": 20,
  750 + "language": "zh",
  751 + "facets": [
  752 + {
  753 + "field": "min_price",
  754 + "type": "range",
  755 + "ranges": [
  756 + {"key": "0-50", "to": 50},
  757 + {"key": "50-100", "from": 50, "to": 100},
  758 + {"key": "100-200", "from": 100, "to": 200},
  759 + {"key": "200+", "from": 200}
  760 + ]
  761 + },
  762 + {
  763 + "field": "specifications",
  764 + "size": 10,
  765 + "type": "terms"
  766 + }
  767 + ]
  768 +}
  769 +```
  770 +
  771 +### 8.4 规格过滤与分面
  772 +
  773 +**需求**: 搜索"手机",筛选color为"white"的商品
  774 +
  775 +```json
  776 +{
  777 + "query": "手机",
  778 + "size": 20,
  779 + "language": "zh",
  780 + "filters": {
  781 + "specifications": {
  782 + "name": "color",
  783 + "value": "white"
  784 + }
  785 + }
  786 +}
  787 +```
  788 +
  789 +**需求**: 搜索"手机",筛选color为"white"且size为"256GB"的商品
  790 +
  791 +```json
  792 +{
  793 + "query": "手机",
  794 + "size": 20,
  795 + "language": "zh",
  796 + "filters": {
  797 + "specifications": [
  798 + {"name": "color", "value": "white"},
  799 + {"name": "size", "value": "256GB"}
  800 + ]
  801 + }
  802 +}
  803 +```
  804 +
  805 +**需求**: 搜索"手机",筛选size为"3"、"4"或"5",且color为"green"的商品
  806 +
  807 +```json
  808 +{
  809 + "query": "手机",
  810 + "size": 20,
  811 + "language": "zh",
  812 + "filters": {
  813 + "specifications": [
  814 + {"name": "size", "value": "3"},
  815 + {"name": "size", "value": "4"},
  816 + {"name": "size", "value": "5"},
  817 + {"name": "color", "value": "green"}
  818 + ]
  819 + }
  820 +}
  821 +```
  822 +
  823 +**需求**: 搜索"手机",获取所有规格的分面统计
  824 +
  825 +```json
  826 +{
  827 + "query": "手机",
  828 + "size": 20,
  829 + "language": "zh",
  830 + "facets": [
  831 + {"field": "specifications", "size": 10, "type": "terms"}
  832 + ]
  833 +}
  834 +```
  835 +
  836 +**需求**: 只获取"color"和"size"规格的分面统计
  837 +
  838 +```json
  839 +{
  840 + "query": "手机",
  841 + "size": 20,
  842 + "language": "zh",
  843 + "facets": [
  844 + {"field": "specifications.color", "size": 20, "type": "terms"},
  845 + {"field": "specifications.size", "size": 15, "type": "terms"}
  846 + ]
  847 +}
  848 +```
  849 +
  850 +**需求**: 搜索"手机",筛选类目和规格,并获取对应的分面统计
  851 +
  852 +```json
  853 +{
  854 + "query": "手机",
  855 + "size": 20,
  856 + "language": "zh",
  857 + "filters": {
  858 + "category_name": "手机",
  859 + "specifications": {
  860 + "name": "color",
  861 + "value": "white"
  862 + }
  863 + },
  864 + "facets": [
  865 + {"field": "category1_name", "size": 15, "type": "terms"},
  866 + {"field": "category2_name", "size": 10, "type": "terms"},
  867 + {"field": "specifications.color", "size": 20, "type": "terms"},
  868 + {"field": "specifications.size", "size": 15, "type": "terms"}
  869 + ]
  870 +}
  871 +```
  872 +
  873 +### 8.5 SKU筛选
  874 +
  875 +**需求**: 搜索"芭比娃娃",每个SPU下按颜色筛选,每种颜色只显示一个SKU
  876 +
  877 +```json
  878 +{
  879 + "query": "芭比娃娃",
  880 + "size": 20,
  881 + "sku_filter_dimension": ["color"]
  882 +}
  883 +```
  884 +
  885 +**说明**:
  886 +- 如果 `option1_name` 为 `"color"`,则使用 `sku_filter_dimension: ["color"]` 可以按颜色分组
  887 +- 每个SPU下,每种颜色只会返回第一个SKU
  888 +- 如果维度不匹配,返回所有SKU(不进行过滤)
  889 +
  890 +### 8.6 分页查询
  891 +
  892 +**需求**: 获取第2页结果(每页20条)
  893 +
  894 +```json
  895 +{
  896 + "query": "手机",
  897 + "size": 20,
  898 + "from": 20
  899 +}
  900 +```
  901 +
  902 +---
  903 +
docs/搜索API对接指南-02-搜索建议与即时搜索.md 0 → 100644
@@ -0,0 +1,81 @@ @@ -0,0 +1,81 @@
  1 +# 搜索API对接指南-02-搜索建议与即时搜索
  2 +
  3 +本篇面向前端联想词/搜索框团队,独立阅读 `GET /search/suggestions` 与 `GET /search/instant`。
  4 +
  5 +## 搜索接口
  6 +
  7 +### 3.7 搜索建议接口
  8 +
  9 +- **端点**: `GET /search/suggestions`
  10 +- **描述**: 返回搜索建议(自动补全/热词),支持多语言。
  11 +
  12 +#### 查询参数
  13 +
  14 +| 参数 | 类型 | 必填 | 默认值 | 描述 |
  15 +|------|------|------|--------|------|
  16 +| `q` | string | Y | - | 查询字符串(至少 1 个字符) |
  17 +| `size` | integer | N | 10 | 返回建议数量(1-50) |
  18 +| `language` | string | N | `en` | 请求语言,如 `zh` / `en` / `ar` / `ru`,用于路由到对应语种 suggestion 索引 |
  19 +| `debug` | bool | N | `false` | 是否开启调试(目前主要用于排查 suggestion 排序与语言解析) |
  20 +
  21 +> **租户标识**:同 [-01-搜索接口](./搜索API对接指南-01-搜索接口.md#31-接口信息),通过请求头 `X-Tenant-ID` 或 query 参数 `tenant_id` 传递。
  22 +
  23 +#### 响应示例
  24 +
  25 +```json
  26 +{
  27 + "query": "iph",
  28 + "language": "en",
  29 + "resolved_language": "en",
  30 + "suggestions": [
  31 + {
  32 + "text": "iphone 15",
  33 + "lang": "en",
  34 + "score": 12.37,
  35 + "rank_score": 5.1,
  36 + "sources": ["query_log", "qanchor"],
  37 + "lang_source": "log_field",
  38 + "lang_confidence": 1.0,
  39 + "lang_conflict": false
  40 + }
  41 + ],
  42 + "took_ms": 12
  43 +}
  44 +```
  45 +
  46 +#### 请求示例
  47 +
  48 +```bash
  49 +curl "http://localhost:6002/search/suggestions?q=芭&size=5&language=zh" \
  50 + -H "X-Tenant-ID: 162"
  51 +```
  52 +
  53 +### 3.8 即时搜索接口
  54 +
  55 +> ⚠️ 当前版本未开放该能力。接口会明确返回 `501 Not Implemented`,避免误用未完成实现。
  56 +
  57 +- **端点**: `GET /search/instant`
  58 +- **描述**: 即时搜索预留端点,后续会在独立实现完成后开放。
  59 +
  60 +#### 查询参数
  61 +
  62 +| 参数 | 类型 | 必填 | 默认值 | 描述 |
  63 +|------|------|------|--------|------|
  64 +| `q` | string | Y | - | 搜索查询(至少 2 个字符) |
  65 +| `size` | integer | N | 5 | 返回结果数量(1-20) |
  66 +
  67 +#### 请求示例
  68 +
  69 +```bash
  70 +curl "http://localhost:6002/search/instant?q=玩具&size=5"
  71 +```
  72 +
  73 +#### 当前响应
  74 +
  75 +```json
  76 +{
  77 + "error": "/search/instant is not implemented yet. Use POST /search/ for production traffic.",
  78 + "status_code": 501
  79 +}
  80 +```
  81 +
docs/搜索API对接指南-03-获取文档.md 0 → 100644
@@ -0,0 +1,40 @@ @@ -0,0 +1,40 @@
  1 +# 搜索API对接指南-03-获取文档(GET /search/{doc_id})
  2 +
  3 +用于点击结果后的详情页回源,或排查某个文档在检索侧的字段情况。
  4 +
  5 +## 搜索接口
  6 +
  7 +### 3.9 获取单个文档
  8 +
  9 +- **端点**: `GET /search/{doc_id}`
  10 +- **描述**: 根据文档 ID 获取单个商品详情,用于点击结果后的详情页或排查问题。
  11 +- **租户标识**:同 [-01-搜索接口](./搜索API对接指南-01-搜索接口.md#31-接口信息),通过请求头 `X-Tenant-ID` 或 query 参数 `tenant_id` 传递。
  12 +
  13 +#### 路径参数
  14 +
  15 +| 参数 | 类型 | 描述 |
  16 +|------|------|------|
  17 +| `doc_id` | string | 商品或文档 ID |
  18 +
  19 +#### 响应示例
  20 +
  21 +```json
  22 +{
  23 + "id": "12345",
  24 + "source": {
  25 + "title": {
  26 + "zh": "芭比时尚娃娃"
  27 + },
  28 + "min_price": 89.99,
  29 + "category1_name": "玩具"
  30 + }
  31 +}
  32 +```
  33 +
  34 +#### 请求示例
  35 +
  36 +```bash
  37 +curl "http://localhost:6002/search/12345" -H "X-Tenant-ID: 162"
  38 +# 或使用 query 参数:curl "http://localhost:6002/search/12345?tenant_id=162"
  39 +```
  40 +
docs/搜索API对接指南-05-索引接口(Indexer).md 0 → 100644
@@ -0,0 +1,767 @@ @@ -0,0 +1,767 @@
  1 +# 搜索API对接指南-05-索引接口(Indexer)
  2 +
  3 +本篇覆盖数据同步/索引构建相关的所有接口(原文第 5 章),用于 `external indexer` 和 `Indexer 服务` 的对接。
  4 +
  5 +## 索引接口
  6 +
  7 +本节内容与 `api/routes/indexer.py` 中的索引相关服务一致,包含以下接口:
  8 +
  9 +| 接口 | 方法 | 路径 | 说明 |
  10 +|------|------|------|------|
  11 +| 全量重建索引 | POST | `/indexer/reindex` | 将指定租户所有 SPU 导入 ES(不删现有索引) |
  12 +| 增量索引 | POST | `/indexer/index` | 按 SPU ID 列表索引/删除,支持自动检测删除与显式删除 |
  13 +| 查询文档 | POST | `/indexer/documents` | 按 SPU ID 列表查询 ES 文档,不写入 ES |
  14 +| 构建 ES 文档(正式) | POST | `/indexer/build-docs` | 由上游提供 MySQL 行数据,返回 ES-ready 文档,不写 ES |
  15 +| 构建 ES 文档(测试) | POST | `/indexer/build-docs-from-db` | 由本服务查库并构建文档,仅测试/调试用 |
  16 +| 内容理解字段生成 | POST | `/indexer/enrich-content` | 根据商品标题批量生成 qanchors、semantic_attributes、tags(供微服务组合方式使用) |
  17 +| 索引健康检查 | GET | `/indexer/health` | 检查索引服务与数据库连接状态 |
  18 +
  19 +#### 5.0 支撑外部 indexer 的三种方式
  20 +
  21 +本服务对**外部 indexer 程序**(如 Java 索引系统)提供三种对接方式,可按需选择:
  22 +
  23 +| 方式 | 说明 | 适用场景 |
  24 +|------|------|----------|
  25 +| **1)doc 填充接口** | 调用 `POST /indexer/build-docs` 或 `POST /indexer/build-docs-from-db`,由本服务基于 MySQL 行数据构建完整 ES 文档(含多语言、向量、规格等),**不写入 ES**,由调用方自行写入。 | 希望一站式拿到 ES-ready doc,由己方控制写 ES 的时机与索引名。 |
  26 +| **2)微服务组合** | 单独调用**翻译**、**向量化**、**内容理解字段生成**等接口,由 indexer 程序自己组装 doc 并写入 ES。翻译与向量化为独立微服务(见第 7 节);内容理解为 Indexer 服务内接口 `POST /indexer/enrich-content`。 | 需要灵活编排、或希望将 LLM/向量等耗时步骤与主链路解耦(如异步补齐 qanchors/tags)。 |
  27 +| **3)本服务直接写 ES** | 调用全量索引 `POST /indexer/reindex`、增量索引 `POST /indexer/index`(指定 SPU ID 列表),由本服务从 MySQL 拉数并直接写入 ES。 | 自建运维、联调或不需要由 Java 写 ES 的场景。 |
  28 +
  29 +- **方式 1** 与 **方式 2** 下,ES 的写入方均为外部 indexer(或 Java),职责清晰。
  30 +- **方式 3** 下,本服务同时负责读库、构建 doc 与写 ES。
  31 +
  32 +### 5.1 为租户创建索引
  33 +
  34 +为租户创建索引需要两个步骤:
  35 +
  36 +1. **创建索引结构**(可选,仅在需要更新 mapping 或在新环境首次创建时执行)
  37 + - 使用脚本创建 ES 索引结构(基于 `mappings/search_products.json`)
  38 + - 如果索引已存在,会提示用户确认(会删除现有数据)
  39 +
  40 +2. **导入数据**(必需)
  41 + - 使用全量索引接口 `/indexer/reindex` 导入数据
  42 +
  43 +**创建索引结构(支持多环境 namespace)**:
  44 +
  45 +```bash
  46 +# 以 UAT 环境为例:
  47 +# 1. 准备 UAT 环境的 .env(包含 UAT 的 ES_HOST/DB_HOST 等)
  48 +# 2. 设置环境前缀(也可以直接在 .env 中配置):
  49 +export RUNTIME_ENV=uat
  50 +export ES_INDEX_NAMESPACE=uat_
  51 +
  52 +# 3. 为 tenant_id=170 创建索引结构
  53 +./scripts/create_tenant_index.sh 170
  54 +```
  55 +
  56 +脚本会自动从项目根目录的 `.env` 文件加载 ES 配置,并根据 `ES_INDEX_NAMESPACE` 创建:
  57 +
  58 +- prod 环境(ES_INDEX_NAMESPACE 为空):`search_products_tenant_170`
  59 +- UAT 环境(ES_INDEX_NAMESPACE=uat_):`uat_search_products_tenant_170`
  60 +
  61 +**注意事项**:
  62 +- ⚠️ 如果索引已存在,脚本会提示确认,确认后会删除现有数据
  63 +- 创建索引后,**必须**调用 `/indexer/reindex` 导入数据
  64 +- 如果只是更新数据而不需要修改索引结构,直接使用 `/indexer/reindex` 即可
  65 +
  66 +---
  67 +
  68 +### 5.2 全量索引接口
  69 +
  70 +- **端点**: `POST /indexer/reindex`
  71 +- **描述**: 全量索引,将指定租户的所有SPU数据导入到ES索引(不会删除现有索引)。**推荐仅用于自测/运维场景**;生产环境下更推荐由 Java 等上游控制调度与写 ES。
  72 +
  73 +#### 请求参数
  74 +
  75 +```json
  76 +{
  77 + "tenant_id": "162",
  78 + "batch_size": 500
  79 +}
  80 +```
  81 +
  82 +| 参数 | 类型 | 必填 | 默认值 | 说明 |
  83 +|------|------|------|--------|------|
  84 +| `tenant_id` | string | Y | - | 租户ID |
  85 +| `batch_size` | integer | N | 500 | 批量导入大小 |
  86 +
  87 +#### 响应格式
  88 +
  89 +**成功响应(200 OK)**(示例,实际 `index_name` 会带上 tenant 和环境前缀):
  90 +
  91 +```json
  92 +{
  93 + "success": true,
  94 + "total": 1000,
  95 + "indexed": 1000,
  96 + "failed": 0,
  97 + "elapsed_time": 12.34,
  98 + "index_name": "search_products_tenant_162",
  99 + "tenant_id": "162"
  100 +}
  101 +```
  102 +
  103 +**错误响应**:
  104 +- `400 Bad Request`: 参数错误
  105 +- `503 Service Unavailable`: 服务未初始化
  106 +
  107 +#### 请求示例
  108 +
  109 +**全量索引(不会删除现有索引)**:
  110 +
  111 +```bash
  112 +curl -X POST "http://localhost:6004/indexer/reindex" \
  113 + -H "Content-Type: application/json" \
  114 + -d '{
  115 + "tenant_id": "162",
  116 + "batch_size": 500
  117 + }'
  118 +```
  119 +
  120 +**查看日志**:
  121 +
  122 +```bash
  123 +# 查看API日志(包含索引操作日志)
  124 +tail -f logs/api.log
  125 +
  126 +# 或者查看所有日志文件
  127 +tail -f logs/*.log
  128 +```
  129 +
  130 +> ⚠️ **重要提示**:如需 **创建索引结构**,请参考 [5.1 为租户创建索引](#51-为租户创建索引) 章节,使用 `./scripts/create_tenant_index.sh <tenant_id>`。创建后需要调用 `/indexer/reindex` 导入数据。
  131 +
  132 +**查看索引日志**:
  133 +
  134 +索引操作的所有关键信息都会记录到 `logs/indexer.log` 文件中(JSON 格式),包括:
  135 +- 请求开始和结束时间
  136 +- 租户ID、SPU ID、操作类型
  137 +- 每个SPU的处理状态
  138 +- ES批量写入结果
  139 +- 成功/失败统计和详细错误信息
  140 +
  141 +```bash
  142 +# 实时查看索引日志(包含全量和增量索引的所有操作)
  143 +tail -f logs/indexer.log
  144 +
  145 +# 使用 grep 查询(简单方式)
  146 +# 查看全量索引日志
  147 +grep "\"index_type\":\"bulk\"" logs/indexer.log | tail -100
  148 +
  149 +# 查看增量索引日志
  150 +grep "\"index_type\":\"incremental\"" logs/indexer.log | tail -100
  151 +
  152 +# 查看特定租户的索引日志
  153 +grep "\"tenant_id\":\"162\"" logs/indexer.log | tail -100
  154 +
  155 +# 使用 jq 查询(推荐,更精确的 JSON 查询)
  156 +# 安装 jq: sudo apt-get install jq 或 brew install jq
  157 +
  158 +# 查看全量索引日志
  159 +cat logs/indexer.log | jq 'select(.index_type == "bulk")' | tail -100
  160 +
  161 +# 查看增量索引日志
  162 +cat logs/indexer.log | jq 'select(.index_type == "incremental")' | tail -100
  163 +
  164 +# 查看特定租户的索引日志
  165 +cat logs/indexer.log | jq 'select(.tenant_id == "162")' | tail -100
  166 +
  167 +# 查看失败的索引操作
  168 +cat logs/indexer.log | jq 'select(.operation == "request_complete" and .failed_count > 0)'
  169 +
  170 +# 查看特定SPU的处理日志
  171 +cat logs/indexer.log | jq 'select(.spu_id == "123")'
  172 +
  173 +# 查看最近的索引请求统计
  174 +cat logs/indexer.log | jq 'select(.operation == "request_complete") | {timestamp, index_type, tenant_id, total_count, success_count, failed_count, elapsed_time}'
  175 +```
  176 +
  177 +### 5.3 增量索引接口
  178 +
  179 +- **端点**: `POST /indexer/index`
  180 +- **描述**: 增量索引接口,根据指定的SPU ID列表进行索引,直接将数据写入ES。用于增量更新指定商品。**推荐仅作为内部/调试入口**;正式对接建议改用 `/indexer/build-docs`,由上游写 ES。
  181 +
  182 +**删除说明**:
  183 +- `spu_ids`中的SPU:如果数据库`deleted=1`,自动从ES删除,响应状态为`deleted`
  184 +- `delete_spu_ids`中的SPU:直接删除,响应状态为`deleted`、`not_found`或`failed`
  185 +
  186 +#### 请求参数
  187 +
  188 +```json
  189 +{
  190 + "tenant_id": "162",
  191 + "spu_ids": ["123", "456", "789"],
  192 + "delete_spu_ids": ["100", "101"]
  193 +}
  194 +```
  195 +
  196 +| 参数 | 类型 | 必填 | 说明 |
  197 +|------|------|------|------|
  198 +| `tenant_id` | string | Y | 租户ID |
  199 +| `spu_ids` | array[string] | N | SPU ID列表(1-100个),要索引的SPU。如果为空,则只执行删除操作 |
  200 +| `delete_spu_ids` | array[string] | N | 显式指定要删除的SPU ID列表(1-100个),可选。无论数据库状态如何,都会从ES中删除这些SPU |
  201 +
  202 +**注意**:
  203 +- `spu_ids` 和 `delete_spu_ids` 不能同时为空
  204 +- 每个列表最多支持100个SPU ID
  205 +- 如果SPU在`spu_ids`中且数据库`deleted=1`,会自动从ES删除(自动检测删除)
  206 +
  207 +#### 响应格式
  208 +
  209 +```json
  210 +{
  211 + "spu_ids": [
  212 + {
  213 + "spu_id": "123",
  214 + "status": "indexed"
  215 + },
  216 + {
  217 + "spu_id": "456",
  218 + "status": "deleted"
  219 + },
  220 + {
  221 + "spu_id": "789",
  222 + "status": "failed",
  223 + "msg": "SPU not found (unexpected)"
  224 + }
  225 + ],
  226 + "delete_spu_ids": [
  227 + {
  228 + "spu_id": "100",
  229 + "status": "deleted"
  230 + },
  231 + {
  232 + "spu_id": "101",
  233 + "status": "not_found"
  234 + },
  235 + {
  236 + "spu_id": "102",
  237 + "status": "failed",
  238 + "msg": "Failed to delete from ES: Connection timeout"
  239 + }
  240 + ],
  241 + "total": 6,
  242 + "success_count": 4,
  243 + "failed_count": 2,
  244 + "elapsed_time": 1.23,
  245 + "index_name": "search_products",
  246 + "tenant_id": "162"
  247 +}
  248 +```
  249 +
  250 +| 字段 | 类型 | 说明 |
  251 +|------|------|------|
  252 +| `spu_ids` | array | spu_ids对应的响应列表,每个元素包含 `spu_id` 和 `status` |
  253 +| `spu_ids[].status` | string | 状态:`indexed`(已索引)、`deleted`(已删除,自动检测)、`failed`(失败) |
  254 +| `spu_ids[].msg` | string | 当status为`failed`时,包含失败原因(可选) |
  255 +| `delete_spu_ids` | array | delete_spu_ids对应的响应列表,每个元素包含 `spu_id` 和 `status` |
  256 +| `delete_spu_ids[].status` | string | 状态:`deleted`(已删除)、`not_found`(ES中不存在)、`failed`(失败) |
  257 +| `delete_spu_ids[].msg` | string | 当status为`failed`时,包含失败原因(可选) |
  258 +| `total` | integer | 总处理数量(spu_ids数量 + delete_spu_ids数量) |
  259 +| `success_count` | integer | 成功数量(indexed + deleted + not_found) |
  260 +| `failed_count` | integer | 失败数量 |
  261 +| `elapsed_time` | float | 耗时(秒) |
  262 +| `index_name` | string | 索引名称 |
  263 +| `tenant_id` | string | 租户ID |
  264 +
  265 +**状态说明**:
  266 +- `spu_ids` 的状态:
  267 + - `indexed`: SPU已成功索引到ES
  268 + - `deleted`: SPU在数据库中被标记为deleted=1,已从ES删除(自动检测)
  269 + - `failed`: 处理失败,会包含`msg`字段说明失败原因
  270 +- `delete_spu_ids` 的状态:
  271 + - `deleted`: SPU已从ES成功删除
  272 + - `not_found`: SPU在ES中不存在(也算成功,可能已经被删除过)
  273 + - `failed`: 删除失败,会包含`msg`字段说明失败原因
  274 +
  275 +#### 请求示例
  276 +
  277 +**示例1:普通增量索引(自动检测删除)**:
  278 +
  279 +```bash
  280 +curl -X POST "http://localhost:6004/indexer/index" \
  281 + -H "Content-Type: application/json" \
  282 + -d '{
  283 + "tenant_id": "162",
  284 + "spu_ids": ["123", "456", "789"]
  285 + }'
  286 +```
  287 +
  288 +说明:如果SPU 456在数据库中`deleted=1`,会自动从ES删除,在响应中`spu_ids`列表里456的状态为`deleted`。
  289 +
  290 +**示例2:显式删除(批量删除)**:
  291 +
  292 +```bash
  293 +curl -X POST "http://localhost:6004/indexer/index" \
  294 + -H "Content-Type: application/json" \
  295 + -d '{
  296 + "tenant_id": "162",
  297 + "spu_ids": ["123", "456"],
  298 + "delete_spu_ids": ["100", "101", "102"]
  299 + }'
  300 +```
  301 +
  302 +说明:SPU 100、101、102会被显式删除,无论数据库状态如何。
  303 +
  304 +**示例3:仅删除(不索引)**:
  305 +
  306 +```bash
  307 +curl -X POST "http://localhost:6004/indexer/index" \
  308 + -H "Content-Type: application/json" \
  309 + -d '{
  310 + "tenant_id": "162",
  311 + "spu_ids": [],
  312 + "delete_spu_ids": ["100", "101"]
  313 + }'
  314 +```
  315 +
  316 +说明:只执行删除操作,不进行索引。
  317 +
  318 +**示例4:混合操作(索引+删除)**:
  319 +
  320 +```bash
  321 +curl -X POST "http://localhost:6004/indexer/index" \
  322 + -H "Content-Type: application/json" \
  323 + -d '{
  324 + "tenant_id": "162",
  325 + "spu_ids": ["123", "456", "789"],
  326 + "delete_spu_ids": ["100", "101"]
  327 + }'
  328 +```
  329 +
  330 +说明:同时执行索引和删除操作。
  331 +
  332 +#### 日志说明
  333 +
  334 +增量索引操作的所有关键信息都会记录到 `logs/indexer.log` 文件中(JSON格式),包括:
  335 +- 请求开始和结束时间
  336 +- 每个SPU的处理状态(获取、转换、索引、删除)
  337 +- ES批量写入结果
  338 +- 成功/失败统计
  339 +- 详细的错误信息
  340 +
  341 +日志查询方式请参考[5.1节查看索引日志](#51-全量重建索引接口)部分。
  342 +
  343 +### 5.4 查询文档接口
  344 +
  345 +- **端点**: `POST /indexer/documents`
  346 +- **描述**: 查询文档接口,根据SPU ID列表获取ES文档数据(**不写入ES**)。用于查看、调试或验证SPU数据。
  347 +
  348 +#### 请求参数
  349 +
  350 +```json
  351 +{
  352 + "tenant_id": "162",
  353 + "spu_ids": ["123", "456", "789"]
  354 +}
  355 +```
  356 +
  357 +| 参数 | 类型 | 必填 | 说明 |
  358 +|------|------|------|------|
  359 +| `tenant_id` | string | Y | 租户ID |
  360 +| `spu_ids` | array[string] | Y | SPU ID列表(1-100个) |
  361 +
  362 +#### 响应格式
  363 +
  364 +```json
  365 +{
  366 + "success": [
  367 + {
  368 + "spu_id": "123",
  369 + "document": {
  370 + "tenant_id": "162",
  371 + "spu_id": "123",
  372 + "title": {
  373 + "zh": "商品标题"
  374 + },
  375 + ...
  376 + }
  377 + },
  378 + {
  379 + "spu_id": "456",
  380 + "document": {...}
  381 + }
  382 + ],
  383 + "failed": [
  384 + {
  385 + "spu_id": "789",
  386 + "error": "SPU not found or deleted"
  387 + }
  388 + ],
  389 + "total": 3,
  390 + "success_count": 2,
  391 + "failed_count": 1
  392 +}
  393 +```
  394 +
  395 +| 字段 | 类型 | 说明 |
  396 +|------|------|------|
  397 +| `success` | array | 成功获取的SPU列表,每个元素包含 `spu_id` 和 `document`(完整的ES文档数据) |
  398 +| `failed` | array | 失败的SPU列表,每个元素包含 `spu_id` 和 `error`(失败原因) |
  399 +| `total` | integer | 总SPU数量 |
  400 +| `success_count` | integer | 成功数量 |
  401 +| `failed_count` | integer | 失败数量 |
  402 +
  403 +#### 请求示例
  404 +
  405 +**单个SPU查询**:
  406 +
  407 +```bash
  408 +curl -X POST "http://localhost:6004/indexer/documents" \
  409 + -H "Content-Type: application/json" \
  410 + -d '{
  411 + "tenant_id": "162",
  412 + "spu_ids": ["123"]
  413 + }'
  414 +```
  415 +
  416 +**批量SPU查询**:
  417 +
  418 +```bash
  419 +curl -X POST "http://localhost:6004/indexer/documents" \
  420 + -H "Content-Type: application/json" \
  421 + -d '{
  422 + "tenant_id": "162",
  423 + "spu_ids": ["123", "456", "789"]
  424 + }'
  425 +```
  426 +
  427 +#### 与 `/indexer/index` 的区别
  428 +
  429 +| 接口 | 功能 | 是否写入ES | 返回内容 |
  430 +|------|------|-----------|----------|
  431 +| `/indexer/documents` | 查询SPU文档数据 | 否 | 返回完整的ES文档数据 |
  432 +| `/indexer/index` | 增量索引 | 是 | 返回成功/失败列表和统计信息 |
  433 +
  434 +**使用场景**:
  435 +- `/indexer/documents`:用于查看、调试或验证SPU数据,不修改ES索引
  436 +- `/indexer/index`:用于实际的增量索引操作,将更新的SPU数据同步到ES
  437 +
  438 +### 5.5 索引健康检查接口
  439 +
  440 +- **端点**: `GET /indexer/health`
  441 +- **描述**: 检查索引服务健康状态(与 `api/routes/indexer.py` 中 `indexer_health_check` 一致)
  442 +
  443 +#### 响应格式
  444 +
  445 +```json
  446 +{
  447 + "status": "available",
  448 + "database": "connected",
  449 + "preloaded_data": {
  450 + "category_mappings": 150
  451 + }
  452 +}
  453 +```
  454 +
  455 +| 字段 | 类型 | 说明 |
  456 +|------|------|------|
  457 +| `status` | string | `available`(服务可用)、`unavailable`(未初始化)、`error`(异常) |
  458 +| `database` | string | 数据库连接状态,如 `connected` 或 `disconnected: ...` |
  459 +| `preloaded_data.category_mappings` | integer | 已加载的分类映射数量 |
  460 +
  461 +#### 请求示例
  462 +
  463 +```bash
  464 +curl -X GET "http://localhost:6004/indexer/health"
  465 +```
  466 +
  467 +### 5.6 文档构建接口(正式对接推荐)
  468 +
  469 +#### 5.6.1 `POST /indexer/build-docs`
  470 +
  471 +- **描述**:
  472 + 基于调用方(通常是 Java 索引程序)提供的 **MySQL 行数据** 构建 ES 文档(doc),**不写入 ES**。
  473 + 由本服务负责“如何构建 doc”(多语言、翻译、向量、规格聚合等),由调用方负责“何时调度 + 如何写 ES”。
  474 +
  475 +#### 请求参数
  476 +
  477 +```json
  478 +{
  479 + "tenant_id": "170",
  480 + "items": [
  481 + {
  482 + "spu": { "id": 223167, "tenant_id": 170, "title": "..." },
  483 + "skus": [
  484 + { "id": 3988393, "spu_id": 223167, "price": 25.99, "compare_at_price": 25.99 }
  485 + ],
  486 + "options": []
  487 + }
  488 + ]
  489 +}
  490 +```
  491 +
  492 +| 参数 | 类型 | 必填 | 说明 |
  493 +|------|------|------|------|
  494 +| `tenant_id` | string | Y | 租户 ID |
  495 +| `items` | array | Y | 需构建 doc 的 SPU 列表(每项含 `spu`、`skus`、`options`),**单次最多 200 条** |
  496 +
  497 +> `spu` / `skus` / `options` 字段应当直接使用从 `shoplazza_product_spu` / `shoplazza_product_sku` / `shoplazza_product_option` 查询出的行字段。
  498 +
  499 +#### 请求示例(完整 curl)
  500 +
  501 +> 完整请求体参考 `scripts/test_build_docs_api.py` 中的 `build_sample_request()`。
  502 +
  503 +```bash
  504 +# 单条 SPU 示例(含 spu、skus、options)
  505 +curl -X POST "http://localhost:6004/indexer/build-docs" \
  506 + -H "Content-Type: application/json" \
  507 + -d '{
  508 + "tenant_id": "162",
  509 + "items": [
  510 + {
  511 + "spu": {
  512 + "id": 10001,
  513 + "tenant_id": "162",
  514 + "title": "测试T恤 纯棉短袖",
  515 + "brief": "舒适纯棉,多色可选",
  516 + "description": "这是一款适合日常穿着的纯棉T恤,透气吸汗。",
  517 + "vendor": "测试品牌",
  518 + "category": "服装/上衣/T恤",
  519 + "category_id": 100,
  520 + "category_level": 2,
  521 + "category_path": "服装/上衣/T恤",
  522 + "fake_sales": 1280,
  523 + "image_src": "https://oss.essa.cn/98532128-cf8e-456c-9e30-6f2a5ea0c19f.jpg",
  524 + "tags": "T恤,纯棉,短袖,夏季",
  525 + "create_time": "2024-01-01T00:00:00Z",
  526 + "update_time": "2024-01-01T00:00:00Z"
  527 + },
  528 + "skus": [
  529 + {
  530 + "id": 20001,
  531 + "spu_id": 10001,
  532 + "price": 99.0,
  533 + "compare_at_price": 129.0,
  534 + "sku": "SKU-TSHIRT-001",
  535 + "inventory_quantity": 50,
  536 + "option1": "黑色",
  537 + "option2": "M",
  538 + "option3": null
  539 + },
  540 + {
  541 + "id": 20002,
  542 + "spu_id": 10001,
  543 + "price": 99.0,
  544 + "compare_at_price": 129.0,
  545 + "sku": "SKU-TSHIRT-002",
  546 + "inventory_quantity": 30,
  547 + "option1": "白色",
  548 + "option2": "L",
  549 + "option3": null
  550 + }
  551 + ],
  552 + "options": [
  553 + {"id": 1, "position": 1, "name": "颜色"},
  554 + {"id": 2, "position": 2, "name": "尺码"}
  555 + ]
  556 + }
  557 + ]
  558 +}'
  559 +```
  560 +
  561 +生产环境替换 `localhost:6004` 为实际 Indexer 地址,如 `http://43.166.252.75:6004`。
  562 +
  563 +#### 响应示例(节选)
  564 +
  565 +```json
  566 +{
  567 + "tenant_id": "170",
  568 + "docs": [
  569 + {
  570 + "tenant_id": "170",
  571 + "spu_id": "223167",
  572 + "title": { "en": "...", "zh": "..." },
  573 + "tags": ["Floerns", "Clothing", "Shoes & Jewelry"],
  574 + "skus": [
  575 + {
  576 + "sku_id": "3988393",
  577 + "price": 25.99,
  578 + "compare_at_price": 25.99,
  579 + "stock": 100
  580 + }
  581 + ],
  582 + "min_price": 25.99,
  583 + "max_price": 25.99,
  584 + "compare_at_price": 25.99,
  585 + "total_inventory": 100,
  586 + "title_embedding": [/* 1024 维向量 */]
  587 + // 其余字段与 mappings/search_products.json 一致
  588 + }
  589 + ],
  590 + "total": 1,
  591 + "success_count": 1,
  592 + "failed_count": 0,
  593 + "failed": []
  594 +}
  595 +```
  596 +
  597 +| 字段 | 类型 | 说明 |
  598 +|------|------|------|
  599 +| `tenant_id` | string | 租户 ID |
  600 +| `docs` | array | 构建成功的 ES 文档列表,与 `mappings/search_products.json` 一致 |
  601 +| `total` | integer | 请求的 items 总数 |
  602 +| `success_count` | integer | 成功构建数量 |
  603 +| `failed_count` | integer | 失败数量 |
  604 +| `failed` | array | 失败项列表,每项含 `spu_id`、`error` |
  605 +
  606 +#### 使用建议
  607 +
  608 +- **生产环境推荐流程**:
  609 + 1. Java 根据业务逻辑决定哪些 SPU 需要(全量/增量)处理;
  610 + 2. Java 从 MySQL 查询 SPU/SKU/Option 行,拼成 `items`;
  611 + 3. 调用 `/indexer/build-docs` 获取 ES-ready `docs`;
  612 + 4. Java 使用自己的 ES 客户端写入 `search_products_tenant_{tenant_id}`。
  613 +
  614 +### 5.7 文档构建接口(测试 / 自测)
  615 +
  616 +#### 5.7.1 `POST /indexer/build-docs-from-db`
  617 +
  618 +- **描述**:
  619 + 仅用于测试/调试:调用方只提供 `tenant_id` 和 `spu_ids`,由 indexer 服务内部从 MySQL 查询 SPU/SKU/Option,然后调用与 `/indexer/build-docs` 相同的文档构建逻辑,返回 ES-ready doc。**生产环境请使用 `/indexer/build-docs`,由上游查库并写 ES。**
  620 +
  621 +#### 请求参数
  622 +
  623 +```json
  624 +{
  625 + "tenant_id": "170",
  626 + "spu_ids": ["223167", "223168"]
  627 +}
  628 +```
  629 +
  630 +| 参数 | 类型 | 必填 | 说明 |
  631 +|------|------|------|------|
  632 +| `tenant_id` | string | Y | 租户 ID |
  633 +| `spu_ids` | array[string] | Y | SPU ID 列表,**单次最多 200 个** |
  634 +
  635 +#### 响应格式
  636 +
  637 +与 `/indexer/build-docs` 相同:`tenant_id`、`docs`、`total`、`success_count`、`failed_count`、`failed`。
  638 +
  639 +#### 请求示例
  640 +
  641 +```bash
  642 +curl -X POST "http://127.0.0.1:6004/indexer/build-docs-from-db" \
  643 + -H "Content-Type: application/json" \
  644 + -d '{"tenant_id": "170", "spu_ids": ["223167"]}'
  645 +```
  646 +
  647 +返回结构与 `/indexer/build-docs` 相同,可直接用于对比 ES 实际文档或调试字段映射问题。
  648 +
  649 +### 5.8 内容理解字段生成接口
  650 +
  651 +- **端点**: `POST /indexer/enrich-content`
  652 +- **描述**: 根据商品内容信息批量生成 **qanchors**(锚文本)、**semantic_attributes**(语义属性)、**tags**(细分标签),供外部 indexer 在「微服务组合」方式下自行拼装 doc 时使用。请求以 `items[]` 传入商品内容字段(必填/可选见下表)。内部逻辑与 `indexer.product_enrich` 一致,支持多语言与 Redis 缓存;单次请求在线程池中执行,避免阻塞其他接口。
  653 +
  654 +#### 请求参数
  655 +
  656 +```json
  657 +{
  658 + "tenant_id": "170",
  659 + "items": [
  660 + {
  661 + "spu_id": "223167",
  662 + "title": "纯棉短袖T恤 夏季男装",
  663 + "brief": "夏季透气纯棉短袖,舒适亲肤",
  664 + "description": "100%棉,圆领版型,适合日常通勤与休闲穿搭。",
  665 + "image_url": "https://example.com/images/223167.jpg"
  666 + },
  667 + {
  668 + "spu_id": "223168",
  669 + "title": "12PCS Dolls with Bottles",
  670 + "image_url": "https://example.com/images/223168.jpg"
  671 + }
  672 + ],
  673 + "languages": ["zh", "en"]
  674 +}
  675 +```
  676 +
  677 +| 参数 | 类型 | 必填 | 默认值 | 说明 |
  678 +|------|------|------|--------|------|
  679 +| `tenant_id` | string | Y | - | 租户 ID。目前仅用于记录日志,不产生实际作用|
  680 +| `items` | array | Y | - | 待分析列表;**单次最多 50 条** |
  681 +| `languages` | array[string] | N | `["zh", "en"]` | 目标语言,需在支持范围内:`zh`、`en`、`de`、`ru`、`fr` |
  682 +
  683 +`items[]` 字段说明:
  684 +
  685 +| 字段 | 类型 | 必填 | 说明 |
  686 +|------|------|------|------|
  687 +| `spu_id` | string | Y | SPU ID,用于回填结果;目前仅用于记录日志,不产生实际作用|
  688 +| `title` | string | Y | 商品标题 |
  689 +| `image_url` | string | N | 商品主图 URL;当前会参与内容缓存键,后续可用于图像/多模态内容理解 |
  690 +| `brief` | string | N | 商品简介/短描述;当前会参与内容缓存键 |
  691 +| `description` | string | N | 商品详情/长描述;当前会参与内容缓存键 |
  692 +
  693 +缓存说明:
  694 +
  695 +- 内容缓存键仅由 `target_lang + items[]` 中会影响内容理解结果的输入文本构成,目前包括:`title`、`brief`、`description`、`image_url` 的规范化内容 hash。
  696 +- `tenant_id`、`spu_id` 只用于请求归属与结果回填,不参与缓存键。
  697 +- 因此,输入内容不变时可跨请求直接命中缓存;任一输入字段变化时,会自然落到新的缓存 key。
  698 +
  699 +批量请求建议:
  700 +- **全量**:强烈建议 尽可能 **20 个 SPU/doc** 攒成一个批次后再请求一次。
  701 +- **增量**:可按时效要求设置时间窗口(例如 **5 分钟**),在窗口内尽可能攒到 **20 个**;达到 20 或窗口到期就发送一次请求。
  702 +- 允许超过20,服务内部会拆分成小批次逐个处理。也允许小于20,但是将造成费用和耗时的成本上升,特别是每次请求一个doc的情况。
  703 +
  704 +#### 响应格式
  705 +
  706 +```json
  707 +{
  708 + "tenant_id": "170",
  709 + "total": 2,
  710 + "results": [
  711 + {
  712 + "spu_id": "223167",
  713 + "qanchors": {
  714 + "zh": "短袖T恤,纯棉,男装,夏季",
  715 + "en": "cotton t-shirt, short sleeve, men, summer"
  716 + },
  717 + "semantic_attributes": [
  718 + { "lang": "zh", "name": "tags", "value": "纯棉" },
  719 + { "lang": "zh", "name": "usage_scene", "value": "日常" },
  720 + { "lang": "en", "name": "tags", "value": "cotton" }
  721 + ],
  722 + "tags": ["纯棉", "短袖", "男装", "cotton", "short sleeve"]
  723 + },
  724 + {
  725 + "spu_id": "223168",
  726 + "qanchors": { "en": "dolls, toys, 12pcs" },
  727 + "semantic_attributes": [],
  728 + "tags": ["dolls", "toys"]
  729 + }
  730 + ]
  731 +}
  732 +```
  733 +
  734 +| 字段 | 类型 | 说明 |
  735 +|------|------|------|
  736 +| `results` | array | 与请求 `items` 一一对应,每项含 `spu_id`、`qanchors`、`semantic_attributes`、`tags` |
  737 +| `results[].qanchors` | object | 按语言键的锚文本(逗号分隔短语),可写入 ES 文档的 `qanchors.{lang}` |
  738 +| `results[].semantic_attributes` | array | 语义属性列表,每项为 `{ "lang", "name", "value" }`,可写入 ES 的 `semantic_attributes` nested 字段 |
  739 +| `results[].tags` | array | 从语义属性中抽取的 `name=tags` 的 value 集合,可与业务原有 `tags` 合并后写入 ES 的 `tags` 字段 |
  740 +| `results[].error` | string | 若该条处理失败(如 LLM 异常),会在此字段返回错误信息 |
  741 +
  742 +**错误响应**:
  743 +- `400`: `items` 为空或超过 50 条
  744 +- `503`: 未配置 `DASHSCOPE_API_KEY`,内容理解服务不可用
  745 +
  746 +#### 请求示例
  747 +
  748 +```bash
  749 +curl -X POST "http://localhost:6004/indexer/enrich-content" \
  750 + -H "Content-Type: application/json" \
  751 + -d '{
  752 + "tenant_id": "170",
  753 + "items": [
  754 + {
  755 + "spu_id": "223167",
  756 + "title": "纯棉短袖T恤 夏季男装",
  757 + "brief": "夏季透气纯棉短袖,舒适亲肤",
  758 + "description": "100%棉,圆领版型,适合日常通勤与休闲穿搭。",
  759 + "image_url": "https://example.com/images/223167.jpg"
  760 + }
  761 + ],
  762 + "languages": ["zh", "en"]
  763 + }'
  764 +```
  765 +
  766 +---
  767 +
docs/搜索API对接指南-06-管理接口(Admin).md 0 → 100644
@@ -0,0 +1,53 @@ @@ -0,0 +1,53 @@
  1 +# 搜索API对接指南-06-管理接口(Admin)
  2 +
  3 +用于查看服务健康状态、获取租户配置与索引统计信息(原文第 6 章)。
  4 +
  5 +## 管理接口
  6 +
  7 +### 6.1 健康检查
  8 +
  9 +- **端点**: `GET /admin/health`
  10 +- **描述**: 检查服务与依赖(如 Elasticsearch)状态。
  11 +
  12 +```json
  13 +{
  14 + "status": "healthy",
  15 + "elasticsearch": "connected",
  16 + "tenant_id": "tenant1"
  17 +}
  18 +```
  19 +
  20 +### 6.2 获取配置
  21 +
  22 +- **端点**: `GET /admin/config`
  23 +- **描述**: 返回当前租户的脱敏配置,便于核对索引及排序表达式。
  24 +
  25 +```json
  26 +{
  27 + "tenant_id": "tenant1",
  28 + "tenant_name": "Tenant1 Test Instance",
  29 + "es_index_name": "search_tenant1",
  30 + "num_fields": 20,
  31 + "num_indexes": 4,
  32 + "supported_languages": ["zh", "en", "ru"],
  33 + "spu_enabled": false
  34 +}
  35 +```
  36 +
  37 +### 6.3 索引统计
  38 +
  39 +- **端点**: `GET /admin/stats`
  40 +- **描述**: 获取指定租户索引文档数量与磁盘大小,方便监控。
  41 +- **租户标识**:通过请求头 `X-Tenant-ID` 或 query 参数 `tenant_id` 传递(必填)。
  42 +
  43 +```json
  44 +{
  45 + "tenant_id": "162",
  46 + "index_name": "search_products_tenant_162",
  47 + "document_count": 10000,
  48 + "size_mb": 523.45
  49 +}
  50 +```
  51 +
  52 +---
  53 +
docs/搜索API对接指南-07-微服务接口(Embedding-Reranker-Translation).md 0 → 100644
@@ -0,0 +1,401 @@ @@ -0,0 +1,401 @@
  1 +# 搜索API对接指南-07-微服务接口(Embedding-Reranker-Translation)
  2 +
  3 +本篇覆盖向量服务(Embedding)、重排服务(Reranker)、翻译服务(Translation)以及 Indexer 服务内的内容理解字段生成(原文第 7 章)。
  4 +
  5 +## 7. 微服务接口(向量、重排、翻译)
  6 +
  7 +以下三个微服务独立部署,**外部系统可直接调用**。它们被搜索后端(6002)和索引服务(6004)内部使用,也可供其他业务系统直接对接。
  8 +
  9 +| 服务 | 默认端口 | Base URL | 说明 |
  10 +|------|----------|----------|------|
  11 +| 向量服务(文本) | 6005 | `http://localhost:6005` | 文本向量化,用于 query/doc 语义检索 |
  12 +| 向量服务(图片) | 6008 | `http://localhost:6008` | 图片向量化,用于以图搜图 |
  13 +| 翻译服务 | 6006 | `http://localhost:6006` | 多语言翻译(云端与本地模型统一入口) |
  14 +| 重排服务 | 6007 | `http://localhost:6007` | 对检索结果进行二次排序 |
  15 +
  16 +生产环境请将 `localhost` 替换为实际服务地址。
  17 +服务管理入口与完整启停规则见:`docs/Usage-Guide.md` -> `服务管理总览`。
  18 +
  19 +### 7.1 向量服务(Embedding)
  20 +
  21 +- **Base URL**:
  22 + - 文本:`http://localhost:6005`(可通过 `EMBEDDING_TEXT_SERVICE_URL` 覆盖)
  23 + - 图片:`http://localhost:6008`(可通过 `EMBEDDING_IMAGE_SERVICE_URL` 覆盖)
  24 +- **启动**:
  25 + - 文本:`./scripts/start_embedding_text_service.sh`
  26 + - 图片:`./scripts/start_embedding_image_service.sh`
  27 +- **依赖**:
  28 + - 文本向量后端默认走 TEI(`http://127.0.0.1:8080`)
  29 + - 图片向量依赖 `cnclip`(`grpc://127.0.0.1:51000`)
  30 + - TEI 默认使用 GPU(`TEI_DEVICE=cuda`);当配置为 GPU 且不可用时会启动失败(不会自动降级到 CPU)
  31 + - cnclip 默认使用 `cuda`;若配置为 `cuda` 但 GPU 不可用会启动失败(不会自动降级到 `cpu`)
  32 + - 当前单机部署建议保持单实例,通过**文本/图片拆分 + 独立限流**隔离压力
  33 +
  34 +补充说明:
  35 +
  36 +- 文本和图片现在已经拆成**不同进程 / 不同端口**,避免图片下载与编码波动影响文本向量化。
  37 +- 服务端对 text / image 有**独立 admission control**:
  38 + - `TEXT_MAX_INFLIGHT`
  39 + - `IMAGE_MAX_INFLIGHT`
  40 +- 当超过处理能力时,服务会直接返回过载错误,而不是无限排队。
  41 +- `GET /health` 会返回各自的 `limits`、`stats`、`cache_enabled` 等状态;`GET /ready` 用于就绪探针。
  42 +
  43 +#### 7.1.1 `POST /embed/text` — 文本向量化
  44 +
  45 +将文本列表转为 1024 维向量,用于语义搜索、文档索引等。
  46 +
  47 +**请求体**(JSON 数组):
  48 +
  49 +```json
  50 +["文本1", "文本2", "文本3"]
  51 +```
  52 +
  53 +**响应**(JSON 数组,与输入一一对应):
  54 +
  55 +```json
  56 +[[0.01, -0.02, ...], [0.03, 0.01, ...], ...]
  57 +```
  58 +
  59 +**完整 curl 示例**:
  60 +
  61 +```bash
  62 +curl -X POST "http://localhost:6005/embed/text?normalize=true" \
  63 + -H "Content-Type: application/json" \
  64 + -d '["芭比娃娃 儿童玩具", "纯棉T恤 短袖"]'
  65 +```
  66 +
  67 +#### 7.1.2 `POST /embed/image` — 图片向量化
  68 +
  69 +将图片 URL 或路径转为向量,用于以图搜图。
  70 +
  71 +前置条件:`cnclip` 服务已启动(默认端口 `51000`)。若未启动,图片 embedding 服务启动会失败或请求返回错误。
  72 +
  73 +**请求体**(JSON 数组):
  74 +
  75 +```json
  76 +["https://example.com/image1.jpg", "https://example.com/image2.jpg"]
  77 +```
  78 +
  79 +**响应**(JSON 数组,与输入一一对应):
  80 +
  81 +```json
  82 +[[0.01, -0.02, ...], [0.03, 0.01, ...], ...]
  83 +```
  84 +
  85 +**完整 curl 示例**:
  86 +
  87 +```bash
  88 +curl -X POST "http://localhost:6008/embed/image?normalize=true" \
  89 + -H "Content-Type: application/json" \
  90 + -d '["https://oss.essa.cn/98532128-cf8e-456c-9e30-6f2a5ea0c19f.jpg"]'
  91 +```
  92 +
  93 +#### 7.1.3 `GET /health` — 健康检查
  94 +
  95 +```bash
  96 +curl "http://localhost:6005/health"
  97 +curl "http://localhost:6008/health"
  98 +```
  99 +
  100 +返回中会包含:
  101 +
  102 +- `service_kind`:`text` / `image` / `all`
  103 +- `cache_enabled`:text/image Redis 缓存是否可用
  104 +- `limits`:当前 inflight limit、active、rejected_total 等
  105 +- `stats`:request_total、cache_hits、cache_misses、avg_latency_ms 等
  106 +
  107 +#### 7.1.4 `GET /ready` — 就绪检查
  108 +
  109 +```bash
  110 +curl "http://localhost:6005/ready"
  111 +curl "http://localhost:6008/ready"
  112 +```
  113 +
  114 +#### 7.1.5 缓存与限流说明
  115 +
  116 +- 文本与图片都会先查 Redis 向量缓存。
  117 +- Redis 中 value 仍是 **BF16 bytes**,读取后恢复成 `float32` 返回。
  118 +- cache key 已区分 `normalize=true/false`,避免不同归一化策略命中同一条缓存。
  119 +- 当服务端发现请求是 **full-cache-hit** 时,会直接返回,不占用模型并发槽位。
  120 +- 当服务端发现超过 `TEXT_MAX_INFLIGHT` / `IMAGE_MAX_INFLIGHT` 时,会直接拒绝,而不是无限排队。
  121 +
  122 +#### 7.1.6 TEI 统一调优建议(主服务)
  123 +
  124 +使用单套主服务即可同时兼顾:
  125 +- 在线 query 向量化(低延迟,常见 `batch=1~4`)
  126 +- 索引构建向量化(高吞吐,常见 `batch=15~20`)
  127 +
  128 +统一启动(主链路):
  129 +
  130 +```bash
  131 +./scripts/start_tei_service.sh
  132 +./scripts/service_ctl.sh restart embedding
  133 +```
  134 +
  135 +默认端口:
  136 +- TEI: `http://127.0.0.1:8080`
  137 +- 文本向量服务(`/embed/text`): `http://127.0.0.1:6005`
  138 +- 图片向量服务(`/embed/image`): `http://127.0.0.1:6008`
  139 +
  140 +当前主 TEI 启动默认值(已按 T4/短文本场景调优):
  141 +- `TEI_MAX_BATCH_TOKENS=4096`
  142 +- `TEI_MAX_CLIENT_BATCH_SIZE=24`
  143 +- `TEI_DTYPE=float16`
  144 +
  145 +### 7.2 重排服务(Reranker)
  146 +
  147 +- **Base URL**: `http://localhost:6007`(可通过 `RERANKER_SERVICE_URL` 覆盖)
  148 +- **启动**: `./scripts/start_reranker.sh`
  149 +
  150 +说明:默认后端为 `qwen3_vllm`(`Qwen/Qwen3-Reranker-0.6B`),需要可用 GPU 显存。
  151 +
  152 +补充:`docs` 的请求大小与模型推理 `batch size` 解耦。即使一次传入 1000 条文档,服务端也会按 `services.rerank.backends.qwen3_vllm.infer_batch_size` 自动拆分;若 `sort_by_doc_length=true`,会先按文档长度排序后分批,减少 padding,再按原输入顺序返回分数。`length_sort_mode` 可选 `char`(更快)或 `token`(更精确)。
  153 +
  154 +#### 7.2.1 `POST /rerank` — 结果重排
  155 +
  156 +根据 query 与 doc 的相关性对文档列表重新打分排序。
  157 +
  158 +**请求体**:
  159 +```json
  160 +{
  161 + "query": "玩具 芭比",
  162 + "docs": [
  163 + "12PCS 6 Types of Dolls with Bottles",
  164 + "纯棉T恤 短袖 夏季"
  165 + ],
  166 + "normalize": true
  167 +}
  168 +```
  169 +
  170 +| 参数 | 类型 | 必填 | 说明 |
  171 +|------|------|------|------|
  172 +| `query` | string | Y | 搜索查询 |
  173 +| `docs` | array[string] | Y | 待重排的文档列表(单次最多由服务端配置限制) |
  174 +| `normalize` | boolean | N | 是否对分数做 sigmoid 归一化,默认 true |
  175 +
  176 +**响应**:
  177 +```json
  178 +{
  179 + "scores": [0.92, 0.15],
  180 + "meta": {
  181 + "service_elapsed_ms": 45.2,
  182 + "input_docs": 2,
  183 + "unique_docs": 2
  184 + }
  185 +}
  186 +```
  187 +
  188 +**完整 curl 示例**:
  189 +```bash
  190 +curl -X POST "http://localhost:6007/rerank" \
  191 + -H "Content-Type: application/json" \
  192 + -d '{
  193 + "query": "玩具 芭比",
  194 + "docs": ["12PCS 6 Types of Dolls with Bottles", "纯棉T恤 短袖"],
  195 + "top_n":386,
  196 + "normalize": true
  197 + }'
  198 +```
  199 +
  200 +#### 7.2.2 `GET /health` — 健康检查
  201 +
  202 +```bash
  203 +curl "http://localhost:6007/health"
  204 +```
  205 +
  206 +### 7.3 翻译服务(Translation)
  207 +
  208 +- **Base URL**: `http://localhost:6006`(以 `config/config.yaml -> services.translation.service_url` 为准)
  209 +- **启动**: `./scripts/start_translator.sh`
  210 +
  211 +#### 7.3.1 `POST /translate` — 文本翻译
  212 +
  213 +支持 translator service 内所有已启用 capability,适用于商品名称、描述、query 等电商场景。当前可配置能力包括 `qwen-mt`、`llm`、`deepl` 以及本地模型 `nllb-200-distilled-600m`、`opus-mt-zh-en`、`opus-mt-en-zh`。
  214 +
  215 +**请求体**(支持单条字符串或字符串列表):
  216 +```json
  217 +{
  218 + "text": "商品名称",
  219 + "target_lang": "en",
  220 + "source_lang": "zh",
  221 + "model": "qwen-mt",
  222 + "scene": "sku_name"
  223 +}
  224 +```
  225 +
  226 +也支持批量列表形式:
  227 +```json
  228 +{
  229 + "text": ["商品名称1", "商品名称2"],
  230 + "target_lang": "en",
  231 + "source_lang": "zh",
  232 + "model": "qwen-mt",
  233 + "scene": "sku_name"
  234 +}
  235 +```
  236 +
  237 +| 参数 | 类型 | 必填 | 说明 |
  238 +|------|------|------|------|
  239 +| `text` | string \| string[] | Y | 待翻译文本,既支持单条字符串,也支持字符串列表(批量翻译) |
  240 +| `target_lang` | string | Y | 目标语言:`zh`、`en`、`ru` 等 |
  241 +| `source_lang` | string | N | 源语言。云端模型可不传;`nllb-200-distilled-600m` 建议显式传入 |
  242 +| `model` | string | N | 已启用 capability 名称,如 `qwen-mt`、`llm`、`deepl`、`nllb-200-distilled-600m`、`opus-mt-zh-en`、`opus-mt-en-zh` |
  243 +| `scene` | string | N | 翻译场景参数,与 `model` 配套使用;当前标准值为 `sku_name`、`ecommerce_search_query`、`general` |
  244 +
  245 +说明:
  246 +- 外部接口不接受 `prompt`;LLM prompt 由服务端按 `scene` 自动生成。
  247 +- 传入未定义的 `scene` 或未启用的 `model` 会返回 `400`。
  248 +
  249 +**SKU 名称场景选型建议**:
  250 +- 批量 SKU 名称翻译,优先考虑本地大吞吐方案时,可使用 `"model": "nllb-200-distilled-600m"`(该模型"scene":参数无效)。
  251 +- 如果目标是更高质量,且可以接受更慢速度与额外 LLM API 费用,可使用 `"model": "llm"` + `"scene": "sku_name"`。
  252 +- 如果是en-zh互译、期待更高的速度,可以考虑`opus-mt-zh-en` / `opus-mt-en-zh`。(质量未详细评测,一些文章说比blib-200-600m更好,但是我看了些case感觉要差不少)
  253 +
  254 +**实时翻译选型建议**:
  255 +- 在线 query 翻译如果只是 `en/zh` 互译,优先使用 `opus-mt-zh-en` 或 `opus-mt-en-zh`,它们是当前已测本地模型里延迟最低的一档。
  256 +- 如果涉及其他语言,或对质量要求高于本地轻量模型,优先考虑 `deepl`。
  257 +- `nllb-200-distilled-600m` 不建议作为在线 query 翻译默认方案;我们在 `Tesla T4` 上测到 `batch_size=1` 时,`zh -> en` p50 约 `292.54 ms`、p95 约 `624.12 ms`,`en -> zh` p50 约 `481.61 ms`、p95 约 `1171.71 ms`。
  258 +
  259 +**Batch Size / 调用方式建议**:
  260 +- 本接口支持 `text: string[]`;离线或批量索引翻译时,应尽量合并请求,让底层 backend 发挥批处理能力。
  261 +- `nllb-200-distilled-600m` 在当前 `Tesla T4` 压测中,推荐配置是 `batch_size=16`、`max_new_tokens=64`、`attn_implementation=sdpa`;继续升到 `batch_size=32` 虽可能提高吞吐,但 tail latency 会明显变差。
  262 +- 在线 query 场景可直接把“单条请求”理解为 `batch_size=1`;更关注 request latency,而不是离线吞吐。
  263 +- `opus-mt-zh-en` / `opus-mt-en-zh` 当前生产配置也是 `batch_size=16`,适合作为中英互译的低延迟本地默认值;若走在线单条调用,同样按 `batch_size=1` 理解即可。
  264 +- `llm` 按单条请求即可。
  265 +
  266 +**响应**:
  267 +```json
  268 +{
  269 + "text": "商品名称",
  270 + "target_lang": "en",
  271 + "source_lang": "zh",
  272 + "translated_text": "Product name",
  273 + "status": "success",
  274 + "model": "qwen-mt",
  275 + "scene": "sku_name"
  276 +}
  277 +```
  278 +
  279 +当请求为列表形式时,`text` 与 `translated_text` 均为等长数组:
  280 +```json
  281 +{
  282 + "text": ["商品名称1", "商品名称2"],
  283 + "target_lang": "en",
  284 + "source_lang": "zh",
  285 + "translated_text": ["Product name 1", "Product name 2"],
  286 + "status": "success",
  287 + "model": "qwen-mt",
  288 + "scene": "sku_name"
  289 +}
  290 +```
  291 +
  292 +> **失败语义(批量)**:当 `text` 为列表时,如果其中某条翻译失败,对应位置返回 `null`(即 `translated_text[i] = null`),并保持数组长度与顺序不变;接口整体仍返回 `status="success"`,用于避免“部分失败”导致整批请求失败。
  293 +
  294 +> **实现提示(可忽略)**:服务端会尽可能使用底层 backend 的批量能力(若支持),否则自动拆分逐条翻译;无论采用哪种方式,上述批量契约保持一致。
  295 +
  296 +**完整 curl 示例**:
  297 +
  298 +中文 → 英文:
  299 +```bash
  300 +curl -X POST "http://localhost:6006/translate" \
  301 + -H "Content-Type: application/json" \
  302 + -d '{
  303 + "text": "商品名称",
  304 + "target_lang": "en",
  305 + "source_lang": "zh"
  306 + }'
  307 +```
  308 +
  309 +俄文 → 英文:
  310 +```bash
  311 +curl -X POST "http://localhost:6006/translate" \
  312 + -H "Content-Type: application/json" \
  313 + -d '{
  314 + "text": "Название товара",
  315 + "target_lang": "en",
  316 + "source_lang": "ru"
  317 + }'
  318 +```
  319 +
  320 +使用 DeepL 模型:
  321 +```bash
  322 +curl -X POST "http://localhost:6006/translate" \
  323 + -H "Content-Type: application/json" \
  324 + -d '{
  325 + "text": "商品名称",
  326 + "target_lang": "en",
  327 + "source_lang": "zh",
  328 + "model": "deepl"
  329 + }'
  330 +```
  331 +
  332 +使用本地 OPUS 模型(中文 → 英文):
  333 +```bash
  334 +curl -X POST "http://localhost:6006/translate" \
  335 + -H "Content-Type: application/json" \
  336 + -d '{
  337 + "text": "蓝牙耳机",
  338 + "target_lang": "en",
  339 + "source_lang": "zh",
  340 + "model": "opus-mt-zh-en",
  341 + "scene": "sku_name"
  342 + }'
  343 +```
  344 +
  345 +使用本地 NLLB 做 SKU 名称批量翻译:
  346 +```bash
  347 +curl -X POST "http://localhost:6006/translate" \
  348 + -H "Content-Type: application/json" \
  349 + -d '{
  350 + "text": ["商品名称1", "商品名称2", "商品名称3"],
  351 + "target_lang": "en",
  352 + "source_lang": "zh",
  353 + "model": "nllb-200-distilled-600m",
  354 + "scene": "sku_name"
  355 + }'
  356 +```
  357 +
  358 +使用 LLM 做高质量 SKU 名称翻译:
  359 +```bash
  360 +curl -X POST "http://localhost:6006/translate" \
  361 + -H "Content-Type: application/json" \
  362 + -d '{
  363 + "text": "男士偏光飞行员太阳镜",
  364 + "target_lang": "en",
  365 + "source_lang": "zh",
  366 + "model": "llm",
  367 + "scene": "sku_name"
  368 + }'
  369 +```
  370 +
  371 +#### 7.3.2 `GET /health` — 健康检查
  372 +
  373 +```bash
  374 +curl "http://localhost:6006/health"
  375 +```
  376 +
  377 +典型响应:
  378 +```json
  379 +{
  380 + "status": "healthy",
  381 + "service": "translation",
  382 + "default_model": "llm",
  383 + "default_scene": "general",
  384 + "available_models": ["qwen-mt", "llm", "opus-mt-zh-en"],
  385 + "enabled_capabilities": ["qwen-mt", "llm", "opus-mt-zh-en"],
  386 + "loaded_models": ["llm"]
  387 +}
  388 +```
  389 +
  390 +### 7.4 内容理解字段生成(Indexer 服务内)
  391 +
  392 +内容理解字段生成接口部署在 **Indexer 服务**(默认端口 6004)内,与「翻译、向量化」等独立端口微服务并列,供采用**微服务组合**方式的 indexer 调用。
  393 +
  394 +- **Base URL**: Indexer 服务地址,如 `http://localhost:6004`
  395 +- **路径**: `POST /indexer/enrich-content`
  396 +- **说明**: 根据商品标题批量生成 `qanchors`、`semantic_attributes`、`tags`,用于拼装 ES 文档。内部使用大模型(需配置 `DASHSCOPE_API_KEY`),支持多语言与 Redis 缓存;单次最多 50 条,建议批量调用以提升效率。
  397 +
  398 +请求/响应格式、示例及错误码见 [-05-索引接口(Indexer)](./搜索API对接指南-05-索引接口(Indexer).md#58-内容理解字段生成接口)。
  399 +
  400 +---
  401 +
docs/搜索API对接指南-08-数据模型与字段速查.md 0 → 100644
@@ -0,0 +1,97 @@ @@ -0,0 +1,97 @@
  1 +# 搜索API对接指南-08-数据模型与字段速查
  2 +
  3 +本篇覆盖原文第 9 章:商品字段定义、字段类型速查、常用字段列表、支持的分析器。
  4 +
  5 +## 9. 数据模型
  6 +
  7 +### 9.1 商品字段定义
  8 +
  9 +| 字段名 | 类型 | 描述 |
  10 +|--------|------|------|
  11 +| `tenant_id` | keyword | 租户ID(多租户隔离) |
  12 +| `spu_id` | keyword | SPU ID |
  13 +| `title.<lang>` | object/text | 商品标题(多语言对象,如 `title.zh`, `title.en`) |
  14 +| `brief.<lang>` | object/text | 商品短描述(多语言对象,如 `brief.zh`, `brief.en`) |
  15 +| `description.<lang>` | object/text | 商品详细描述(多语言对象,如 `description.zh`, `description.en`) |
  16 +| `vendor.<lang>` | object/text | 供应商/品牌(多语言对象,且带 keyword 子字段,如 `vendor.zh.keyword`) |
  17 +| `category_path.<lang>` | object/text | 类目路径(多语言对象,用于搜索,如 `category_path.zh`) |
  18 +| `category_name_text.<lang>` | object/text | 类目名称(多语言对象,用于搜索,如 `category_name_text.zh`) |
  19 +| `category_id` | keyword | 类目ID |
  20 +| `category_name` | keyword | 类目名称(用于过滤) |
  21 +| `category_level` | integer | 类目层级 |
  22 +| `category1_name`, `category2_name`, `category3_name` | keyword | 多级类目名称(用于过滤和分面) |
  23 +| `tags` | keyword | 标签(数组) |
  24 +| `specifications` | nested | 规格(嵌套对象数组) |
  25 +| `option1_name`, `option2_name`, `option3_name` | keyword | 选项名称 |
  26 +| `min_price`, `max_price` | float | 最低/最高价格 |
  27 +| `compare_at_price` | float | 原价 |
  28 +| `sku_prices` | float | SKU价格列表(数组) |
  29 +| `sku_weights` | long | SKU重量列表(数组) |
  30 +| `sku_weight_units` | keyword | SKU重量单位列表(数组) |
  31 +| `total_inventory` | long | 总库存 |
  32 +| `sales` | long | 销量(展示销量) |
  33 +| `skus` | nested | SKU详细信息(嵌套对象数组) |
  34 +| `create_time`, `update_time` | date | 创建/更新时间 |
  35 +| `title_embedding` | dense_vector | 标题向量(1024维,仅用于搜索) |
  36 +| `image_embedding` | nested | 图片向量(嵌套,仅用于搜索) |
  37 +
  38 +> 所有租户共享统一的索引结构。文本字段支持中英文双语,后端根据 `language` 参数自动选择对应字段返回。
  39 +
  40 +### 9.2 字段类型速查
  41 +
  42 +| 类型 | ES Mapping | 用途 |
  43 +|------|------------|------|
  44 +| `text` | `text` | 全文检索(支持中英文分析器) |
  45 +| `keyword` | `keyword` | 精确匹配、聚合、排序 |
  46 +| `integer` | `integer` | 整数 |
  47 +| `long` | `long` | 长整数 |
  48 +| `float` | `float` | 浮点数 |
  49 +| `date` | `date` | 日期时间 |
  50 +| `nested` | `nested` | 嵌套对象(specifications, skus, image_embedding) |
  51 +| `dense_vector` | `dense_vector` | 向量字段(title_embedding,仅用于搜索) |
  52 +
  53 +### 9.3 常用字段列表
  54 +
  55 +#### 过滤字段
  56 +
  57 +- `category_name`: 类目名称
  58 +- `category1_name`, `category2_name`, `category3_name`: 多级类目
  59 +- `category_id`: 类目ID
  60 +- `vendor.zh.keyword`, `vendor.en.keyword`: 供应商/品牌(使用keyword子字段)
  61 +- `tags`: 标签(keyword类型)
  62 +- `option1_name`, `option2_name`, `option3_name`: 选项名称
  63 +- `specifications`: 规格过滤(嵌套字段,格式见[过滤器详解](./搜索API对接指南-01-搜索接口.md#33-过滤器详解))
  64 +
  65 +#### 范围字段
  66 +
  67 +- `min_price`: 最低价格
  68 +- `max_price`: 最高价格
  69 +- `compare_at_price`: 原价
  70 +- `create_time`: 创建时间
  71 +- `update_time`: 更新时间
  72 +
  73 +#### 排序字段
  74 +
  75 +- `price`: 价格(后端自动根据sort_order映射:asc→min_price,desc→max_price)
  76 +- `sales`: 销量
  77 +- `create_time`: 创建时间
  78 +- `update_time`: 更新时间
  79 +- `relevance_score`: 相关性分数(默认,不指定sort_by时使用)
  80 +
  81 +**注意**: 前端只需传 `price`,后端会自动处理:
  82 +- `sort_by: "price"` + `sort_order: "asc"` → 按 `min_price` 升序(价格从低到高)
  83 +- `sort_by: "price"` + `sort_order: "desc"` → 按 `max_price` 降序(价格从高到低)
  84 +
  85 +### 9.4 支持的分析器
  86 +
  87 +| 分析器 | 语言 | 描述 |
  88 +|--------|------|------|
  89 +| `index_ik` | 中文 | 中文索引分析器(用于中文字段) |
  90 +| `query_ik` | 中文 | 中文查询分析器(用于中文字段) |
  91 +| `hanlp_index` ⚠️ TODO(暂不支持) | 中文 | 中文索引分析器(用于中文字段) |
  92 +| `hanlp_standard` ⚠️ TODO(暂不支持) | 中文 | 中文查询分析器(用于中文字段) |
  93 +| `english` | 英文 | 标准英文分析器(用于英文字段) |
  94 +| `lowercase` | - | 小写标准化器(用于keyword子字段) |
  95 +
  96 +---
  97 +
docs/搜索API对接指南-10-接口级压测脚本.md 0 → 100644
@@ -0,0 +1,61 @@ @@ -0,0 +1,61 @@
  1 +# 搜索API对接指南-10-接口级压测脚本
  2 +
  3 +原文第 10 章:压测脚本与用例。
  4 +
  5 +## 10. 接口级压测脚本
  6 +
  7 +仓库提供统一压测脚本:`scripts/perf_api_benchmark.py`,用于对以下接口做并发压测:
  8 +
  9 +- 后端搜索:`POST /search/`
  10 +- 搜索建议:`GET /search/suggestions`
  11 +- 向量服务:`POST /embed/text`
  12 +- 翻译服务:`POST /translate`
  13 +- 重排服务:`POST /rerank`
  14 +
  15 +说明:脚本对 `embed_text` 场景会校验返回向量内容有效性(必须是有限数值,不允许 `null/NaN/Inf`),不是只看 HTTP 200。
  16 +
  17 +### 10.1 快速示例
  18 +
  19 +```bash
  20 +# suggest 压测(tenant 162)
  21 +python scripts/perf_api_benchmark.py \
  22 + --scenario backend_suggest \
  23 + --tenant-id 162 \
  24 + --duration 30 \
  25 + --concurrency 50
  26 +
  27 +# search 压测
  28 +python scripts/perf_api_benchmark.py \
  29 + --scenario backend_search \
  30 + --tenant-id 162 \
  31 + --duration 30 \
  32 + --concurrency 20
  33 +
  34 +# 全链路压测(search + suggest + embedding + translate + rerank)
  35 +python scripts/perf_api_benchmark.py \
  36 + --scenario all \
  37 + --tenant-id 162 \
  38 + --duration 60 \
  39 + --concurrency 30 \
  40 + --output perf_reports/all.json
  41 +```
  42 +
  43 +### 10.2 自定义用例
  44 +
  45 +可通过 `--cases-file` 覆盖默认请求模板。示例文件:
  46 +
  47 +```bash
  48 +scripts/perf_cases.json.example
  49 +```
  50 +
  51 +执行示例:
  52 +
  53 +```bash
  54 +python scripts/perf_api_benchmark.py \
  55 + --scenario all \
  56 + --tenant-id 162 \
  57 + --cases-file scripts/perf_cases.json.example \
  58 + --duration 60 \
  59 + --concurrency 40
  60 +```
  61 +
docs/搜索API对接指南.md renamed to docs/搜索API对接指南—拆分前版本存档.md
embeddings/config.py
1 -"""  
2 -Embedding module configuration. 1 +"""Embedding service compatibility config derived from unified app config."""
3 2
4 -This module is intentionally a plain Python file (no env var parsing, no extra deps).  
5 -Edit values here to configure:  
6 -- server host/port  
7 -- local model settings (paths/devices/batch sizes)  
8 -""" 3 +from __future__ import annotations
9 4
10 from typing import Optional 5 from typing import Optional
11 -import os 6 +
  7 +from config.loader import get_app_config
12 8
13 9
14 class EmbeddingConfig(object): 10 class EmbeddingConfig(object):
15 - # Server  
16 - HOST = os.getenv("EMBEDDING_HOST", "0.0.0.0")  
17 - PORT = int(os.getenv("EMBEDDING_PORT", 6005))  
18 -  
19 - # Text backend defaults  
20 - TEXT_MODEL_ID = os.getenv("TEXT_MODEL_ID", "Qwen/Qwen3-Embedding-0.6B")  
21 - # Keep TEXT_MODEL_DIR as an alias so code can refer to one canonical text model value.  
22 - TEXT_MODEL_DIR = TEXT_MODEL_ID  
23 - TEXT_DEVICE = os.getenv("TEXT_DEVICE", "cuda") # "cuda" or "cpu"  
24 - TEXT_BATCH_SIZE = int(os.getenv("TEXT_BATCH_SIZE", "32"))  
25 - TEXT_NORMALIZE_EMBEDDINGS = os.getenv("TEXT_NORMALIZE_EMBEDDINGS", "true").lower() in ("1", "true", "yes")  
26 - TEI_BASE_URL = os.getenv("TEI_BASE_URL", "http://127.0.0.1:8080")  
27 - TEI_TIMEOUT_SEC = int(os.getenv("TEI_TIMEOUT_SEC", "60"))  
28 -  
29 - # Image embeddings  
30 - # Option A: clip-as-service (Jina CLIP server, recommended)  
31 - USE_CLIP_AS_SERVICE = os.getenv("USE_CLIP_AS_SERVICE", "true").lower() in ("1", "true", "yes")  
32 - CLIP_AS_SERVICE_SERVER = os.getenv("CLIP_AS_SERVICE_SERVER", "grpc://127.0.0.1:51000")  
33 - CLIP_AS_SERVICE_MODEL_NAME = os.getenv("CLIP_AS_SERVICE_MODEL_NAME", "CN-CLIP/ViT-L-14")  
34 -  
35 - # Option B: local CN-CLIP (when USE_CLIP_AS_SERVICE=false)  
36 - IMAGE_MODEL_NAME = os.getenv("IMAGE_MODEL_NAME", "ViT-L-14")  
37 - IMAGE_DEVICE = None # type: Optional[str] # "cuda" / "cpu" / None(auto)  
38 -  
39 - # Service behavior  
40 - IMAGE_BATCH_SIZE = 8  
41 - IMAGE_NORMALIZE_EMBEDDINGS = os.getenv("IMAGE_NORMALIZE_EMBEDDINGS", "true").lower() in ("1", "true", "yes") 11 + def __init__(self) -> None:
  12 + app_config = get_app_config()
  13 + runtime = app_config.runtime
  14 + services = app_config.services.embedding
  15 + text_backend = services.get_backend_config()
  16 + image_backend = services.get_image_backend_config()
  17 +
  18 + self.HOST = runtime.embedding_host
  19 + self.PORT = runtime.embedding_port
  20 +
  21 + self.TEXT_MODEL_ID = str(text_backend.get("model_id") or "Qwen/Qwen3-Embedding-0.6B")
  22 + self.TEXT_MODEL_DIR = self.TEXT_MODEL_ID
  23 + self.TEXT_DEVICE = str(text_backend.get("device") or "cuda")
  24 + self.TEXT_BATCH_SIZE = int(text_backend.get("batch_size", 32))
  25 + self.TEXT_NORMALIZE_EMBEDDINGS = bool(text_backend.get("normalize_embeddings", True))
  26 + self.TEI_BASE_URL = str(text_backend.get("base_url") or "http://127.0.0.1:8080")
  27 + self.TEI_TIMEOUT_SEC = int(text_backend.get("timeout_sec", 60))
  28 +
  29 + self.USE_CLIP_AS_SERVICE = services.image_backend == "clip_as_service"
  30 + self.CLIP_AS_SERVICE_SERVER = str(image_backend.get("server") or "grpc://127.0.0.1:51000")
  31 + self.CLIP_AS_SERVICE_MODEL_NAME = str(image_backend.get("model_name") or "CN-CLIP/ViT-L-14")
  32 +
  33 + self.IMAGE_MODEL_NAME = str(image_backend.get("model_name") or "ViT-L-14")
  34 + self.IMAGE_DEVICE = image_backend.get("device") # type: Optional[str]
  35 + self.IMAGE_BATCH_SIZE = int(image_backend.get("batch_size", 8))
  36 + self.IMAGE_NORMALIZE_EMBEDDINGS = bool(image_backend.get("normalize_embeddings", True))
42 37
43 38
44 CONFIG = EmbeddingConfig() 39 CONFIG = EmbeddingConfig()
embeddings/image_encoder.py
@@ -9,8 +9,8 @@ from PIL import Image @@ -9,8 +9,8 @@ from PIL import Image
9 9
10 logger = logging.getLogger(__name__) 10 logger = logging.getLogger(__name__)
11 11
  12 +from config.loader import get_app_config
12 from config.services_config import get_embedding_image_base_url 13 from config.services_config import get_embedding_image_base_url
13 -from config.env_config import REDIS_CONFIG  
14 from embeddings.cache_keys import build_image_cache_key 14 from embeddings.cache_keys import build_image_cache_key
15 from embeddings.redis_embedding_cache import RedisEmbeddingCache 15 from embeddings.redis_embedding_cache import RedisEmbeddingCache
16 16
@@ -24,10 +24,11 @@ class CLIPImageEncoder: @@ -24,10 +24,11 @@ class CLIPImageEncoder:
24 24
25 def __init__(self, service_url: Optional[str] = None): 25 def __init__(self, service_url: Optional[str] = None):
26 resolved_url = service_url or get_embedding_image_base_url() 26 resolved_url = service_url or get_embedding_image_base_url()
  27 + redis_config = get_app_config().infrastructure.redis
27 self.service_url = str(resolved_url).rstrip("/") 28 self.service_url = str(resolved_url).rstrip("/")
28 self.endpoint = f"{self.service_url}/embed/image" 29 self.endpoint = f"{self.service_url}/embed/image"
29 # Reuse embedding cache prefix, but separate namespace for images to avoid collisions. 30 # Reuse embedding cache prefix, but separate namespace for images to avoid collisions.
30 - self.cache_prefix = str(REDIS_CONFIG.get("embedding_cache_prefix", "embedding")).strip() or "embedding" 31 + self.cache_prefix = str(redis_config.embedding_cache_prefix).strip() or "embedding"
31 logger.info("Creating CLIPImageEncoder instance with service URL: %s", self.service_url) 32 logger.info("Creating CLIPImageEncoder instance with service URL: %s", self.service_url)
32 self.cache = RedisEmbeddingCache( 33 self.cache = RedisEmbeddingCache(
33 key_prefix=self.cache_prefix, 34 key_prefix=self.cache_prefix,
embeddings/redis_embedding_cache.py
@@ -20,7 +20,7 @@ try: @@ -20,7 +20,7 @@ try:
20 except ImportError: # pragma: no cover - runtime fallback for minimal envs 20 except ImportError: # pragma: no cover - runtime fallback for minimal envs
21 redis = None # type: ignore[assignment] 21 redis = None # type: ignore[assignment]
22 22
23 -from config.env_config import REDIS_CONFIG 23 +from config.loader import get_app_config
24 from embeddings.bf16 import decode_embedding_from_redis, encode_embedding_for_redis 24 from embeddings.bf16 import decode_embedding_from_redis, encode_embedding_for_redis
25 25
26 logger = logging.getLogger(__name__) 26 logger = logging.getLogger(__name__)
@@ -37,7 +37,8 @@ class RedisEmbeddingCache: @@ -37,7 +37,8 @@ class RedisEmbeddingCache:
37 ): 37 ):
38 self.key_prefix = (key_prefix or "").strip() or "embedding" 38 self.key_prefix = (key_prefix or "").strip() or "embedding"
39 self.namespace = (namespace or "").strip() 39 self.namespace = (namespace or "").strip()
40 - self.expire_time = expire_time or timedelta(days=REDIS_CONFIG.get("cache_expire_days", 180)) 40 + redis_config = get_app_config().infrastructure.redis
  41 + self.expire_time = expire_time or timedelta(days=redis_config.cache_expire_days)
41 42
42 if redis_client is not None: 43 if redis_client is not None:
43 self.redis_client = redis_client 44 self.redis_client = redis_client
@@ -50,13 +51,13 @@ class RedisEmbeddingCache: @@ -50,13 +51,13 @@ class RedisEmbeddingCache:
50 51
51 try: 52 try:
52 client = redis.Redis( 53 client = redis.Redis(
53 - host=REDIS_CONFIG.get("host", "localhost"),  
54 - port=REDIS_CONFIG.get("port", 6479),  
55 - password=REDIS_CONFIG.get("password"), 54 + host=redis_config.host,
  55 + port=redis_config.port,
  56 + password=redis_config.password,
56 decode_responses=False, 57 decode_responses=False,
57 - socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),  
58 - socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),  
59 - retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False), 58 + socket_timeout=redis_config.socket_timeout,
  59 + socket_connect_timeout=redis_config.socket_connect_timeout,
  60 + retry_on_timeout=redis_config.retry_on_timeout,
60 health_check_interval=10, 61 health_check_interval=10,
61 ) 62 )
62 client.ping() 63 client.ping()
embeddings/server.py
@@ -470,16 +470,8 @@ def load_models(): @@ -470,16 +470,8 @@ def load_models():
470 if backend_name == "tei": 470 if backend_name == "tei":
471 from embeddings.text_embedding_tei import TEITextModel 471 from embeddings.text_embedding_tei import TEITextModel
472 472
473 - base_url = (  
474 - os.getenv("TEI_BASE_URL")  
475 - or backend_cfg.get("base_url")  
476 - or CONFIG.TEI_BASE_URL  
477 - )  
478 - timeout_sec = int(  
479 - os.getenv("TEI_TIMEOUT_SEC")  
480 - or backend_cfg.get("timeout_sec")  
481 - or CONFIG.TEI_TIMEOUT_SEC  
482 - ) 473 + base_url = backend_cfg.get("base_url") or CONFIG.TEI_BASE_URL
  474 + timeout_sec = int(backend_cfg.get("timeout_sec") or CONFIG.TEI_TIMEOUT_SEC)
483 logger.info("Loading text backend: tei (base_url=%s)", base_url) 475 logger.info("Loading text backend: tei (base_url=%s)", base_url)
484 _text_model = TEITextModel( 476 _text_model = TEITextModel(
485 base_url=str(base_url), 477 base_url=str(base_url),
@@ -488,11 +480,7 @@ def load_models(): @@ -488,11 +480,7 @@ def load_models():
488 elif backend_name == "local_st": 480 elif backend_name == "local_st":
489 from embeddings.text_embedding_sentence_transformers import Qwen3TextModel 481 from embeddings.text_embedding_sentence_transformers import Qwen3TextModel
490 482
491 - model_id = (  
492 - os.getenv("TEXT_MODEL_ID")  
493 - or backend_cfg.get("model_id")  
494 - or CONFIG.TEXT_MODEL_ID  
495 - ) 483 + model_id = backend_cfg.get("model_id") or CONFIG.TEXT_MODEL_ID
496 logger.info("Loading text backend: local_st (model=%s)", model_id) 484 logger.info("Loading text backend: local_st (model=%s)", model_id)
497 _text_model = Qwen3TextModel(model_id=str(model_id)) 485 _text_model = Qwen3TextModel(model_id=str(model_id))
498 _start_text_batch_worker() 486 _start_text_batch_worker()
embeddings/text_encoder.py
@@ -9,13 +9,11 @@ import requests @@ -9,13 +9,11 @@ import requests
9 9
10 logger = logging.getLogger(__name__) 10 logger = logging.getLogger(__name__)
11 11
  12 +from config.loader import get_app_config
12 from config.services_config import get_embedding_text_base_url 13 from config.services_config import get_embedding_text_base_url
13 from embeddings.cache_keys import build_text_cache_key 14 from embeddings.cache_keys import build_text_cache_key
14 from embeddings.redis_embedding_cache import RedisEmbeddingCache 15 from embeddings.redis_embedding_cache import RedisEmbeddingCache
15 16
16 -# Try to import REDIS_CONFIG, but allow import to fail  
17 -from config.env_config import REDIS_CONFIG  
18 -  
19 17
20 class TextEmbeddingEncoder: 18 class TextEmbeddingEncoder:
21 """ 19 """
@@ -24,10 +22,11 @@ class TextEmbeddingEncoder: @@ -24,10 +22,11 @@ class TextEmbeddingEncoder:
24 22
25 def __init__(self, service_url: Optional[str] = None): 23 def __init__(self, service_url: Optional[str] = None):
26 resolved_url = service_url or get_embedding_text_base_url() 24 resolved_url = service_url or get_embedding_text_base_url()
  25 + redis_config = get_app_config().infrastructure.redis
27 self.service_url = str(resolved_url).rstrip("/") 26 self.service_url = str(resolved_url).rstrip("/")
28 self.endpoint = f"{self.service_url}/embed/text" 27 self.endpoint = f"{self.service_url}/embed/text"
29 - self.expire_time = timedelta(days=REDIS_CONFIG.get("cache_expire_days", 180))  
30 - self.cache_prefix = str(REDIS_CONFIG.get("embedding_cache_prefix", "embedding")).strip() or "embedding" 28 + self.expire_time = timedelta(days=redis_config.cache_expire_days)
  29 + self.cache_prefix = str(redis_config.embedding_cache_prefix).strip() or "embedding"
31 logger.info("Creating TextEmbeddingEncoder instance with service URL: %s", self.service_url) 30 logger.info("Creating TextEmbeddingEncoder instance with service URL: %s", self.service_url)
32 31
33 self.cache = RedisEmbeddingCache( 32 self.cache = RedisEmbeddingCache(
indexer/document_transformer.py
@@ -13,7 +13,6 @@ import numpy as np @@ -13,7 +13,6 @@ import numpy as np
13 import logging 13 import logging
14 import re 14 import re
15 from typing import Dict, Any, Optional, List 15 from typing import Dict, Any, Optional, List
16 -from config import ConfigLoader  
17 from indexer.product_enrich import analyze_products 16 from indexer.product_enrich import analyze_products
18 17
19 logger = logging.getLogger(__name__) 18 logger = logging.getLogger(__name__)
indexer/incremental_service.py
@@ -13,7 +13,7 @@ from indexer.mapping_generator import get_tenant_index_name @@ -13,7 +13,7 @@ from indexer.mapping_generator import get_tenant_index_name
13 from indexer.indexer_logger import ( 13 from indexer.indexer_logger import (
14 get_indexer_logger, log_index_request, log_index_result, log_spu_processing 14 get_indexer_logger, log_index_request, log_index_result, log_spu_processing
15 ) 15 )
16 -from config import ConfigLoader 16 +from config import get_app_config
17 from translation import create_translation_client 17 from translation import create_translation_client
18 18
19 # Configure logger 19 # Configure logger
@@ -51,7 +51,7 @@ class IncrementalIndexerService: @@ -51,7 +51,7 @@ class IncrementalIndexerService:
51 51
52 def _eager_init(self) -> None: 52 def _eager_init(self) -> None:
53 """Strict eager initialization. Any dependency failure should fail fast.""" 53 """Strict eager initialization. Any dependency failure should fail fast."""
54 - self._config = ConfigLoader("config/config.yaml").load_config() 54 + self._config = get_app_config().search
55 self._searchable_option_dimensions = ( 55 self._searchable_option_dimensions = (
56 getattr(self._config.spu_config, "searchable_option_dimensions", None) 56 getattr(self._config.spu_config, "searchable_option_dimensions", None)
57 or ["option1", "option2", "option3"] 57 or ["option1", "option2", "option3"]
indexer/indexing_utils.py
@@ -7,7 +7,7 @@ @@ -7,7 +7,7 @@
7 import logging 7 import logging
8 from typing import Dict, Any, Optional 8 from typing import Dict, Any, Optional
9 from sqlalchemy import Engine, text 9 from sqlalchemy import Engine, text
10 -from config import ConfigLoader 10 +from config import get_app_config
11 from config.tenant_config_loader import get_tenant_config_loader 11 from config.tenant_config_loader import get_tenant_config_loader
12 from indexer.document_transformer import SPUDocumentTransformer 12 from indexer.document_transformer import SPUDocumentTransformer
13 from translation import create_translation_client 13 from translation import create_translation_client
@@ -92,8 +92,7 @@ def create_document_transformer( @@ -92,8 +92,7 @@ def create_document_transformer(
92 or config is None 92 or config is None
93 ): 93 ):
94 if config is None: 94 if config is None:
95 - config_loader = ConfigLoader()  
96 - config = config_loader.load_config() 95 + config = get_app_config().search
97 96
98 if searchable_option_dimensions is None: 97 if searchable_option_dimensions is None:
99 searchable_option_dimensions = config.spu_config.searchable_option_dimensions 98 searchable_option_dimensions = config.spu_config.searchable_option_dimensions
indexer/mapping_generator.py
@@ -9,7 +9,7 @@ import json @@ -9,7 +9,7 @@ import json
9 import logging 9 import logging
10 from pathlib import Path 10 from pathlib import Path
11 11
12 -from config.env_config import ES_INDEX_NAMESPACE 12 +from config.loader import get_app_config
13 13
14 logger = logging.getLogger(__name__) 14 logger = logging.getLogger(__name__)
15 15
@@ -30,7 +30,7 @@ def get_tenant_index_name(tenant_id: str) -&gt; str: @@ -30,7 +30,7 @@ def get_tenant_index_name(tenant_id: str) -&gt; str:
30 其中 ES_INDEX_NAMESPACE 由 config.env_config.ES_INDEX_NAMESPACE 控制, 30 其中 ES_INDEX_NAMESPACE 由 config.env_config.ES_INDEX_NAMESPACE 控制,
31 用于区分 prod/uat/test 等不同运行环境。 31 用于区分 prod/uat/test 等不同运行环境。
32 """ 32 """
33 - prefix = ES_INDEX_NAMESPACE or "" 33 + prefix = get_app_config().runtime.index_namespace or ""
34 return f"{prefix}search_products_tenant_{tenant_id}" 34 return f"{prefix}search_products_tenant_{tenant_id}"
35 35
36 36
indexer/product_enrich.py
@@ -12,15 +12,18 @@ import logging @@ -12,15 +12,18 @@ import logging
12 import re 12 import re
13 import time 13 import time
14 import hashlib 14 import hashlib
  15 +import uuid
  16 +import threading
15 from collections import OrderedDict 17 from collections import OrderedDict
16 from datetime import datetime 18 from datetime import datetime
  19 +from concurrent.futures import ThreadPoolExecutor
17 from typing import List, Dict, Tuple, Any, Optional 20 from typing import List, Dict, Tuple, Any, Optional
18 21
19 import redis 22 import redis
20 import requests 23 import requests
21 from pathlib import Path 24 from pathlib import Path
22 25
23 -from config.env_config import REDIS_CONFIG 26 +from config.loader import get_app_config
24 from config.tenant_config_loader import SOURCE_LANG_CODE_MAP 27 from config.tenant_config_loader import SOURCE_LANG_CODE_MAP
25 from indexer.product_enrich_prompts import ( 28 from indexer.product_enrich_prompts import (
26 SYSTEM_MESSAGE, 29 SYSTEM_MESSAGE,
@@ -31,6 +34,9 @@ from indexer.product_enrich_prompts import ( @@ -31,6 +34,9 @@ from indexer.product_enrich_prompts import (
31 34
32 # 配置 35 # 配置
33 BATCH_SIZE = 20 36 BATCH_SIZE = 20
  37 +# enrich-content LLM 批次并发 worker 上限(线程池;仅对 uncached batch 并发)
  38 +_APP_CONFIG = get_app_config()
  39 +CONTENT_UNDERSTANDING_MAX_WORKERS = int(_APP_CONFIG.product_enrich.max_workers)
34 # 华北2(北京):https://dashscope.aliyuncs.com/compatible-mode/v1 40 # 华北2(北京):https://dashscope.aliyuncs.com/compatible-mode/v1
35 # 新加坡:https://dashscope-intl.aliyuncs.com/compatible-mode/v1 41 # 新加坡:https://dashscope-intl.aliyuncs.com/compatible-mode/v1
36 # 美国(弗吉尼亚):https://dashscope-us.aliyuncs.com/compatible-mode/v1 42 # 美国(弗吉尼亚):https://dashscope-us.aliyuncs.com/compatible-mode/v1
@@ -56,6 +62,24 @@ timestamp = datetime.now().strftime(&quot;%Y%m%d_%H%M%S&quot;) @@ -56,6 +62,24 @@ timestamp = datetime.now().strftime(&quot;%Y%m%d_%H%M%S&quot;)
56 log_file = LOG_DIR / f"product_enrich_{timestamp}.log" 62 log_file = LOG_DIR / f"product_enrich_{timestamp}.log"
57 verbose_log_file = LOG_DIR / "product_enrich_verbose.log" 63 verbose_log_file = LOG_DIR / "product_enrich_verbose.log"
58 _logged_shared_context_keys: "OrderedDict[str, None]" = OrderedDict() 64 _logged_shared_context_keys: "OrderedDict[str, None]" = OrderedDict()
  65 +_logged_shared_context_lock = threading.Lock()
  66 +
  67 +_content_understanding_executor: Optional[ThreadPoolExecutor] = None
  68 +_content_understanding_executor_lock = threading.Lock()
  69 +
  70 +
  71 +def _get_content_understanding_executor() -> ThreadPoolExecutor:
  72 + """
  73 + 使用模块级单例线程池,避免同一进程内多次请求叠加创建线程池导致并发失控。
  74 + """
  75 + global _content_understanding_executor
  76 + with _content_understanding_executor_lock:
  77 + if _content_understanding_executor is None:
  78 + _content_understanding_executor = ThreadPoolExecutor(
  79 + max_workers=CONTENT_UNDERSTANDING_MAX_WORKERS,
  80 + thread_name_prefix="product-enrich-llm",
  81 + )
  82 + return _content_understanding_executor
59 83
60 # 主日志 logger:执行流程、批次信息等 84 # 主日志 logger:执行流程、批次信息等
61 logger = logging.getLogger("product_enrich") 85 logger = logging.getLogger("product_enrich")
@@ -91,19 +115,20 @@ logger.info(&quot;Verbose LLM logs are written to: %s&quot;, verbose_log_file) @@ -91,19 +115,20 @@ logger.info(&quot;Verbose LLM logs are written to: %s&quot;, verbose_log_file)
91 115
92 116
93 # Redis 缓存(用于 anchors / 语义属性) 117 # Redis 缓存(用于 anchors / 语义属性)
94 -ANCHOR_CACHE_PREFIX = REDIS_CONFIG.get("anchor_cache_prefix", "product_anchors")  
95 -ANCHOR_CACHE_EXPIRE_DAYS = int(REDIS_CONFIG.get("anchor_cache_expire_days", 30)) 118 +_REDIS_CONFIG = _APP_CONFIG.infrastructure.redis
  119 +ANCHOR_CACHE_PREFIX = _REDIS_CONFIG.anchor_cache_prefix
  120 +ANCHOR_CACHE_EXPIRE_DAYS = int(_REDIS_CONFIG.anchor_cache_expire_days)
96 _anchor_redis: Optional[redis.Redis] = None 121 _anchor_redis: Optional[redis.Redis] = None
97 122
98 try: 123 try:
99 _anchor_redis = redis.Redis( 124 _anchor_redis = redis.Redis(
100 - host=REDIS_CONFIG.get("host", "localhost"),  
101 - port=REDIS_CONFIG.get("port", 6479),  
102 - password=REDIS_CONFIG.get("password"), 125 + host=_REDIS_CONFIG.host,
  126 + port=_REDIS_CONFIG.port,
  127 + password=_REDIS_CONFIG.password,
103 decode_responses=True, 128 decode_responses=True,
104 - socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),  
105 - socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),  
106 - retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False), 129 + socket_timeout=_REDIS_CONFIG.socket_timeout,
  130 + socket_connect_timeout=_REDIS_CONFIG.socket_connect_timeout,
  131 + retry_on_timeout=_REDIS_CONFIG.retry_on_timeout,
107 health_check_interval=10, 132 health_check_interval=10,
108 ) 133 )
109 _anchor_redis.ping() 134 _anchor_redis.ping()
@@ -242,19 +267,21 @@ def _hash_text(text: str) -&gt; str: @@ -242,19 +267,21 @@ def _hash_text(text: str) -&gt; str:
242 267
243 268
244 def _mark_shared_context_logged_once(shared_context_key: str) -> bool: 269 def _mark_shared_context_logged_once(shared_context_key: str) -> bool:
245 - if shared_context_key in _logged_shared_context_keys:  
246 - _logged_shared_context_keys.move_to_end(shared_context_key)  
247 - return False 270 + with _logged_shared_context_lock:
  271 + if shared_context_key in _logged_shared_context_keys:
  272 + _logged_shared_context_keys.move_to_end(shared_context_key)
  273 + return False
248 274
249 - _logged_shared_context_keys[shared_context_key] = None  
250 - if len(_logged_shared_context_keys) > LOGGED_SHARED_CONTEXT_CACHE_SIZE:  
251 - _logged_shared_context_keys.popitem(last=False)  
252 - return True 275 + _logged_shared_context_keys[shared_context_key] = None
  276 + if len(_logged_shared_context_keys) > LOGGED_SHARED_CONTEXT_CACHE_SIZE:
  277 + _logged_shared_context_keys.popitem(last=False)
  278 + return True
253 279
254 280
255 def reset_logged_shared_context_keys() -> None: 281 def reset_logged_shared_context_keys() -> None:
256 """测试辅助:清理已记录的共享 prompt key。""" 282 """测试辅助:清理已记录的共享 prompt key。"""
257 - _logged_shared_context_keys.clear() 283 + with _logged_shared_context_lock:
  284 + _logged_shared_context_keys.clear()
258 285
259 286
260 def create_prompt( 287 def create_prompt(
@@ -625,7 +652,9 @@ def process_batch( @@ -625,7 +652,9 @@ def process_batch(
625 "final_results": results_with_ids, 652 "final_results": results_with_ids,
626 } 653 }
627 654
628 - batch_log_file = LOG_DIR / f"batch_{batch_num:04d}_{timestamp}.json" 655 + # 并发写 batch json 日志时,保证文件名唯一避免覆盖
  656 + batch_call_id = uuid.uuid4().hex[:12]
  657 + batch_log_file = LOG_DIR / f"batch_{batch_num:04d}_{timestamp}_{batch_call_id}.json"
629 with open(batch_log_file, "w", encoding="utf-8") as f: 658 with open(batch_log_file, "w", encoding="utf-8") as f:
630 json.dump(batch_log, f, ensure_ascii=False, indent=2) 659 json.dump(batch_log, f, ensure_ascii=False, indent=2)
631 660
@@ -707,28 +736,70 @@ def analyze_products( @@ -707,28 +736,70 @@ def analyze_products(
707 bs = max(1, min(req_bs, BATCH_SIZE)) 736 bs = max(1, min(req_bs, BATCH_SIZE))
708 total_batches = (len(uncached_items) + bs - 1) // bs 737 total_batches = (len(uncached_items) + bs - 1) // bs
709 738
  739 + batch_jobs: List[Tuple[int, List[Tuple[int, Dict[str, str]]], List[Dict[str, str]]]] = []
710 for i in range(0, len(uncached_items), bs): 740 for i in range(0, len(uncached_items), bs):
711 batch_num = i // bs + 1 741 batch_num = i // bs + 1
712 batch_slice = uncached_items[i : i + bs] 742 batch_slice = uncached_items[i : i + bs]
713 batch = [item for _, item in batch_slice] 743 batch = [item for _, item in batch_slice]
  744 + batch_jobs.append((batch_num, batch_slice, batch))
  745 +
  746 + # 只有一个批次时走串行,减少线程池创建开销与日志/日志文件的不可控交织
  747 + if total_batches <= 1 or CONTENT_UNDERSTANDING_MAX_WORKERS <= 1:
  748 + for batch_num, batch_slice, batch in batch_jobs:
  749 + logger.info(
  750 + f"[analyze_products] Processing batch {batch_num}/{total_batches}, "
  751 + f"size={len(batch)}, target_lang={target_lang}"
  752 + )
  753 + batch_results = process_batch(batch, batch_num=batch_num, target_lang=target_lang)
  754 +
  755 + for (original_idx, product), item in zip(batch_slice, batch_results):
  756 + results_by_index[original_idx] = item
  757 + title_input = str(item.get("title_input") or "").strip()
  758 + if not title_input:
  759 + continue
  760 + if item.get("error"):
  761 + # 不缓存错误结果,避免放大临时故障
  762 + continue
  763 + try:
  764 + _set_cached_anchor_result(product, target_lang, item)
  765 + except Exception:
  766 + # 已在内部记录 warning
  767 + pass
  768 + else:
  769 + max_workers = min(CONTENT_UNDERSTANDING_MAX_WORKERS, len(batch_jobs))
714 logger.info( 770 logger.info(
715 - f"[analyze_products] Processing batch {batch_num}/{total_batches}, "  
716 - f"size={len(batch)}, target_lang={target_lang}" 771 + "[analyze_products] Using ThreadPoolExecutor for uncached batches: "
  772 + "max_workers=%s, total_batches=%s, bs=%s, target_lang=%s",
  773 + max_workers,
  774 + total_batches,
  775 + bs,
  776 + target_lang,
717 ) 777 )
718 - batch_results = process_batch(batch, batch_num=batch_num, target_lang=target_lang)  
719 778
720 - for (original_idx, product), item in zip(batch_slice, batch_results):  
721 - results_by_index[original_idx] = item  
722 - title_input = str(item.get("title_input") or "").strip()  
723 - if not title_input:  
724 - continue  
725 - if item.get("error"):  
726 - # 不缓存错误结果,避免放大临时故障  
727 - continue  
728 - try:  
729 - _set_cached_anchor_result(product, target_lang, item)  
730 - except Exception:  
731 - # 已在内部记录 warning  
732 - pass 779 + # 只把“LLM 调用 + markdown 解析”放到线程里;Redis get/set 保持在主线程,避免并发写入带来额外风险。
  780 + # 注意:线程池是模块级单例,因此这里的 max_workers 主要用于日志语义(实际并发受单例池上限约束)。
  781 + executor = _get_content_understanding_executor()
  782 + future_by_batch_num: Dict[int, Any] = {}
  783 + for batch_num, _batch_slice, batch in batch_jobs:
  784 + future_by_batch_num[batch_num] = executor.submit(
  785 + process_batch, batch, batch_num=batch_num, target_lang=target_lang
  786 + )
  787 +
  788 + # 按 batch_num 回填,确保输出稳定(results_by_index 是按原始 input index 映射的)
  789 + for batch_num, batch_slice, _batch in batch_jobs:
  790 + batch_results = future_by_batch_num[batch_num].result()
  791 + for (original_idx, product), item in zip(batch_slice, batch_results):
  792 + results_by_index[original_idx] = item
  793 + title_input = str(item.get("title_input") or "").strip()
  794 + if not title_input:
  795 + continue
  796 + if item.get("error"):
  797 + # 不缓存错误结果,避免放大临时故障
  798 + continue
  799 + try:
  800 + _set_cached_anchor_result(product, target_lang, item)
  801 + except Exception:
  802 + # 已在内部记录 warning
  803 + pass
733 804
734 return [item for item in results_by_index if item is not None] 805 return [item for item in results_by_index if item is not None]
@@ -16,8 +16,7 @@ import json @@ -16,8 +16,7 @@ import json
16 # Add parent directory to path 16 # Add parent directory to path
17 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) 17 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
18 18
19 -from config import ConfigLoader  
20 -from config.env_config import ES_CONFIG 19 +from config import get_app_config
21 from utils import ESClient 20 from utils import ESClient
22 from search import Searcher 21 from search import Searcher
23 from suggestion import SuggestionIndexBuilder 22 from suggestion import SuggestionIndexBuilder
@@ -61,8 +60,7 @@ def cmd_serve_indexer(args): @@ -61,8 +60,7 @@ def cmd_serve_indexer(args):
61 def cmd_search(args): 60 def cmd_search(args):
62 """Test search from command line.""" 61 """Test search from command line."""
63 # Load config 62 # Load config
64 - config_loader = ConfigLoader("config/config.yaml")  
65 - config = config_loader.load_config() 63 + config = get_app_config().search
66 64
67 # Initialize ES and searcher 65 # Initialize ES and searcher
68 es_client = ESClient(hosts=[args.es_host]) 66 es_client = ESClient(hosts=[args.es_host])
@@ -106,8 +104,9 @@ def cmd_search(args): @@ -106,8 +104,9 @@ def cmd_search(args):
106 def cmd_build_suggestions(args): 104 def cmd_build_suggestions(args):
107 """Build/update suggestion index for a tenant.""" 105 """Build/update suggestion index for a tenant."""
108 # Initialize ES client with optional authentication 106 # Initialize ES client with optional authentication
109 - es_username = os.getenv("ES_USERNAME") or ES_CONFIG.get("username")  
110 - es_password = os.getenv("ES_PASSWORD") or ES_CONFIG.get("password") 107 + es_cfg = get_app_config().infrastructure.elasticsearch
  108 + es_username = es_cfg.username
  109 + es_password = es_cfg.password
111 if es_username and es_password: 110 if es_username and es_password:
112 es_client = ESClient(hosts=[args.es_host], username=es_username, password=es_password) 111 es_client = ESClient(hosts=[args.es_host], username=es_username, password=es_password)
113 else: 112 else:
@@ -117,11 +116,12 @@ def cmd_build_suggestions(args): @@ -117,11 +116,12 @@ def cmd_build_suggestions(args):
117 return 1 116 return 1
118 117
119 # Build DB config directly from environment to avoid dotenv dependency 118 # Build DB config directly from environment to avoid dotenv dependency
120 - db_host = os.getenv("DB_HOST")  
121 - db_port = int(os.getenv("DB_PORT", "3306"))  
122 - db_name = os.getenv("DB_DATABASE")  
123 - db_user = os.getenv("DB_USERNAME")  
124 - db_pass = os.getenv("DB_PASSWORD") 119 + db_cfg = get_app_config().infrastructure.database
  120 + db_host = db_cfg.host
  121 + db_port = db_cfg.port
  122 + db_name = db_cfg.database
  123 + db_user = db_cfg.username
  124 + db_pass = db_cfg.password
125 if not all([db_host, db_name, db_user, db_pass]): 125 if not all([db_host, db_name, db_user, db_pass]):
126 print("ERROR: DB_HOST/DB_PORT/DB_DATABASE/DB_USERNAME/DB_PASSWORD must be set in environment") 126 print("ERROR: DB_HOST/DB_PORT/DB_DATABASE/DB_USERNAME/DB_PASSWORD must be set in environment")
127 return 1 127 return 1
@@ -170,7 +170,7 @@ def main(): @@ -170,7 +170,7 @@ def main():
170 serve_parser = subparsers.add_parser('serve', help='Start API service (multi-tenant)') 170 serve_parser = subparsers.add_parser('serve', help='Start API service (multi-tenant)')
171 serve_parser.add_argument('--host', default='0.0.0.0', help='Host to bind to') 171 serve_parser.add_argument('--host', default='0.0.0.0', help='Host to bind to')
172 serve_parser.add_argument('--port', type=int, default=6002, help='Port to bind to') 172 serve_parser.add_argument('--port', type=int, default=6002, help='Port to bind to')
173 - serve_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host') 173 + serve_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
174 serve_parser.add_argument('--reload', action='store_true', help='Enable auto-reload') 174 serve_parser.add_argument('--reload', action='store_true', help='Enable auto-reload')
175 175
176 # Serve-indexer command 176 # Serve-indexer command
@@ -180,14 +180,14 @@ def main(): @@ -180,14 +180,14 @@ def main():
180 ) 180 )
181 serve_indexer_parser.add_argument('--host', default='0.0.0.0', help='Host to bind to') 181 serve_indexer_parser.add_argument('--host', default='0.0.0.0', help='Host to bind to')
182 serve_indexer_parser.add_argument('--port', type=int, default=6004, help='Port to bind to') 182 serve_indexer_parser.add_argument('--port', type=int, default=6004, help='Port to bind to')
183 - serve_indexer_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host') 183 + serve_indexer_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
184 serve_indexer_parser.add_argument('--reload', action='store_true', help='Enable auto-reload') 184 serve_indexer_parser.add_argument('--reload', action='store_true', help='Enable auto-reload')
185 185
186 # Search command 186 # Search command
187 search_parser = subparsers.add_parser('search', help='Test search from command line') 187 search_parser = subparsers.add_parser('search', help='Test search from command line')
188 search_parser.add_argument('query', help='Search query') 188 search_parser.add_argument('query', help='Search query')
189 search_parser.add_argument('--tenant-id', required=True, help='Tenant ID (required)') 189 search_parser.add_argument('--tenant-id', required=True, help='Tenant ID (required)')
190 - search_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host') 190 + search_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
191 search_parser.add_argument('--size', type=int, default=10, help='Number of results') 191 search_parser.add_argument('--size', type=int, default=10, help='Number of results')
192 search_parser.add_argument('--no-translation', action='store_true', help='Disable translation') 192 search_parser.add_argument('--no-translation', action='store_true', help='Disable translation')
193 search_parser.add_argument('--no-embedding', action='store_true', help='Disable embeddings') 193 search_parser.add_argument('--no-embedding', action='store_true', help='Disable embeddings')
@@ -199,7 +199,7 @@ def main(): @@ -199,7 +199,7 @@ def main():
199 help='Build tenant suggestion index (full/incremental)' 199 help='Build tenant suggestion index (full/incremental)'
200 ) 200 )
201 suggest_build_parser.add_argument('--tenant-id', required=True, help='Tenant ID') 201 suggest_build_parser.add_argument('--tenant-id', required=True, help='Tenant ID')
202 - suggest_build_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host') 202 + suggest_build_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
203 suggest_build_parser.add_argument( 203 suggest_build_parser.add_argument(
204 '--mode', 204 '--mode',
205 choices=['full', 'incremental'], 205 choices=['full', 'incremental'],
query/query_parser.py
@@ -336,13 +336,13 @@ class QueryParser: @@ -336,13 +336,13 @@ class QueryParser:
336 translations = {} 336 translations = {}
337 translation_futures = {} 337 translation_futures = {}
338 translation_executor = None 338 translation_executor = None
339 - index_langs = ["en", "zh"] 339 + index_langs: List[str] = []
340 try: 340 try:
341 # 根据租户配置的 index_languages 决定翻译目标语言 341 # 根据租户配置的 index_languages 决定翻译目标语言
342 from config.tenant_config_loader import get_tenant_config_loader 342 from config.tenant_config_loader import get_tenant_config_loader
343 tenant_loader = get_tenant_config_loader() 343 tenant_loader = get_tenant_config_loader()
344 tenant_cfg = tenant_loader.get_tenant_config(tenant_id or "default") 344 tenant_cfg = tenant_loader.get_tenant_config(tenant_id or "default")
345 - raw_index_langs = tenant_cfg.get("index_languages") or ["en", "zh"] 345 + raw_index_langs = tenant_cfg.get("index_languages") or []
346 index_langs = [] 346 index_langs = []
347 seen_langs = set() 347 seen_langs = set()
348 for lang in raw_index_langs: 348 for lang in raw_index_langs:
@@ -618,17 +618,3 @@ class QueryParser: @@ -618,17 +618,3 @@ class QueryParser:
618 queries.append(translation) 618 queries.append(translation)
619 619
620 return queries 620 return queries
621 -  
622 - def update_rewrite_rules(self, rules: Dict[str, str]) -> None:  
623 - """  
624 - Update query rewrite rules.  
625 -  
626 - Args:  
627 - rules: Dictionary of pattern -> replacement mappings  
628 - """  
629 - for pattern, replacement in rules.items():  
630 - self.rewriter.add_rule(pattern, replacement)  
631 -  
632 - def get_rewrite_rules(self) -> Dict[str, str]:  
633 - """Get current rewrite rules."""  
634 - return self.rewriter.get_rules()  
reranker/backends/dashscope_rerank.py
@@ -63,43 +63,19 @@ class DashScopeRerankBackend: @@ -63,43 +63,19 @@ class DashScopeRerankBackend:
63 - max_retries: int, default 1 63 - max_retries: int, default 1
64 - retry_backoff_sec: float, default 0.2 64 - retry_backoff_sec: float, default 0.2
65 65
66 - Env overrides:  
67 - - RERANK_DASHSCOPE_ENDPOINT  
68 - - RERANK_DASHSCOPE_MODEL  
69 - - RERANK_DASHSCOPE_TIMEOUT_SEC  
70 - - RERANK_DASHSCOPE_TOP_N_CAP  
71 - - RERANK_DASHSCOPE_BATCHSIZE  
72 """ 66 """
73 67
74 def __init__(self, config: Dict[str, Any]) -> None: 68 def __init__(self, config: Dict[str, Any]) -> None:
75 self._config = config or {} 69 self._config = config or {}
76 - self._model_name = str(  
77 - os.getenv("RERANK_DASHSCOPE_MODEL")  
78 - or self._config.get("model_name")  
79 - or "qwen3-rerank"  
80 - ) 70 + self._model_name = str(self._config.get("model_name") or "qwen3-rerank")
81 self._endpoint = str( 71 self._endpoint = str(
82 - os.getenv("RERANK_DASHSCOPE_ENDPOINT")  
83 - or self._config.get("endpoint")  
84 - or "https://dashscope.aliyuncs.com/compatible-api/v1/reranks" 72 + self._config.get("endpoint") or "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
85 ).strip() 73 ).strip()
86 self._api_key_env = str(self._config.get("api_key_env") or "").strip() 74 self._api_key_env = str(self._config.get("api_key_env") or "").strip()
87 self._api_key = str(os.getenv(self._api_key_env) or "").strip().strip('"').strip("'") 75 self._api_key = str(os.getenv(self._api_key_env) or "").strip().strip('"').strip("'")
88 - self._timeout_sec = float(  
89 - os.getenv("RERANK_DASHSCOPE_TIMEOUT_SEC")  
90 - or self._config.get("timeout_sec")  
91 - or 15.0  
92 - )  
93 - self._top_n_cap = int(  
94 - os.getenv("RERANK_DASHSCOPE_TOP_N_CAP")  
95 - or self._config.get("top_n_cap")  
96 - or 0  
97 - )  
98 - self._batchsize = int(  
99 - os.getenv("RERANK_DASHSCOPE_BATCHSIZE")  
100 - or self._config.get("batchsize")  
101 - or 0  
102 - ) 76 + self._timeout_sec = float(self._config.get("timeout_sec") or 15.0)
  77 + self._top_n_cap = int(self._config.get("top_n_cap") or 0)
  78 + self._batchsize = int(self._config.get("batchsize") or 0)
103 self._instruct = str(self._config.get("instruct") or "").strip() 79 self._instruct = str(self._config.get("instruct") or "").strip()
104 self._max_retries = int(self._config.get("max_retries", 1)) 80 self._max_retries = int(self._config.get("max_retries", 1))
105 self._retry_backoff_sec = float(self._config.get("retry_backoff_sec", 0.2)) 81 self._retry_backoff_sec = float(self._config.get("retry_backoff_sec", 0.2))
reranker/config.py
1 -"""Reranker service configuration (simple Python config).""" 1 +"""Reranker service compatibility config derived from unified app config."""
2 2
3 -import os 3 +from __future__ import annotations
  4 +
  5 +from config.loader import get_app_config
4 6
5 7
6 class RerankerConfig(object): 8 class RerankerConfig(object):
7 - # Server  
8 - HOST = os.getenv("RERANKER_HOST", "0.0.0.0")  
9 - PORT = int(os.getenv("RERANKER_PORT", 6007))  
10 -  
11 - # Model  
12 - MODEL_NAME = "Qwen/Qwen3-Reranker-0.6B"  
13 - DEVICE = None # None -> auto (cuda if available)  
14 - USE_FP16 = True  
15 - BATCH_SIZE = 64  
16 - MAX_LENGTH = 512  
17 - CACHE_DIR = "./model_cache"  
18 - ENABLE_WARMUP = True  
19 -  
20 - # Request limits  
21 - MAX_DOCS = 1000  
22 -  
23 - # Output  
24 - NORMALIZE = True 9 + def __init__(self) -> None:
  10 + app_config = get_app_config()
  11 + runtime = app_config.runtime
  12 + service = app_config.services.rerank
  13 + backend = service.get_backend_config()
  14 + request = service.request
  15 +
  16 + self.HOST = runtime.reranker_host
  17 + self.PORT = runtime.reranker_port
  18 +
  19 + self.MODEL_NAME = str(backend.get("model_name") or "Qwen/Qwen3-Reranker-0.6B")
  20 + self.DEVICE = backend.get("device")
  21 + self.USE_FP16 = bool(backend.get("use_fp16", True))
  22 + self.BATCH_SIZE = int(backend.get("batch_size", backend.get("infer_batch_size", 64)))
  23 + self.MAX_LENGTH = int(backend.get("max_length", 512))
  24 + self.CACHE_DIR = str(backend.get("cache_dir") or "./model_cache")
  25 + self.ENABLE_WARMUP = bool(backend.get("enable_warmup", True))
  26 +
  27 + self.MAX_DOCS = int(request.get("max_docs", 1000))
  28 + self.NORMALIZE = bool(request.get("normalize", True))
25 29
26 30
27 CONFIG = RerankerConfig() 31 CONFIG = RerankerConfig()
suggestion/builder.py
@@ -18,7 +18,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple @@ -18,7 +18,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple
18 18
19 from sqlalchemy import text 19 from sqlalchemy import text
20 20
21 -from config.env_config import ES_INDEX_NAMESPACE 21 +from config.loader import get_app_config
22 from config.tenant_config_loader import get_tenant_config_loader 22 from config.tenant_config_loader import get_tenant_config_loader
23 from suggestion.mapping import build_suggestion_mapping 23 from suggestion.mapping import build_suggestion_mapping
24 from utils.es_client import ESClient 24 from utils.es_client import ESClient
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
27 27
28 28
29 def _index_prefix() -> str: 29 def _index_prefix() -> str:
30 - return ES_INDEX_NAMESPACE or "" 30 + return get_app_config().runtime.index_namespace or ""
31 31
32 32
33 def get_suggestion_alias_name(tenant_id: str) -> str: 33 def get_suggestion_alias_name(tenant_id: str) -> str:
tests/test_process_products_batching.py
@@ -45,7 +45,8 @@ def test_analyze_products_caps_batch_size_to_20(monkeypatch): @@ -45,7 +45,8 @@ def test_analyze_products_caps_batch_size_to_20(monkeypatch):
45 ) 45 )
46 46
47 assert len(out) == 45 47 assert len(out) == 45
48 - assert seen_batch_sizes == [20, 20, 5] 48 + # 并发执行时 batch 调用顺序可能变化,因此校验“批大小集合”而不是严格顺序
  49 + assert sorted(seen_batch_sizes) == [5, 20, 20]
49 50
50 51
51 def test_analyze_products_uses_min_batch_size_1(monkeypatch): 52 def test_analyze_products_uses_min_batch_size_1(monkeypatch):
translation/backends/deepl.py
@@ -3,7 +3,6 @@ @@ -3,7 +3,6 @@
3 from __future__ import annotations 3 from __future__ import annotations
4 4
5 import logging 5 import logging
6 -import os  
7 import re 6 import re
8 from typing import List, Optional, Sequence, Tuple, Union 7 from typing import List, Optional, Sequence, Tuple, Union
9 8
@@ -24,7 +23,7 @@ class DeepLTranslationBackend: @@ -24,7 +23,7 @@ class DeepLTranslationBackend:
24 timeout: float, 23 timeout: float,
25 glossary_id: Optional[str] = None, 24 glossary_id: Optional[str] = None,
26 ) -> None: 25 ) -> None:
27 - self.api_key = api_key or os.getenv("DEEPL_AUTH_KEY") 26 + self.api_key = api_key
28 self.api_url = api_url 27 self.api_url = api_url
29 self.timeout = float(timeout) 28 self.timeout = float(timeout)
30 self.glossary_id = glossary_id 29 self.glossary_id = glossary_id
translation/backends/llm.py
@@ -3,13 +3,11 @@ @@ -3,13 +3,11 @@
3 from __future__ import annotations 3 from __future__ import annotations
4 4
5 import logging 5 import logging
6 -import os  
7 import time 6 import time
8 from typing import List, Optional, Sequence, Union 7 from typing import List, Optional, Sequence, Union
9 8
10 from openai import OpenAI 9 from openai import OpenAI
11 10
12 -from config.env_config import DASHSCOPE_API_KEY  
13 from translation.languages import LANGUAGE_LABELS 11 from translation.languages import LANGUAGE_LABELS
14 from translation.prompts import TRANSLATION_PROMPTS 12 from translation.prompts import TRANSLATION_PROMPTS
15 from translation.scenes import normalize_scene_name 13 from translation.scenes import normalize_scene_name
@@ -52,11 +50,13 @@ class LLMTranslationBackend: @@ -52,11 +50,13 @@ class LLMTranslationBackend:
52 model: str, 50 model: str,
53 timeout_sec: float, 51 timeout_sec: float,
54 base_url: str, 52 base_url: str,
  53 + api_key: Optional[str],
55 ) -> None: 54 ) -> None:
56 self.capability_name = capability_name 55 self.capability_name = capability_name
57 self.model = model 56 self.model = model
58 self.timeout_sec = float(timeout_sec) 57 self.timeout_sec = float(timeout_sec)
59 self.base_url = base_url 58 self.base_url = base_url
  59 + self.api_key = api_key
60 self.client = self._create_client() 60 self.client = self._create_client()
61 61
62 @property 62 @property
@@ -64,12 +64,11 @@ class LLMTranslationBackend: @@ -64,12 +64,11 @@ class LLMTranslationBackend:
64 return True 64 return True
65 65
66 def _create_client(self) -> Optional[OpenAI]: 66 def _create_client(self) -> Optional[OpenAI]:
67 - api_key = DASHSCOPE_API_KEY or os.getenv("DASHSCOPE_API_KEY")  
68 - if not api_key: 67 + if not self.api_key:
69 logger.warning("DASHSCOPE_API_KEY not set; llm translation unavailable") 68 logger.warning("DASHSCOPE_API_KEY not set; llm translation unavailable")
70 return None 69 return None
71 try: 70 try:
72 - return OpenAI(api_key=api_key, base_url=self.base_url) 71 + return OpenAI(api_key=self.api_key, base_url=self.base_url)
73 except Exception as exc: 72 except Exception as exc:
74 logger.error("Failed to initialize llm translation client: %s", exc, exc_info=True) 73 logger.error("Failed to initialize llm translation client: %s", exc, exc_info=True)
75 return None 74 return None
translation/backends/qwen_mt.py
@@ -3,14 +3,12 @@ @@ -3,14 +3,12 @@
3 from __future__ import annotations 3 from __future__ import annotations
4 4
5 import logging 5 import logging
6 -import os  
7 import re 6 import re
8 import time 7 import time
9 from typing import List, Optional, Sequence, Union 8 from typing import List, Optional, Sequence, Union
10 9
11 from openai import OpenAI 10 from openai import OpenAI
12 11
13 -from config.env_config import DASHSCOPE_API_KEY  
14 from translation.languages import QWEN_LANGUAGE_CODES 12 from translation.languages import QWEN_LANGUAGE_CODES
15 13
16 logger = logging.getLogger(__name__) 14 logger = logging.getLogger(__name__)
@@ -64,7 +62,7 @@ class QwenMTTranslationBackend: @@ -64,7 +62,7 @@ class QwenMTTranslationBackend:
64 @staticmethod 62 @staticmethod
65 def _default_api_key(model: str) -> Optional[str]: 63 def _default_api_key(model: str) -> Optional[str]:
66 del model 64 del model
67 - return DASHSCOPE_API_KEY or os.getenv("DASHSCOPE_API_KEY") 65 + return None
68 66
69 def translate( 67 def translate(
70 self, 68 self,
translation/cache.py
@@ -6,9 +6,12 @@ import hashlib @@ -6,9 +6,12 @@ import hashlib
6 import logging 6 import logging
7 from typing import Mapping, Optional 7 from typing import Mapping, Optional
8 8
9 -import redis 9 +try:
  10 + import redis
  11 +except ImportError: # pragma: no cover - runtime fallback for minimal envs
  12 + redis = None # type: ignore[assignment]
10 13
11 -from config.env_config import REDIS_CONFIG 14 +from config.loader import get_app_config
12 15
13 logger = logging.getLogger(__name__) 16 logger = logging.getLogger(__name__)
14 17
@@ -70,15 +73,19 @@ class TranslationCache: @@ -70,15 +73,19 @@ class TranslationCache:
70 73
71 @staticmethod 74 @staticmethod
72 def _init_redis_client() -> Optional[redis.Redis]: 75 def _init_redis_client() -> Optional[redis.Redis]:
  76 + if redis is None:
  77 + logger.warning("redis package is not installed; translation cache disabled")
  78 + return None
  79 + redis_config = get_app_config().infrastructure.redis
73 try: 80 try:
74 client = redis.Redis( 81 client = redis.Redis(
75 - host=REDIS_CONFIG.get("host", "localhost"),  
76 - port=REDIS_CONFIG.get("port", 6479),  
77 - password=REDIS_CONFIG.get("password"), 82 + host=redis_config.host,
  83 + port=redis_config.port,
  84 + password=redis_config.password,
78 decode_responses=True, 85 decode_responses=True,
79 - socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),  
80 - socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),  
81 - retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False), 86 + socket_timeout=redis_config.socket_timeout,
  87 + socket_connect_timeout=redis_config.socket_connect_timeout,
  88 + retry_on_timeout=redis_config.retry_on_timeout,
82 health_check_interval=10, 89 health_check_interval=10,
83 ) 90 )
84 client.ping() 91 client.ping()
translation/client.py
@@ -7,7 +7,7 @@ from typing import List, Optional, Sequence, Union @@ -7,7 +7,7 @@ from typing import List, Optional, Sequence, Union
7 7
8 import requests 8 import requests
9 9
10 -from config.services_config import get_translation_config 10 +from config.loader import get_app_config
11 from translation.settings import normalize_translation_model, normalize_translation_scene 11 from translation.settings import normalize_translation_model, normalize_translation_scene
12 12
13 logger = logging.getLogger(__name__) 13 logger = logging.getLogger(__name__)
@@ -24,7 +24,7 @@ class TranslationServiceClient: @@ -24,7 +24,7 @@ class TranslationServiceClient:
24 default_scene: Optional[str] = None, 24 default_scene: Optional[str] = None,
25 timeout_sec: Optional[float] = None, 25 timeout_sec: Optional[float] = None,
26 ) -> None: 26 ) -> None:
27 - cfg = get_translation_config() 27 + cfg = get_app_config().services.translation.as_dict()
28 self.base_url = str(base_url or cfg["service_url"]).rstrip("/") 28 self.base_url = str(base_url or cfg["service_url"]).rstrip("/")
29 self.default_model = normalize_translation_model(cfg, default_model or cfg["default_model"]) 29 self.default_model = normalize_translation_model(cfg, default_model or cfg["default_model"])
30 self.default_scene = normalize_translation_scene(cfg, default_scene or cfg["default_scene"]) 30 self.default_scene = normalize_translation_scene(cfg, default_scene or cfg["default_scene"])
translation/service.py
@@ -5,7 +5,8 @@ from __future__ import annotations @@ -5,7 +5,8 @@ from __future__ import annotations
5 import logging 5 import logging
6 from typing import Dict, List, Optional 6 from typing import Dict, List, Optional
7 7
8 -from config.services_config import get_translation_config 8 +from config.loader import get_app_config
  9 +from config.schema import AppConfig
9 from translation.cache import TranslationCache 10 from translation.cache import TranslationCache
10 from translation.protocols import TranslateInput, TranslateOutput, TranslationBackendProtocol 11 from translation.protocols import TranslateInput, TranslateOutput, TranslationBackendProtocol
11 from translation.settings import ( 12 from translation.settings import (
@@ -22,8 +23,9 @@ logger = logging.getLogger(__name__) @@ -22,8 +23,9 @@ logger = logging.getLogger(__name__)
22 class TranslationService: 23 class TranslationService:
23 """Owns translation backends and routes calls by model and scene.""" 24 """Owns translation backends and routes calls by model and scene."""
24 25
25 - def __init__(self, config: Optional[TranslationConfig] = None) -> None:  
26 - self.config = config or get_translation_config() 26 + def __init__(self, config: Optional[TranslationConfig] = None, app_config: Optional[AppConfig] = None) -> None:
  27 + self._app_config = app_config or get_app_config()
  28 + self.config = config or self._app_config.services.translation.as_dict()
27 self._enabled_capabilities = self._collect_enabled_capabilities() 29 self._enabled_capabilities = self._collect_enabled_capabilities()
28 if not self._enabled_capabilities: 30 if not self._enabled_capabilities:
29 raise ValueError("No enabled translation backends found in services.translation.capabilities") 31 raise ValueError("No enabled translation backends found in services.translation.capabilities")
@@ -85,7 +87,7 @@ class TranslationService: @@ -85,7 +87,7 @@ class TranslationService:
85 capability_name=name, 87 capability_name=name,
86 model=str(cfg["model"]).strip(), 88 model=str(cfg["model"]).strip(),
87 base_url=str(cfg["base_url"]).strip(), 89 base_url=str(cfg["base_url"]).strip(),
88 - api_key=cfg.get("api_key"), 90 + api_key=self._app_config.infrastructure.secrets.dashscope_api_key,
89 timeout=int(cfg["timeout_sec"]), 91 timeout=int(cfg["timeout_sec"]),
90 glossary_id=cfg.get("glossary_id"), 92 glossary_id=cfg.get("glossary_id"),
91 ) 93 )
@@ -94,7 +96,7 @@ class TranslationService: @@ -94,7 +96,7 @@ class TranslationService:
94 from translation.backends.deepl import DeepLTranslationBackend 96 from translation.backends.deepl import DeepLTranslationBackend
95 97
96 return DeepLTranslationBackend( 98 return DeepLTranslationBackend(
97 - api_key=cfg.get("api_key"), 99 + api_key=self._app_config.infrastructure.secrets.deepl_auth_key,
98 api_url=str(cfg["api_url"]).strip(), 100 api_url=str(cfg["api_url"]).strip(),
99 timeout=float(cfg["timeout_sec"]), 101 timeout=float(cfg["timeout_sec"]),
100 glossary_id=cfg.get("glossary_id"), 102 glossary_id=cfg.get("glossary_id"),
@@ -108,6 +110,7 @@ class TranslationService: @@ -108,6 +110,7 @@ class TranslationService:
108 model=str(cfg["model"]).strip(), 110 model=str(cfg["model"]).strip(),
109 timeout_sec=float(cfg["timeout_sec"]), 111 timeout_sec=float(cfg["timeout_sec"]),
110 base_url=str(cfg["base_url"]).strip(), 112 base_url=str(cfg["base_url"]).strip(),
  113 + api_key=self._app_config.infrastructure.secrets.dashscope_api_key,
111 ) 114 )
112 115
113 def _create_local_nllb_backend(self, *, name: str, cfg: Dict[str, object]) -> TranslationBackendProtocol: 116 def _create_local_nllb_backend(self, *, name: str, cfg: Dict[str, object]) -> TranslationBackendProtocol:
utils/es_client.py
@@ -5,10 +5,9 @@ Elasticsearch client wrapper. @@ -5,10 +5,9 @@ Elasticsearch client wrapper.
5 from elasticsearch import Elasticsearch 5 from elasticsearch import Elasticsearch
6 from elasticsearch.helpers import bulk 6 from elasticsearch.helpers import bulk
7 from typing import Dict, Any, List, Optional 7 from typing import Dict, Any, List, Optional
8 -import os  
9 import logging 8 import logging
10 9
11 -from config.env_config import ES_CONFIG 10 +from config.loader import get_app_config
12 11
13 logger = logging.getLogger(__name__) 12 logger = logging.getLogger(__name__)
14 13
@@ -33,7 +32,7 @@ class ESClient: @@ -33,7 +32,7 @@ class ESClient:
33 **kwargs: Additional ES client parameters 32 **kwargs: Additional ES client parameters
34 """ 33 """
35 if hosts is None: 34 if hosts is None:
36 - hosts = [os.getenv('ES_HOST', 'http://localhost:9200')] 35 + hosts = [get_app_config().infrastructure.elasticsearch.host]
37 36
38 # Build client config 37 # Build client config
39 client_config = { 38 client_config = {
@@ -325,16 +324,9 @@ def get_es_client_from_env() -&gt; ESClient: @@ -325,16 +324,9 @@ def get_es_client_from_env() -&gt; ESClient:
325 Returns: 324 Returns:
326 ESClient instance 325 ESClient instance
327 """ 326 """
328 - if ES_CONFIG:  
329 - return ESClient(  
330 - hosts=[ES_CONFIG['host']],  
331 - username=ES_CONFIG.get('username'),  
332 - password=ES_CONFIG.get('password')  
333 - )  
334 - else:  
335 - # Fallback to env variables  
336 - return ESClient(  
337 - hosts=[os.getenv('ES_HOST', 'http://localhost:9200')],  
338 - username=os.getenv('ES_USERNAME'),  
339 - password=os.getenv('ES_PASSWORD')  
340 - ) 327 + cfg = get_app_config().infrastructure.elasticsearch
  328 + return ESClient(
  329 + hosts=[cfg.host],
  330 + username=cfg.username,
  331 + password=cfg.password,
  332 + )