ai-saas / saas-search

Compare View

switch
from
...
to
 
Commits (3)
Showing 48 changed files   Show diff stats
Inline Side-by-side
api/app.py
  Show/Hide comments   View file @0342d89
... ... @@ -86,8 +86,7 @@ limiter = Limiter(key_func=get_remote_address)
86 86 # Add parent directory to path
87 87 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
88 88  
89   -from config.env_config import ES_CONFIG, DB_CONFIG
90   -from config import ConfigLoader
  89 +from config import get_app_config
91 90 from utils import ESClient
92 91 from search import Searcher
93 92 from query import QueryParser
... ... @@ -99,7 +98,7 @@ _es_client: Optional[ESClient] = None
99 98 _searcher: Optional[Searcher] = None
100 99 _query_parser: Optional[QueryParser] = None
101 100 _suggestion_service: Optional[SuggestionService] = None
102   -_config = None
  101 +_app_config = None
103 102  
104 103  
105 104 def init_service(es_host: str = "http://localhost:9200"):
... ... @@ -109,20 +108,20 @@ def init_service(es_host: str = "http://localhost:9200"):
109 108 Args:
110 109 es_host: Elasticsearch host URL
111 110 """
112   - global _es_client, _searcher, _query_parser, _suggestion_service, _config
  111 + global _es_client, _searcher, _query_parser, _suggestion_service, _app_config
113 112  
114 113 start_time = time.time()
115 114 logger.info("Initializing search service (multi-tenant)")
116 115  
117 116 # Load configuration
118 117 logger.info("Loading configuration...")
119   - config_loader = ConfigLoader("config/config.yaml")
120   - _config = config_loader.load_config()
  118 + _app_config = get_app_config()
  119 + search_config = _app_config.search
121 120 logger.info("Configuration loaded")
122 121  
123 122 # Get ES credentials
124   - es_username = os.getenv('ES_USERNAME') or ES_CONFIG.get('username')
125   - es_password = os.getenv('ES_PASSWORD') or ES_CONFIG.get('password')
  123 + es_username = _app_config.infrastructure.elasticsearch.username
  124 + es_password = _app_config.infrastructure.elasticsearch.password
126 125  
127 126 # Connect to Elasticsearch
128 127 logger.info(f"Connecting to Elasticsearch at {es_host}...")
... ... @@ -139,15 +138,15 @@ def init_service(es_host: str = "http://localhost:9200"):
139 138  
140 139 # Initialize components
141 140 logger.info("Initializing query parser...")
142   - _query_parser = QueryParser(_config)
  141 + _query_parser = QueryParser(search_config)
143 142  
144 143 logger.info("Initializing searcher...")
145   - _searcher = Searcher(_es_client, _config, _query_parser)
  144 + _searcher = Searcher(_es_client, search_config, _query_parser)
146 145 logger.info("Initializing suggestion service...")
147 146 _suggestion_service = SuggestionService(_es_client)
148 147  
149 148 elapsed = time.time() - start_time
150   - logger.info(f"Search service ready! (took {elapsed:.2f}s) | Index: {_config.es_index_name}")
  149 + logger.info(f"Search service ready! (took {elapsed:.2f}s) | Index: {search_config.es_index_name}")
151 150  
152 151  
153 152  
... ... @@ -182,9 +181,9 @@ def get_suggestion_service() -> SuggestionService:
182 181  
183 182 def get_config():
184 183 """Get global config instance."""
185   - if _config is None:
  184 + if _app_config is None:
186 185 raise RuntimeError("Service not initialized")
187   - return _config
  186 + return _app_config
188 187  
189 188  
190 189 # Create FastAPI app with enhanced configuration
... ... @@ -240,7 +239,7 @@ async def startup_event():
240 239 except Exception as e:
241 240 logger.warning(f"Failed to set thread pool size: {e}, using default")
242 241  
243   - es_host = os.getenv("ES_HOST", "http://localhost:9200")
  242 + es_host = get_app_config().infrastructure.elasticsearch.host
244 243 logger.info("Starting E-Commerce Search API (Multi-Tenant)")
245 244 logger.info(f"Elasticsearch Host: {es_host}")
246 245  
... ...
api/indexer_app.py
  Show/Hide comments   View file @0342d89
... ... @@ -38,8 +38,7 @@ logger = logging.getLogger(__name__)
38 38 # Add parent directory to path
39 39 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
40 40  
41   -from config.env_config import ES_CONFIG # noqa: E402
42   -from config import ConfigLoader # noqa: E402
  41 +from config import get_app_config # noqa: E402
43 42 from utils import ESClient # noqa: E402
44 43 from utils.db_connector import create_db_connection # noqa: E402
45 44 from indexer.incremental_service import IncrementalIndexerService # noqa: E402
... ... @@ -55,7 +54,7 @@ from .service_registry import (
55 54  
56 55  
57 56 _es_client: Optional[ESClient] = None
58   -_config = None
  57 +_app_config = None
59 58 _incremental_service: Optional[IncrementalIndexerService] = None
60 59 _bulk_indexing_service: Optional[BulkIndexingService] = None
61 60 _suggestion_builder: Optional[SuggestionIndexBuilder] = None
... ... @@ -68,20 +67,19 @@ def init_indexer_service(es_host: str = "http://localhost:9200"):
68 67 This mirrors the indexing-related initialization logic in api.app.init_service
69 68 but without search-related components.
70 69 """
71   - global _es_client, _config, _incremental_service, _bulk_indexing_service, _suggestion_builder
  70 + global _es_client, _app_config, _incremental_service, _bulk_indexing_service, _suggestion_builder
72 71  
73 72 start_time = time.time()
74 73 logger.info("Initializing Indexer service")
75 74  
76 75 # Load configuration (kept for parity/logging; indexer routes don't depend on it)
77 76 logger.info("Loading configuration...")
78   - config_loader = ConfigLoader("config/config.yaml")
79   - _config = config_loader.load_config()
  77 + _app_config = get_app_config()
80 78 logger.info("Configuration loaded")
81 79  
82 80 # Get ES credentials
83   - es_username = os.getenv("ES_USERNAME") or ES_CONFIG.get("username")
84   - es_password = os.getenv("ES_PASSWORD") or ES_CONFIG.get("password")
  81 + es_username = _app_config.infrastructure.elasticsearch.username
  82 + es_password = _app_config.infrastructure.elasticsearch.password
85 83  
86 84 # Connect to Elasticsearch
87 85 logger.info(f"Connecting to Elasticsearch at {es_host} for indexer...")
... ... @@ -97,11 +95,12 @@ def init_indexer_service(es_host: str = "http://localhost:9200"):
97 95 set_es_client(_es_client)
98 96  
99 97 # Initialize indexing services (DB is required here)
100   - db_host = os.getenv("DB_HOST")
101   - db_port = int(os.getenv("DB_PORT", 3306))
102   - db_database = os.getenv("DB_DATABASE")
103   - db_username = os.getenv("DB_USERNAME")
104   - db_password = os.getenv("DB_PASSWORD")
  98 + db_config = _app_config.infrastructure.database
  99 + db_host = db_config.host
  100 + db_port = db_config.port
  101 + db_database = db_config.database
  102 + db_username = db_config.username
  103 + db_password = db_config.password
105 104  
106 105 if all([db_host, db_database, db_username, db_password]):
107 106 logger.info("Initializing database connection for indexing services...")
... ... @@ -166,7 +165,7 @@ async def startup_event():
166 165 except Exception as e:
167 166 logger.warning(f"Failed to set thread pool size: {e}, using default")
168 167  
169   - es_host = os.getenv("ES_HOST", "http://localhost:9200")
  168 + es_host = get_app_config().infrastructure.elasticsearch.host
170 169 logger.info("Starting Indexer API service")
171 170 logger.info(f"Elasticsearch Host: {es_host}")
172 171 try:
... ... @@ -176,14 +175,11 @@ async def startup_event():
176 175 # Eager warmup: build per-tenant transformer bundles at startup to avoid
177 176 # first-request latency (config/provider/encoder + transformer wiring).
178 177 try:
179   - if _incremental_service is not None and _config is not None:
  178 + if _incremental_service is not None and _app_config is not None:
180 179 tenants = []
181   - # config.tenant_config shape: {"default": {...}, "tenants": {"1": {...}, ...}}
182   - tc = getattr(_config, "tenant_config", None) or {}
183   - if isinstance(tc, dict):
184   - tmap = tc.get("tenants")
185   - if isinstance(tmap, dict):
186   - tenants = [str(k) for k in tmap.keys()]
  180 + tmap = _app_config.tenants.tenants
  181 + if isinstance(tmap, dict):
  182 + tenants = [str(k) for k in tmap.keys()]
187 183 # If no explicit tenants configured, skip warmup.
188 184 if tenants:
189 185 warm = _incremental_service.warmup_transformers(tenants)
... ...
api/routes/admin.py
  Show/Hide comments   View file @0342d89
... ... @@ -3,7 +3,6 @@ Admin API routes for configuration and management.
3 3 """
4 4  
5 5 from fastapi import APIRouter, HTTPException, Request
6   -from typing import Dict
7 6  
8 7 from ..models import HealthResponse, ErrorResponse
9 8 from indexer.mapping_generator import get_tenant_index_name
... ... @@ -42,22 +41,12 @@ async def health_check():
42 41 @router.get("/config")
43 42 async def get_configuration():
44 43 """
45   - Get current search configuration (sanitized).
  44 + Get the effective application configuration (sanitized).
46 45 """
47 46 try:
48 47 from ..app import get_config
49 48  
50   - config = get_config()
51   -
52   - return {
53   - "es_index_name": config.es_index_name,
54   - "num_field_boosts": len(config.field_boosts),
55   - "multilingual_fields": config.query_config.multilingual_fields,
56   - "shared_fields": config.query_config.shared_fields,
57   - "core_multilingual_fields": config.query_config.core_multilingual_fields,
58   - "supported_languages": config.query_config.supported_languages,
59   - "spu_enabled": config.spu_config.enabled
60   - }
  49 + return get_config().sanitized_dict()
61 50  
62 51 except HTTPException:
63 52 raise
... ... @@ -65,45 +54,21 @@ async def get_configuration():
65 54 raise HTTPException(status_code=500, detail=str(e))
66 55  
67 56  
68   -@router.post("/rewrite-rules")
69   -async def update_rewrite_rules(rules: Dict[str, str]):
70   - """
71   - Update query rewrite rules.
72   -
73   - Args:
74   - rules: Dictionary of pattern -> replacement mappings
75   - """
  57 +@router.get("/config/meta")
  58 +async def get_configuration_meta():
  59 + """Get configuration metadata for observability."""
76 60 try:
77   - from ..app import get_query_parser
78   -
79   - query_parser = get_query_parser()
80   - query_parser.update_rewrite_rules(rules)
81   -
82   - return {
83   - "status": "success",
84   - "message": f"Updated {len(rules)} rewrite rules"
85   - }
86   -
87   - except Exception as e:
88   - raise HTTPException(status_code=500, detail=str(e))
89   -
90   -
91   -@router.get("/rewrite-rules")
92   -async def get_rewrite_rules():
93   - """
94   - Get current query rewrite rules.
95   - """
96   - try:
97   - from ..app import get_query_parser
98   -
99   - query_parser = get_query_parser()
100   - rules = query_parser.get_rewrite_rules()
  61 + from ..app import get_config
101 62  
  63 + config = get_config()
102 64 return {
103   - "rules": rules,
104   - "count": len(rules)
  65 + "environment": config.runtime.environment,
  66 + "config_hash": config.metadata.config_hash,
  67 + "loaded_files": list(config.metadata.loaded_files),
  68 + "deprecated_keys": list(config.metadata.deprecated_keys),
105 69 }
106   -
  70 + except HTTPException:
  71 + raise
107 72 except Exception as e:
108 73 raise HTTPException(status_code=500, detail=str(e))
109 74  
... ...
config/__init__.py
  Show/Hide comments   View file @0342d89
1   -"""
2   -Configuration package for search engine.
  1 +"""Unified configuration package exports."""
3 2  
4   -Provides configuration loading, validation, and utility functions.
5   -"""
6   -
7   -from .config_loader import (
8   - SearchConfig,
9   - QueryConfig,
10   - IndexConfig,
11   - SPUConfig,
  3 +from config.config_loader import ConfigLoader, ConfigurationError
  4 +from config.loader import AppConfigLoader, get_app_config, reload_app_config
  5 +from config.schema import (
  6 + AppConfig,
12 7 FunctionScoreConfig,
  8 + IndexConfig,
  9 + QueryConfig,
13 10 RerankConfig,
14   - ConfigLoader,
15   - ConfigurationError
16   -)
17   -
18   -from .utils import (
19   - get_match_fields_for_index,
20   - get_domain_fields
  11 + SPUConfig,
  12 + SearchConfig,
  13 + ServicesConfig,
21 14 )
22   -from .services_config import (
23   - get_translation_config,
24   - get_embedding_config,
25   - get_rerank_config,
  15 +from config.services_config import (
26 16 get_embedding_backend_config,
27   - get_rerank_backend_config,
28   - get_translation_base_url,
29   - get_embedding_text_base_url,
  17 + get_embedding_config,
  18 + get_embedding_image_backend_config,
30 19 get_embedding_image_base_url,
  20 + get_embedding_text_base_url,
  21 + get_rerank_backend_config,
  22 + get_rerank_config,
31 23 get_rerank_service_url,
  24 + get_translation_base_url,
32 25 get_translation_cache_config,
33   - ServiceConfig,
  26 + get_translation_config,
34 27 )
  28 +from config.utils import get_domain_fields, get_match_fields_for_index
35 29  
36 30 __all__ = [
37   - # Main config classes
38   - 'SearchConfig',
39   - 'QueryConfig',
40   - 'IndexConfig',
41   - 'SPUConfig',
42   - 'FunctionScoreConfig',
43   - 'RerankConfig',
44   -
45   - # Loader and utilities
46   - 'ConfigLoader',
47   - 'ConfigurationError',
48   - 'get_match_fields_for_index',
49   - 'get_domain_fields',
50   - 'get_translation_config',
51   - 'get_embedding_config',
52   - 'get_rerank_config',
53   - 'get_embedding_backend_config',
54   - 'get_rerank_backend_config',
55   - 'get_translation_base_url',
56   - 'get_embedding_text_base_url',
57   - 'get_embedding_image_base_url',
58   - 'get_rerank_service_url',
59   - 'get_translation_cache_config',
60   - 'ServiceConfig',
  31 + "AppConfig",
  32 + "AppConfigLoader",
  33 + "ConfigLoader",
  34 + "ConfigurationError",
  35 + "FunctionScoreConfig",
  36 + "IndexConfig",
  37 + "QueryConfig",
  38 + "RerankConfig",
  39 + "SPUConfig",
  40 + "SearchConfig",
  41 + "ServicesConfig",
  42 + "get_app_config",
  43 + "reload_app_config",
  44 + "get_domain_fields",
  45 + "get_match_fields_for_index",
  46 + "get_translation_config",
  47 + "get_embedding_config",
  48 + "get_rerank_config",
  49 + "get_embedding_backend_config",
  50 + "get_embedding_image_backend_config",
  51 + "get_rerank_backend_config",
  52 + "get_translation_base_url",
  53 + "get_embedding_text_base_url",
  54 + "get_embedding_image_base_url",
  55 + "get_rerank_service_url",
  56 + "get_translation_cache_config",
61 57 ]
... ...
config/config.yaml
  Show/Hide comments   View file @0342d89
... ... @@ -5,6 +5,14 @@
5 5 # Elasticsearch Index
6 6 es_index_name: "search_products"
7 7  
  8 +# Config assets
  9 +assets:
  10 + query_rewrite_dictionary_path: "config/dictionaries/query_rewrite.dict"
  11 +
  12 +# Product content understanding (LLM enrich-content) configuration
  13 +product_enrich:
  14 + max_workers: 40
  15 +
8 16 # ES Index Settings (基础设置)
9 17 es_settings:
10 18 number_of_shards: 1
... ... @@ -211,6 +219,19 @@ services:
211 219 device: "cuda"
212 220 batch_size: 32
213 221 normalize_embeddings: true
  222 + # 服务内图片后端（embedding 进程启动时读取）
  223 + image_backend: "clip_as_service" # clip_as_service | local_cnclip
  224 + image_backends:
  225 + clip_as_service:
  226 + server: "grpc://127.0.0.1:51000"
  227 + model_name: "CN-CLIP/ViT-L-14"
  228 + batch_size: 8
  229 + normalize_embeddings: true
  230 + local_cnclip:
  231 + model_name: "ViT-L-14"
  232 + device: null
  233 + batch_size: 8
  234 + normalize_embeddings: true
214 235 rerank:
215 236 provider: "http"
216 237 base_url: "http://127.0.0.1:6007"
... ... @@ -218,6 +239,9 @@ services:
218 239 http:
219 240 base_url: "http://127.0.0.1:6007"
220 241 service_url: "http://127.0.0.1:6007/rerank"
  242 + request:
  243 + max_docs: 1000
  244 + normalize: true
221 245 # 服务内后端（reranker 进程启动时读取）
222 246 backend: "qwen3_vllm" # bge | qwen3_vllm | qwen3_transformers | dashscope_rerank
223 247 backends:
... ...
config/config_loader.py
  Show/Hide comments   View file @0342d89
1 1 """
2   -Configuration loader and validator for search engine configurations.
  2 +Compatibility wrapper for search-behavior config access.
3 3  
4   -This module handles loading, parsing, and validating YAML configuration files
5   -that define how search should be executed (NOT how data should be indexed).
6   -
7   -索引结构由 mappings/search_products.json 定义。
8   -此配置只定义搜索行为：字段权重、搜索域、查询策略等。
  4 +The unified loader lives in :mod:`config.loader`. This module now exposes the
  5 +search subtree only, so existing search/indexer code can consume a single
  6 +source-of-truth search config without reparsing YAML separately.
9 7 """
10 8  
11   -import yaml
12   -from typing import Dict, Any, List, Optional
13   -from dataclasses import dataclass, field
14   -from pathlib import Path
15   -
16   -
17   -@dataclass
18   -class IndexConfig:
19   - """Configuration for an index domain (e.g., default, title, brand)."""
20   - name: str
21   - label: str
22   - fields: List[str] # List of field names to include in this search domain
23   - boost: float = 1.0
24   - example: Optional[str] = None
25   -
26   -
27   -@dataclass
28   -class QueryConfig:
29   - """Configuration for query processing."""
30   - supported_languages: List[str] = field(default_factory=lambda: ["zh", "en"])
31   - default_language: str = "en"
32   -
33   - # Feature flags
34   - enable_text_embedding: bool = True
35   - enable_query_rewrite: bool = True
36   -
37   - # Query rewrite dictionary (loaded from external file)
38   - rewrite_dictionary: Dict[str, str] = field(default_factory=dict)
39   -
40   - # Embedding field names
41   - text_embedding_field: Optional[str] = "title_embedding"
42   - image_embedding_field: Optional[str] = None
43   -
44   - # Source fields configuration
45   - source_fields: Optional[List[str]] = None
46   -
47   - # KNN boost configuration
48   - knn_boost: float = 0.25 # Boost value for KNN (embedding recall)
49   -
50   - # Dynamic text fields for multi-language retrieval
51   - multilingual_fields: List[str] = field(
52   - default_factory=lambda: ["title", "brief", "description", "vendor", "category_path", "category_name_text"]
53   - )
54   - shared_fields: List[str] = field(
55   - default_factory=lambda: ["tags", "option1_values", "option2_values", "option3_values"]
56   - )
57   - core_multilingual_fields: List[str] = field(
58   - default_factory=lambda: ["title", "brief", "vendor", "category_name_text"]
59   - )
60   -
61   - # Unified text strategy tuning
62   - base_minimum_should_match: str = "75%"
63   - translation_minimum_should_match: str = "75%"
64   - translation_boost: float = 0.4
65   - translation_boost_when_source_missing: float = 1.0
66   - source_boost_when_missing: float = 0.6
67   - original_query_fallback_boost_when_translation_missing: float = 0.2
68   - tie_breaker_base_query: float = 0.9
69   -
70   - # Query-time translation model selection (configurable)
71   - # - zh_to_en_model: model for zh -> en
72   - # - en_to_zh_model: model for en -> zh
73   - # - default_translation_model: fallback model for all other language pairs
74   - zh_to_en_model: str = "opus-mt-zh-en"
75   - en_to_zh_model: str = "opus-mt-en-zh"
76   - default_translation_model: str = "nllb-200-distilled-600m"
77   -
78   -
79   -@dataclass
80   -class SPUConfig:
81   - """Configuration for SPU aggregation."""
82   - enabled: bool = False
83   - spu_field: Optional[str] = None
84   - inner_hits_size: int = 3
85   - # 配置哪些option维度参与检索（进索引、以及在线搜索）
86   - searchable_option_dimensions: List[str] = field(default_factory=lambda: ['option1', 'option2', 'option3'])
87   -
88   -
89   -@dataclass
90   -class FunctionScoreConfig:
91   - """Function Score配置（ES层打分规则）"""
92   - score_mode: str = "sum"
93   - boost_mode: str = "multiply"
94   - functions: List[Dict[str, Any]] = field(default_factory=list)
95   -
96   -
97   -@dataclass
98   -class RerankConfig:
99   - """重排配置（provider/URL 在 services.rerank）"""
100   - enabled: bool = True
101   - rerank_window: int = 384
102   - timeout_sec: float = 15.0
103   - weight_es: float = 0.4
104   - weight_ai: float = 0.6
105   - rerank_query_template: str = "{query}"
106   - rerank_doc_template: str = "{title}"
107   -
108   -
109   -@dataclass
110   -class SearchConfig:
111   - """Complete configuration for search engine (multi-tenant)."""
112   -
113   - # 字段权重配置（用于搜索）
114   - field_boosts: Dict[str, float]
115   -
116   - # Legacy index domains (deprecated; kept for compatibility)
117   - indexes: List[IndexConfig]
118   -
119   - # Query processing
120   - query_config: QueryConfig
121   -
122   - # Function Score configuration (ES层打分)
123   - function_score: FunctionScoreConfig
124   -
125   - # Rerank configuration (本地重排)
126   - rerank: RerankConfig
127   -
128   - # SPU configuration
129   - spu_config: SPUConfig
130   -
131   - # ES index settings
132   - es_index_name: str
133   -
134   - # Tenant configuration
135   - tenant_config: Dict[str, Any] = field(default_factory=dict)
136   -
137   - # ES settings
138   - es_settings: Dict[str, Any] = field(default_factory=dict)
139   - # Extensible service/provider registry (translation/embedding/rerank/...)
140   - services: Dict[str, Any] = field(default_factory=dict)
  9 +from __future__ import annotations
141 10  
  11 +from dataclasses import asdict
  12 +from pathlib import Path
  13 +from typing import Any, Dict, List, Optional
142 14  
143   -class ConfigurationError(Exception):
144   - """Raised when configuration validation fails."""
145   - pass
  15 +from config.loader import AppConfigLoader, ConfigurationError
  16 +from config.schema import (
  17 + FunctionScoreConfig,
  18 + IndexConfig,
  19 + QueryConfig,
  20 + RerankConfig,
  21 + SPUConfig,
  22 + SearchConfig,
  23 +)
146 24  
147 25  
148 26 class ConfigLoader:
149   - """Loads and validates unified search engine configuration from YAML file."""
150   -
151   - def __init__(self, config_file: Optional[Path] = None):
152   - """
153   - Initialize config loader.
154   -
155   - Args:
156   - config_file: Path to config YAML file (defaults to config/config.yaml)
157   - """
158   - if config_file is None:
159   - config_file = Path(__file__).parent / "config.yaml"
160   - self.config_file = Path(config_file)
161   -
162   - def _load_rewrite_dictionary(self) -> Dict[str, str]:
163   - """Load query rewrite dictionary from external file."""
164   - rewrite_file = Path(__file__).parent / "rewrite_dictionary.txt"
165   - rewrite_dict = {}
166   -
167   - if not rewrite_file.exists():
168   - return rewrite_dict
169   -
170   - try:
171   - with open(rewrite_file, 'r', encoding='utf-8') as f:
172   - for line in f:
173   - line = line.strip()
174   - if not line or line.startswith('#'):
175   - continue
176   -
177   - parts = line.split('\t')
178   - if len(parts) >= 2:
179   - original = parts[0].strip()
180   - replacement = parts[1].strip()
181   - if original and replacement:
182   - rewrite_dict[original] = replacement
183   - except Exception as e:
184   - print(f"Warning: Failed to load rewrite dictionary: {e}")
185   -
186   - return rewrite_dict
187   -
188   - def load_config(self, validate: bool = True) -> SearchConfig:
189   - """
190   - Load unified configuration from YAML file.
191   -
192   - Args:
193   - validate: Whether to validate configuration after loading
194   -
195   - Returns:
196   - SearchConfig object
197   -
198   - Raises:
199   - ConfigurationError: If config file not found, invalid, or validation fails
200   - """
201   - if not self.config_file.exists():
202   - raise ConfigurationError(f"Configuration file not found: {self.config_file}")
203   -
204   - try:
205   - with open(self.config_file, 'r', encoding='utf-8') as f:
206   - config_data = yaml.safe_load(f)
207   - except yaml.YAMLError as e:
208   - raise ConfigurationError(f"Invalid YAML in {self.config_file}: {e}")
209   -
210   - config = self._parse_config(config_data)
211   -
212   - # Auto-validate configuration
213   - if validate:
214   - errors = self.validate_config(config)
215   - if errors:
216   - error_msg = "Configuration validation failed:\n" + "\n".join(f" - {err}" for err in errors)
217   - raise ConfigurationError(error_msg)
218   -
219   - return config
220   -
221   - def _parse_config(self, config_data: Dict[str, Any]) -> SearchConfig:
222   - """Parse configuration dictionary into SearchConfig object."""
223   -
224   - # Parse field_boosts
225   - field_boosts = config_data.get("field_boosts", {})
226   - if not isinstance(field_boosts, dict):
227   - raise ConfigurationError("field_boosts must be a dictionary")
228   -
229   - # Parse indexes (deprecated; compatibility only)
230   - indexes = []
231   - for index_data in config_data.get("indexes", []):
232   - indexes.append(self._parse_index_config(index_data))
233   -
234   - # Parse query config
235   - query_config_data = config_data.get("query_config", {})
236   - rewrite_dictionary = self._load_rewrite_dictionary()
237   - search_fields_cfg = query_config_data.get("search_fields", {})
238   - text_strategy_cfg = query_config_data.get("text_query_strategy", {})
  27 + """Load the unified app config and return the search subtree."""
239 28  
240   - query_config = QueryConfig(
241   - supported_languages=query_config_data.get("supported_languages") or ["zh", "en"],
242   - default_language=query_config_data.get("default_language") or "en",
243   - enable_text_embedding=query_config_data.get("enable_text_embedding", True),
244   - enable_query_rewrite=query_config_data.get("enable_query_rewrite", True),
245   - rewrite_dictionary=rewrite_dictionary,
246   - text_embedding_field=query_config_data.get("text_embedding_field"),
247   - image_embedding_field=query_config_data.get("image_embedding_field"),
248   - source_fields=query_config_data.get("source_fields"),
249   - knn_boost=query_config_data.get("knn_boost", 0.25),
250   - multilingual_fields=search_fields_cfg.get(
251   - "multilingual_fields",
252   - ["title", "brief", "description", "vendor", "category_path", "category_name_text"],
253   - ),
254   - shared_fields=search_fields_cfg.get(
255   - "shared_fields",
256   - ["tags", "option1_values", "option2_values", "option3_values"],
257   - ),
258   - core_multilingual_fields=search_fields_cfg.get(
259   - "core_multilingual_fields",
260   - ["title", "brief", "vendor", "category_name_text"],
261   - ),
262   - base_minimum_should_match=str(text_strategy_cfg.get("base_minimum_should_match", "75%")),
263   - translation_minimum_should_match=str(text_strategy_cfg.get("translation_minimum_should_match", "75%")),
264   - translation_boost=float(text_strategy_cfg.get("translation_boost", 0.4)),
265   - translation_boost_when_source_missing=float(
266   - text_strategy_cfg.get("translation_boost_when_source_missing", 1.0)
267   - ),
268   - source_boost_when_missing=float(text_strategy_cfg.get("source_boost_when_missing", 0.6)),
269   - original_query_fallback_boost_when_translation_missing=float(
270   - text_strategy_cfg.get("original_query_fallback_boost_when_translation_missing", 0.2)
271   - ),
272   - tie_breaker_base_query=float(text_strategy_cfg.get("tie_breaker_base_query", 0.9)),
273   - zh_to_en_model=str(query_config_data.get("zh_to_en_model") or "opus-mt-zh-en"),
274   - en_to_zh_model=str(query_config_data.get("en_to_zh_model") or "opus-mt-en-zh"),
275   - default_translation_model=str(
276   - query_config_data.get("default_translation_model") or "nllb-200-distilled-600m"
277   - ),
278   - )
279   -
280   - # Parse Function Score configuration
281   - fs_data = config_data.get("function_score", {})
282   - function_score = FunctionScoreConfig(
283   - score_mode=fs_data.get("score_mode") or "sum",
284   - boost_mode=fs_data.get("boost_mode") or "multiply",
285   - functions=fs_data.get("functions") or []
286   - )
287   -
288   - # Parse Rerank (provider/URL in services.rerank)
289   - rerank_data = config_data.get("rerank", {})
290   - rerank = RerankConfig(
291   - enabled=bool(rerank_data.get("enabled", True)),
292   - rerank_window=int(rerank_data.get("rerank_window", 384)),
293   - timeout_sec=float(rerank_data.get("timeout_sec", 15.0)),
294   - weight_es=float(rerank_data.get("weight_es", 0.4)),
295   - weight_ai=float(rerank_data.get("weight_ai", 0.6)),
296   - rerank_query_template=str(rerank_data.get("rerank_query_template") or "{query}"),
297   - rerank_doc_template=str(rerank_data.get("rerank_doc_template") or "{title}"),
298   - )
299   -
300   - # Parse SPU config
301   - spu_data = config_data.get("spu_config", {})
302   - spu_config = SPUConfig(
303   - enabled=spu_data.get("enabled", False),
304   - spu_field=spu_data.get("spu_field"),
305   - inner_hits_size=spu_data.get("inner_hits_size", 3),
306   - searchable_option_dimensions=spu_data.get("searchable_option_dimensions", ['option1', 'option2', 'option3'])
307   - )
308   -
309   - # Parse tenant config
310   - tenant_config_data = config_data.get("tenant_config", {})
  29 + def __init__(self, config_file: Optional[Path] = None) -> None:
  30 + self._loader = AppConfigLoader(config_file=Path(config_file) if config_file is not None else None)
311 31  
312   - # Parse extensible services/provider registry
313   - services_data = config_data.get("services", {}) or {}
314   - if not isinstance(services_data, dict):
315   - raise ConfigurationError("services must be a dictionary if provided")
  32 + def load_config(self, validate: bool = True) -> SearchConfig:
  33 + return self._loader.load(validate=validate).search
316 34  
317   - return SearchConfig(
318   - field_boosts=field_boosts,
319   - indexes=indexes,
320   - query_config=query_config,
321   - function_score=function_score,
322   - rerank=rerank,
323   - spu_config=spu_config,
324   - tenant_config=tenant_config_data,
325   - es_index_name=config_data.get("es_index_name", "search_products"),
326   - es_settings=config_data.get("es_settings", {}),
327   - services=services_data
328   - )
329   -
330   - def _parse_index_config(self, index_data: Dict[str, Any]) -> IndexConfig:
331   - """Parse index configuration from dictionary."""
332   - return IndexConfig(
333   - name=index_data["name"],
334   - label=index_data.get("label", index_data["name"]),
335   - fields=index_data.get("fields", []),
336   - boost=index_data.get("boost", 1.0),
337   - example=index_data.get("example")
338   - )
339   -
340 35 def validate_config(self, config: SearchConfig) -> List[str]:
341   - """
342   - Validate configuration for common errors.
343   -
344   - Args:
345   - config: SearchConfig to validate
346   -
347   - Returns:
348   - List of error messages (empty if valid)
349   - """
350   - errors = []
351   -
352   - # Validate es_index_name
  36 + errors: List[str] = []
353 37 if not config.es_index_name:
354 38 errors.append("es_index_name is required")
355   -
356   - # Validate field_boosts
357 39 if not config.field_boosts:
358 40 errors.append("field_boosts is empty")
359   -
360   - for field_name, boost in config.field_boosts.items():
361   - if not isinstance(boost, (int, float)):
362   - errors.append(f"field_boosts['{field_name}']: boost must be a number, got {type(boost).__name__}")
363   - elif boost < 0:
364   - errors.append(f"field_boosts['{field_name}']: boost must be non-negative")
365   -
366   - # Validate indexes (deprecated, optional)
367   - index_names = set()
368   - for index in config.indexes:
369   - # Check for duplicate index names
370   - if index.name in index_names:
371   - errors.append(f"Duplicate index name: {index.name}")
372   - index_names.add(index.name)
373   -
374   - # Validate fields in index
375   - if not index.fields:
376   - errors.append(f"Index '{index.name}': fields list is empty")
377   -
378   - # Validate SPU config
379   - if config.spu_config.enabled:
380   - if not config.spu_config.spu_field:
381   - errors.append("SPU aggregation enabled but no spu_field specified")
382   -
383   - # Validate query config
384   - if not config.query_config.supported_languages:
385   - errors.append("At least one supported language must be specified")
386   -
387 41 if config.query_config.default_language not in config.query_config.supported_languages:
388   - errors.append(
389   - f"Default language '{config.query_config.default_language}' "
390   - f"not in supported languages: {config.query_config.supported_languages}"
391   - )
392   -
393   - # Validate dynamic search fields
394   - def _validate_str_list(name: str, values: List[str]) -> None:
395   - if not isinstance(values, list) or not values:
396   - errors.append(f"query_config.{name} must be a non-empty list[str]")
397   - return
398   - for i, val in enumerate(values):
399   - if not isinstance(val, str) or not val.strip():
400   - errors.append(f"query_config.{name}[{i}] must be a non-empty string")
401   -
402   - _validate_str_list("multilingual_fields", config.query_config.multilingual_fields)
403   - _validate_str_list("shared_fields", config.query_config.shared_fields)
404   - _validate_str_list("core_multilingual_fields", config.query_config.core_multilingual_fields)
405   -
406   - core_set = set(config.query_config.core_multilingual_fields)
407   - multi_set = set(config.query_config.multilingual_fields)
408   - if not core_set.issubset(multi_set):
409   - errors.append("query_config.core_multilingual_fields must be subset of multilingual_fields")
410   -
411   - # Validate text query strategy numbers
412   - for name in (
413   - "translation_boost",
414   - "translation_boost_when_source_missing",
415   - "source_boost_when_missing",
416   - "original_query_fallback_boost_when_translation_missing",
417   - "tie_breaker_base_query",
418   - ):
419   - value = getattr(config.query_config, name, None)
420   - if not isinstance(value, (int, float)):
421   - errors.append(f"query_config.{name} must be a number")
422   - elif value < 0:
423   - errors.append(f"query_config.{name} must be non-negative")
424   -
425   - # Validate source_fields tri-state semantics
426   - source_fields = config.query_config.source_fields
427   - if source_fields is not None:
428   - if not isinstance(source_fields, list):
429   - errors.append("query_config.source_fields must be null or list[str]")
430   - else:
431   - for idx, field_name in enumerate(source_fields):
432   - if not isinstance(field_name, str) or not field_name.strip():
433   - errors.append(
434   - f"query_config.source_fields[{idx}] must be a non-empty string"
435   - )
436   -
437   - # Validate tenant config shape (default must exist in config)
438   - tenant_cfg = config.tenant_config
439   - if not isinstance(tenant_cfg, dict):
440   - errors.append("tenant_config must be an object")
441   - else:
442   - default_cfg = tenant_cfg.get("default")
443   - if not isinstance(default_cfg, dict):
444   - errors.append("tenant_config.default must be configured")
445   - else:
446   - index_languages = default_cfg.get("index_languages")
447   - if not isinstance(index_languages, list) or len(index_languages) == 0:
448   - errors.append("tenant_config.default.index_languages must be a non-empty list")
449   -
  42 + errors.append("default_language must be included in supported_languages")
  43 + if config.spu_config.enabled and not config.spu_config.spu_field:
  44 + errors.append("spu_field is required when SPU is enabled")
450 45 return errors
451   -
  46 +
452 47 def to_dict(self, config: SearchConfig) -> Dict[str, Any]:
453   - """Convert SearchConfig to dictionary representation."""
454   -
455   - # Build query_config dict
456   - query_config_dict = {
457   - "supported_languages": config.query_config.supported_languages,
458   - "default_language": config.query_config.default_language,
459   - "enable_text_embedding": config.query_config.enable_text_embedding,
460   - "enable_query_rewrite": config.query_config.enable_query_rewrite,
461   - "text_embedding_field": config.query_config.text_embedding_field,
462   - "image_embedding_field": config.query_config.image_embedding_field,
463   - "source_fields": config.query_config.source_fields,
464   - "search_fields": {
465   - "multilingual_fields": config.query_config.multilingual_fields,
466   - "shared_fields": config.query_config.shared_fields,
467   - "core_multilingual_fields": config.query_config.core_multilingual_fields,
468   - },
469   - "text_query_strategy": {
470   - "base_minimum_should_match": config.query_config.base_minimum_should_match,
471   - "translation_minimum_should_match": config.query_config.translation_minimum_should_match,
472   - "translation_boost": config.query_config.translation_boost,
473   - "translation_boost_when_source_missing": config.query_config.translation_boost_when_source_missing,
474   - "source_boost_when_missing": config.query_config.source_boost_when_missing,
475   - "original_query_fallback_boost_when_translation_missing": (
476   - config.query_config.original_query_fallback_boost_when_translation_missing
477   - ),
478   - "tie_breaker_base_query": config.query_config.tie_breaker_base_query,
479   - }
480   - }
481   -
482   - return {
483   - "es_index_name": config.es_index_name,
484   - "es_settings": config.es_settings,
485   - "field_boosts": config.field_boosts,
486   - "indexes": [self._index_to_dict(index) for index in config.indexes],
487   - "query_config": query_config_dict,
488   - "function_score": {
489   - "score_mode": config.function_score.score_mode,
490   - "boost_mode": config.function_score.boost_mode,
491   - "functions": config.function_score.functions
492   - },
493   - "rerank": {
494   - "enabled": config.rerank.enabled,
495   - "rerank_window": config.rerank.rerank_window,
496   - "timeout_sec": config.rerank.timeout_sec,
497   - "weight_es": config.rerank.weight_es,
498   - "weight_ai": config.rerank.weight_ai,
499   - "rerank_query_template": config.rerank.rerank_query_template,
500   - "rerank_doc_template": config.rerank.rerank_doc_template,
501   - },
502   - "spu_config": {
503   - "enabled": config.spu_config.enabled,
504   - "spu_field": config.spu_config.spu_field,
505   - "inner_hits_size": config.spu_config.inner_hits_size,
506   - "searchable_option_dimensions": config.spu_config.searchable_option_dimensions
507   - },
508   - "services": config.services,
509   - }
510   -
511   - def _index_to_dict(self, index: IndexConfig) -> Dict[str, Any]:
512   - """Convert IndexConfig to dictionary."""
513   - result = {
514   - "name": index.name,
515   - "label": index.label,
516   - "fields": index.fields,
517   - "boost": index.boost
518   - }
519   -
520   - if index.example:
521   - result["example"] = index.example
522   -
523   - return result
  48 + return asdict(config)
... ...
config/dictionaries/query_rewrite.dict 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,2 @@
  1 +玩具 category.keyword:玩具 OR default:玩具
  2 +消防 category.keyword:消防 OR default:消防
... ...
config/env_config.py
  Show/Hide comments   View file @0342d89
1 1 """
2   -Centralized configuration management for saas-search.
  2 +Compatibility accessors for infrastructure/runtime environment settings.
3 3  
4   -Loads configuration from environment variables and .env file.
5   -This module provides a single point for loading .env and setting defaults.
6   -All configuration variables are exported directly - no need for getter functions.
  4 +All values are derived from the unified application config. This module no
  5 +longer owns any independent loading or precedence rules.
7 6 """
8 7  
9   -import os
10   -from pathlib import Path
11   -from dotenv import load_dotenv
12   -
13   -# Load .env file from project root
14   -PROJECT_ROOT = Path(__file__).parent.parent
15   -load_dotenv(PROJECT_ROOT / '.env')
16   -
17   -
18   -# Elasticsearch Configuration
19   -ES_CONFIG = {
20   - 'host': os.getenv('ES_HOST', 'http://localhost:9200'),
21   - 'username': os.getenv('ES_USERNAME'),
22   - 'password': os.getenv('ES_PASSWORD'),
23   -}
24   -
25   -# Runtime environment & index namespace
26   -# RUNTIME_ENV: 当前运行环境，建议使用 prod / uat / test / dev 等枚举值
27   -RUNTIME_ENV = os.getenv('RUNTIME_ENV', 'prod')
28   -# ES_INDEX_NAMESPACE: 用于按环境隔离索引的命名空间前缀，例如 "uat_" / "test_"
29   -# 为空字符串时表示不加前缀（通常是 prod 环境）
30   -ES_INDEX_NAMESPACE = os.getenv('ES_INDEX_NAMESPACE')
31   -if ES_INDEX_NAMESPACE is None:
32   - # 未显式配置时，非 prod 环境默认加 "<env>_" 前缀，prod 环境默认不加前缀
33   - ES_INDEX_NAMESPACE = '' if RUNTIME_ENV == 'prod' else f'{RUNTIME_ENV}_'
34   -
35   -# Redis Configuration
36   -REDIS_CONFIG = {
37   - 'host': os.getenv('REDIS_HOST', 'localhost'),
38   - 'port': int(os.getenv('REDIS_PORT', 6479)),
39   - 'snapshot_db': int(os.getenv('REDIS_SNAPSHOT_DB', 0)),
40   - 'password': os.getenv('REDIS_PASSWORD'),
41   - 'socket_timeout': int(os.getenv('REDIS_SOCKET_TIMEOUT', 1)),
42   - 'socket_connect_timeout': int(os.getenv('REDIS_SOCKET_CONNECT_TIMEOUT', 1)),
43   - 'retry_on_timeout': os.getenv('REDIS_RETRY_ON_TIMEOUT', 'False').lower() == 'true',
44   - 'cache_expire_days': int(os.getenv('REDIS_CACHE_EXPIRE_DAYS', 360*2)), # 6 months
45   - # Embedding 缓存 key 前缀，例如 "embedding"
46   - 'embedding_cache_prefix': os.getenv('REDIS_EMBEDDING_CACHE_PREFIX', 'embedding'),
47   -}
48   -
49   -# DeepL API Key
50   -DEEPL_AUTH_KEY = os.getenv('DEEPL_AUTH_KEY')
51   -
52   -# DashScope API Key (for Qwen models)
53   -DASHSCOPE_API_KEY = os.getenv('DASHSCOPE_API_KEY')
54   -
55   -# API Service Configuration
56   -API_HOST = os.getenv('API_HOST', '0.0.0.0')
57   -API_PORT = int(os.getenv('API_PORT', 6002))
58   -# Indexer service
59   -INDEXER_HOST = os.getenv('INDEXER_HOST', '0.0.0.0')
60   -INDEXER_PORT = int(os.getenv('INDEXER_PORT', 6004))
61   -# Optional dependent services
62   -# EMBEDDING_HOST / EMBEDDING_PORT are only used by the optional combined embedding mode.
63   -EMBEDDING_HOST = os.getenv('EMBEDDING_HOST', '127.0.0.1')
64   -EMBEDDING_PORT = int(os.getenv('EMBEDDING_PORT', 6005))
65   -EMBEDDING_TEXT_HOST = os.getenv('EMBEDDING_TEXT_HOST', '127.0.0.1')
66   -EMBEDDING_TEXT_PORT = int(os.getenv('EMBEDDING_TEXT_PORT', 6005))
67   -EMBEDDING_IMAGE_HOST = os.getenv('EMBEDDING_IMAGE_HOST', '127.0.0.1')
68   -EMBEDDING_IMAGE_PORT = int(os.getenv('EMBEDDING_IMAGE_PORT', 6008))
69   -TRANSLATION_HOST = os.getenv('TRANSLATION_HOST', '127.0.0.1')
70   -TRANSLATION_PORT = int(os.getenv('TRANSLATION_PORT', 6006))
71   -RERANKER_HOST = os.getenv('RERANKER_HOST', '127.0.0.1')
72   -RERANKER_PORT = int(os.getenv('RERANKER_PORT', 6007))
73   -RERANK_PROVIDER = os.getenv('RERANK_PROVIDER', 'http')
74   -# API_BASE_URL: 如果未设置，根据API_HOST构建（0.0.0.0使用localhost）
75   -API_BASE_URL = os.getenv('API_BASE_URL')
76   -if not API_BASE_URL:
77   - API_BASE_URL = f'http://localhost:{API_PORT}' if API_HOST == '0.0.0.0' else f'http://{API_HOST}:{API_PORT}'
78   -INDEXER_BASE_URL = os.getenv('INDEXER_BASE_URL') or (
79   - f'http://localhost:{INDEXER_PORT}' if INDEXER_HOST == '0.0.0.0' else f'http://{INDEXER_HOST}:{INDEXER_PORT}'
80   -)
81   -EMBEDDING_TEXT_SERVICE_URL = os.getenv('EMBEDDING_TEXT_SERVICE_URL') or (
82   - f'http://{EMBEDDING_TEXT_HOST}:{EMBEDDING_TEXT_PORT}'
83   -)
84   -EMBEDDING_IMAGE_SERVICE_URL = os.getenv('EMBEDDING_IMAGE_SERVICE_URL') or (
85   - f'http://{EMBEDDING_IMAGE_HOST}:{EMBEDDING_IMAGE_PORT}'
86   -)
87   -RERANKER_SERVICE_URL = os.getenv('RERANKER_SERVICE_URL') or f'http://{RERANKER_HOST}:{RERANKER_PORT}/rerank'
  8 +from __future__ import annotations
  9 +
  10 +from typing import Any, Dict
  11 +
  12 +from config.loader import get_app_config
  13 +
  14 +
  15 +def _app():
  16 + return get_app_config()
  17 +
  18 +
  19 +def _runtime():
  20 + return _app().runtime
88 21  
89   -# Model IDs / paths
90   -TEXT_MODEL_DIR = os.getenv('TEXT_MODEL_DIR', os.getenv('TEXT_MODEL_ID', 'Qwen/Qwen3-Embedding-0.6B'))
91   -IMAGE_MODEL_DIR = os.getenv('IMAGE_MODEL_DIR', '/data/tw/models/cn-clip')
92 22  
93   -# Cache Directory
94   -CACHE_DIR = os.getenv('CACHE_DIR', '.cache')
  23 +def _infra():
  24 + return _app().infrastructure
95 25  
96   -# MySQL Database Configuration (Shoplazza)
97   -DB_CONFIG = {
98   - 'host': os.getenv('DB_HOST'),
99   - 'port': int(os.getenv('DB_PORT', 3306)) if os.getenv('DB_PORT') else 3306,
100   - 'database': os.getenv('DB_DATABASE'),
101   - 'username': os.getenv('DB_USERNAME'),
102   - 'password': os.getenv('DB_PASSWORD'),
103   -}
104 26  
  27 +def _elasticsearch_dict() -> Dict[str, Any]:
  28 + cfg = _infra().elasticsearch
  29 + return {
  30 + "host": cfg.host,
  31 + "username": cfg.username,
  32 + "password": cfg.password,
  33 + }
105 34  
106   -def print_config():
107   - """Print current configuration (with sensitive data masked)."""
108   - print("=" * 60)
109   - print("saas-search Configuration")
110   - print("=" * 60)
111 35  
112   - print("\nElasticsearch:")
113   - print(f" Host: {ES_CONFIG['host']}")
114   - print(f" Username: {ES_CONFIG['username']}")
115   - print(f" Password: {'*' * 10 if ES_CONFIG['password'] else 'None'}")
  36 +def _redis_dict() -> Dict[str, Any]:
  37 + cfg = _infra().redis
  38 + return {
  39 + "host": cfg.host,
  40 + "port": cfg.port,
  41 + "snapshot_db": cfg.snapshot_db,
  42 + "password": cfg.password,
  43 + "socket_timeout": cfg.socket_timeout,
  44 + "socket_connect_timeout": cfg.socket_connect_timeout,
  45 + "retry_on_timeout": cfg.retry_on_timeout,
  46 + "cache_expire_days": cfg.cache_expire_days,
  47 + "embedding_cache_prefix": cfg.embedding_cache_prefix,
  48 + "anchor_cache_prefix": cfg.anchor_cache_prefix,
  49 + "anchor_cache_expire_days": cfg.anchor_cache_expire_days,
  50 + }
116 51  
117   - print("\nRedis:")
118   - print(f" Host: {REDIS_CONFIG['host']}")
119   - print(f" Port: {REDIS_CONFIG['port']}")
120   - print(f" Password: {'*' * 10 if REDIS_CONFIG['password'] else 'None'}")
121 52  
122   - print("\nDeepL:")
123   - print(f" API Key: {'*' * 10 if DEEPL_AUTH_KEY else 'None (translation disabled)'}")
  53 +def _db_dict() -> Dict[str, Any]:
  54 + cfg = _infra().database
  55 + return {
  56 + "host": cfg.host,
  57 + "port": cfg.port,
  58 + "database": cfg.database,
  59 + "username": cfg.username,
  60 + "password": cfg.password,
  61 + }
  62 +
  63 +
  64 +ES_CONFIG = _elasticsearch_dict()
  65 +REDIS_CONFIG = _redis_dict()
  66 +DB_CONFIG = _db_dict()
  67 +
  68 +RUNTIME_ENV = _runtime().environment
  69 +ES_INDEX_NAMESPACE = _runtime().index_namespace
  70 +
  71 +DEEPL_AUTH_KEY = _infra().secrets.deepl_auth_key
  72 +DASHSCOPE_API_KEY = _infra().secrets.dashscope_api_key
  73 +
  74 +API_HOST = _runtime().api_host
  75 +API_PORT = _runtime().api_port
  76 +INDEXER_HOST = _runtime().indexer_host
  77 +INDEXER_PORT = _runtime().indexer_port
  78 +EMBEDDING_HOST = _runtime().embedding_host
  79 +EMBEDDING_PORT = _runtime().embedding_port
  80 +EMBEDDING_TEXT_HOST = _runtime().embedding_host
  81 +EMBEDDING_TEXT_PORT = _runtime().embedding_text_port
  82 +EMBEDDING_IMAGE_HOST = _runtime().embedding_host
  83 +EMBEDDING_IMAGE_PORT = _runtime().embedding_image_port
  84 +TRANSLATION_HOST = _runtime().translator_host
  85 +TRANSLATION_PORT = _runtime().translator_port
  86 +RERANKER_HOST = _runtime().reranker_host
  87 +RERANKER_PORT = _runtime().reranker_port
  88 +
  89 +API_BASE_URL = f"http://localhost:{API_PORT}" if API_HOST == "0.0.0.0" else f"http://{API_HOST}:{API_PORT}"
  90 +INDEXER_BASE_URL = (
  91 + f"http://localhost:{INDEXER_PORT}" if INDEXER_HOST == "0.0.0.0" else f"http://{INDEXER_HOST}:{INDEXER_PORT}"
  92 +)
  93 +EMBEDDING_TEXT_SERVICE_URL = _app().services.embedding.get_provider_config().get("text_base_url")
  94 +EMBEDDING_IMAGE_SERVICE_URL = _app().services.embedding.get_provider_config().get("image_base_url")
  95 +RERANKER_SERVICE_URL = (
  96 + _app().services.rerank.get_provider_config().get("service_url")
  97 + or _app().services.rerank.get_provider_config().get("base_url")
  98 +)
  99 +
124 100  
125   - print("\nAPI Service:")
126   - print(f" Host: {API_HOST}")
127   - print(f" Port: {API_PORT}")
  101 +def get_es_config() -> Dict[str, Any]:
  102 + return dict(ES_CONFIG)
128 103  
129   - print("\nModels:")
130   - print(f" Text Model: {TEXT_MODEL_DIR}")
131   - print(f" Image Model: {IMAGE_MODEL_DIR}")
132 104  
133   - print("\nCache:")
134   - print(f" Cache Directory: {CACHE_DIR}")
  105 +def get_db_config() -> Dict[str, Any]:
  106 + return dict(DB_CONFIG)
135 107  
136   - print("\nMySQL Database:")
137   - print(f" Host: {DB_CONFIG['host']}")
138   - print(f" Port: {DB_CONFIG['port']}")
139   - print(f" Database: {DB_CONFIG['database']}")
140   - print(f" Username: {DB_CONFIG['username']}")
141   - print(f" Password: {'*' * 10 if DB_CONFIG['password'] else 'None'}")
142 108  
143   - print("=" * 60)
  109 +def get_redis_config() -> Dict[str, Any]:
  110 + return dict(REDIS_CONFIG)
144 111  
145 112  
146   -if __name__ == "__main__":
147   - print_config()
  113 +def print_config() -> None:
  114 + config = _app().sanitized_dict()
  115 + print(config)
... ...
config/loader.py 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,589 @@
  1 +"""
  2 +Unified application configuration loader.
  3 +
  4 +This module is the single source of truth for loading, merging, normalizing,
  5 +and validating application configuration.
  6 +"""
  7 +
  8 +from __future__ import annotations
  9 +
  10 +import hashlib
  11 +import json
  12 +import os
  13 +from copy import deepcopy
  14 +from dataclasses import asdict
  15 +from functools import lru_cache
  16 +from pathlib import Path
  17 +from typing import Any, Dict, Iterable, List, Optional, Tuple
  18 +
  19 +import yaml
  20 +
  21 +try:
  22 + from dotenv import load_dotenv as _load_dotenv # type: ignore
  23 +except Exception: # pragma: no cover
  24 + _load_dotenv = None
  25 +
  26 +from config.schema import (
  27 + AppConfig,
  28 + AssetsConfig,
  29 + ConfigMetadata,
  30 + DatabaseSettings,
  31 + ElasticsearchSettings,
  32 + EmbeddingServiceConfig,
  33 + FunctionScoreConfig,
  34 + IndexConfig,
  35 + InfrastructureConfig,
  36 + QueryConfig,
  37 + ProductEnrichConfig,
  38 + RedisSettings,
  39 + RerankConfig,
  40 + RerankServiceConfig,
  41 + RuntimeConfig,
  42 + SearchConfig,
  43 + SecretsConfig,
  44 + ServicesConfig,
  45 + SPUConfig,
  46 + TenantCatalogConfig,
  47 + TranslationServiceConfig,
  48 +)
  49 +from translation.settings import build_translation_config
  50 +
  51 +
  52 +class ConfigurationError(Exception):
  53 + """Raised when configuration validation fails."""
  54 +
  55 +
  56 +def _deep_merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
  57 + result = deepcopy(base)
  58 + for key, value in (override or {}).items():
  59 + if (
  60 + key in result
  61 + and isinstance(result[key], dict)
  62 + and isinstance(value, dict)
  63 + ):
  64 + result[key] = _deep_merge(result[key], value)
  65 + else:
  66 + result[key] = deepcopy(value)
  67 + return result
  68 +
  69 +
  70 +def _load_yaml(path: Path) -> Dict[str, Any]:
  71 + with open(path, "r", encoding="utf-8") as handle:
  72 + data = yaml.safe_load(handle) or {}
  73 + if not isinstance(data, dict):
  74 + raise ConfigurationError(f"Configuration file root must be a mapping: {path}")
  75 + return data
  76 +
  77 +
  78 +def _read_rewrite_dictionary(path: Path) -> Dict[str, str]:
  79 + rewrite_dict: Dict[str, str] = {}
  80 + if not path.exists():
  81 + return rewrite_dict
  82 +
  83 + with open(path, "r", encoding="utf-8") as handle:
  84 + for raw_line in handle:
  85 + line = raw_line.strip()
  86 + if not line or line.startswith("#"):
  87 + continue
  88 + parts = line.split("\t")
  89 + if len(parts) < 2:
  90 + continue
  91 + original = parts[0].strip()
  92 + replacement = parts[1].strip()
  93 + if original and replacement:
  94 + rewrite_dict[original] = replacement
  95 + return rewrite_dict
  96 +
  97 +
  98 +class AppConfigLoader:
  99 + """Load the unified application configuration."""
  100 +
  101 + def __init__(
  102 + self,
  103 + *,
  104 + config_dir: Optional[Path] = None,
  105 + config_file: Optional[Path] = None,
  106 + env_file: Optional[Path] = None,
  107 + ) -> None:
  108 + self.config_dir = Path(config_dir or Path(__file__).parent)
  109 + self.config_file = Path(config_file) if config_file is not None else None
  110 + self.project_root = self.config_dir.parent
  111 + self.env_file = Path(env_file) if env_file is not None else self.project_root / ".env"
  112 +
  113 + def load(self, validate: bool = True) -> AppConfig:
  114 + self._load_env()
  115 + raw_config, loaded_files = self._load_raw_config()
  116 + app_config = self._build_app_config(raw_config, loaded_files)
  117 + if validate:
  118 + self._validate(app_config)
  119 + return app_config
  120 +
  121 + def _load_env(self) -> None:
  122 + if _load_dotenv is not None:
  123 + _load_dotenv(self.env_file, override=False)
  124 + return
  125 + _load_env_file_fallback(self.env_file)
  126 +
  127 + def _load_raw_config(self) -> Tuple[Dict[str, Any], List[str]]:
  128 + env_name = (os.getenv("APP_ENV") or os.getenv("RUNTIME_ENV") or "prod").strip().lower() or "prod"
  129 + loaded_files: List[str] = []
  130 + raw: Dict[str, Any] = {}
  131 +
  132 + if self.config_file is not None:
  133 + config_path = self.config_file
  134 + if not config_path.exists():
  135 + raise ConfigurationError(f"Configuration file not found: {config_path}")
  136 + raw = _deep_merge(raw, _load_yaml(config_path))
  137 + loaded_files.append(str(config_path))
  138 + else:
  139 + base_path = self.config_dir / "base.yaml"
  140 + legacy_path = self.config_dir / "config.yaml"
  141 + primary_path = base_path if base_path.exists() else legacy_path
  142 + if not primary_path.exists():
  143 + raise ConfigurationError(f"Configuration file not found: {primary_path}")
  144 + raw = _deep_merge(raw, _load_yaml(primary_path))
  145 + loaded_files.append(str(primary_path))
  146 +
  147 + env_path = self.config_dir / "environments" / f"{env_name}.yaml"
  148 + if env_path.exists():
  149 + raw = _deep_merge(raw, _load_yaml(env_path))
  150 + loaded_files.append(str(env_path))
  151 +
  152 + tenant_dir = self.config_dir / "tenants"
  153 + if tenant_dir.is_dir():
  154 + tenant_files = sorted(tenant_dir.glob("*.yaml"))
  155 + if tenant_files:
  156 + tenant_config = {"default": {}, "tenants": {}}
  157 + default_path = tenant_dir / "_default.yaml"
  158 + if default_path.exists():
  159 + tenant_config["default"] = _load_yaml(default_path)
  160 + loaded_files.append(str(default_path))
  161 + for tenant_path in tenant_files:
  162 + if tenant_path.name == "_default.yaml":
  163 + continue
  164 + tenant_config["tenants"][tenant_path.stem] = _load_yaml(tenant_path)
  165 + loaded_files.append(str(tenant_path))
  166 + raw["tenant_config"] = tenant_config
  167 +
  168 + return raw, loaded_files
  169 +
  170 + def _build_app_config(self, raw: Dict[str, Any], loaded_files: List[str]) -> AppConfig:
  171 + assets_cfg = raw.get("assets") if isinstance(raw.get("assets"), dict) else {}
  172 + rewrite_path = (
  173 + assets_cfg.get("query_rewrite_dictionary_path")
  174 + or assets_cfg.get("rewrite_dictionary_path")
  175 + or self.config_dir / "dictionaries" / "query_rewrite.dict"
  176 + )
  177 + rewrite_path = Path(rewrite_path)
  178 + if not rewrite_path.is_absolute():
  179 + rewrite_path = (self.project_root / rewrite_path).resolve()
  180 + if not rewrite_path.exists():
  181 + legacy_rewrite_path = (self.config_dir / "query_rewrite.dict").resolve()
  182 + if legacy_rewrite_path.exists():
  183 + rewrite_path = legacy_rewrite_path
  184 +
  185 + rewrite_dictionary = _read_rewrite_dictionary(rewrite_path)
  186 + search_config = self._build_search_config(raw, rewrite_dictionary)
  187 + services_config = self._build_services_config(raw.get("services") or {})
  188 + tenants_config = self._build_tenants_config(raw.get("tenant_config") or {})
  189 + runtime_config = self._build_runtime_config()
  190 + infrastructure_config = self._build_infrastructure_config(runtime_config.environment)
  191 +
  192 + product_enrich_raw = raw.get("product_enrich") if isinstance(raw.get("product_enrich"), dict) else {}
  193 + product_enrich_config = ProductEnrichConfig(
  194 + max_workers=int(product_enrich_raw.get("max_workers", 40)),
  195 + )
  196 +
  197 + metadata = ConfigMetadata(
  198 + loaded_files=tuple(loaded_files),
  199 + config_hash="",
  200 + deprecated_keys=tuple(self._detect_deprecated_keys(raw)),
  201 + )
  202 +
  203 + app_config = AppConfig(
  204 + runtime=runtime_config,
  205 + infrastructure=infrastructure_config,
  206 + product_enrich=product_enrich_config,
  207 + search=search_config,
  208 + services=services_config,
  209 + tenants=tenants_config,
  210 + assets=AssetsConfig(query_rewrite_dictionary_path=rewrite_path),
  211 + metadata=metadata,
  212 + )
  213 +
  214 + config_hash = self._compute_hash(app_config)
  215 + return AppConfig(
  216 + runtime=app_config.runtime,
  217 + infrastructure=app_config.infrastructure,
  218 + product_enrich=app_config.product_enrich,
  219 + search=app_config.search,
  220 + services=app_config.services,
  221 + tenants=app_config.tenants,
  222 + assets=app_config.assets,
  223 + metadata=ConfigMetadata(
  224 + loaded_files=app_config.metadata.loaded_files,
  225 + config_hash=config_hash,
  226 + deprecated_keys=app_config.metadata.deprecated_keys,
  227 + ),
  228 + )
  229 +
  230 + def _build_search_config(self, raw: Dict[str, Any], rewrite_dictionary: Dict[str, str]) -> SearchConfig:
  231 + field_boosts = raw.get("field_boosts") or {}
  232 + if not isinstance(field_boosts, dict):
  233 + raise ConfigurationError("field_boosts must be a mapping")
  234 +
  235 + indexes: List[IndexConfig] = []
  236 + for item in raw.get("indexes") or []:
  237 + if not isinstance(item, dict):
  238 + raise ConfigurationError("indexes items must be mappings")
  239 + indexes.append(
  240 + IndexConfig(
  241 + name=str(item["name"]),
  242 + label=str(item.get("label") or item["name"]),
  243 + fields=list(item.get("fields") or []),
  244 + boost=float(item.get("boost", 1.0)),
  245 + example=item.get("example"),
  246 + )
  247 + )
  248 +
  249 + query_cfg = raw.get("query_config") if isinstance(raw.get("query_config"), dict) else {}
  250 + search_fields = query_cfg.get("search_fields") if isinstance(query_cfg.get("search_fields"), dict) else {}
  251 + text_strategy = (
  252 + query_cfg.get("text_query_strategy")
  253 + if isinstance(query_cfg.get("text_query_strategy"), dict)
  254 + else {}
  255 + )
  256 + query_config = QueryConfig(
  257 + supported_languages=list(query_cfg.get("supported_languages") or ["zh", "en"]),
  258 + default_language=str(query_cfg.get("default_language") or "en"),
  259 + enable_text_embedding=bool(query_cfg.get("enable_text_embedding", True)),
  260 + enable_query_rewrite=bool(query_cfg.get("enable_query_rewrite", True)),
  261 + rewrite_dictionary=rewrite_dictionary,
  262 + text_embedding_field=query_cfg.get("text_embedding_field"),
  263 + image_embedding_field=query_cfg.get("image_embedding_field"),
  264 + source_fields=query_cfg.get("source_fields"),
  265 + knn_boost=float(query_cfg.get("knn_boost", 0.25)),
  266 + multilingual_fields=list(
  267 + search_fields.get(
  268 + "multilingual_fields",
  269 + ["title", "brief", "description", "vendor", "category_path", "category_name_text"],
  270 + )
  271 + ),
  272 + shared_fields=list(
  273 + search_fields.get(
  274 + "shared_fields",
  275 + ["tags", "option1_values", "option2_values", "option3_values"],
  276 + )
  277 + ),
  278 + core_multilingual_fields=list(
  279 + search_fields.get(
  280 + "core_multilingual_fields",
  281 + ["title", "brief", "vendor", "category_name_text"],
  282 + )
  283 + ),
  284 + base_minimum_should_match=str(text_strategy.get("base_minimum_should_match", "75%")),
  285 + translation_minimum_should_match=str(text_strategy.get("translation_minimum_should_match", "75%")),
  286 + translation_boost=float(text_strategy.get("translation_boost", 0.4)),
  287 + translation_boost_when_source_missing=float(
  288 + text_strategy.get("translation_boost_when_source_missing", 1.0)
  289 + ),
  290 + source_boost_when_missing=float(text_strategy.get("source_boost_when_missing", 0.6)),
  291 + original_query_fallback_boost_when_translation_missing=float(
  292 + text_strategy.get("original_query_fallback_boost_when_translation_missing", 0.2)
  293 + ),
  294 + tie_breaker_base_query=float(text_strategy.get("tie_breaker_base_query", 0.9)),
  295 + zh_to_en_model=str(query_cfg.get("zh_to_en_model") or "opus-mt-zh-en"),
  296 + en_to_zh_model=str(query_cfg.get("en_to_zh_model") or "opus-mt-en-zh"),
  297 + default_translation_model=str(
  298 + query_cfg.get("default_translation_model") or "nllb-200-distilled-600m"
  299 + ),
  300 + )
  301 +
  302 + function_score_cfg = raw.get("function_score") if isinstance(raw.get("function_score"), dict) else {}
  303 + rerank_cfg = raw.get("rerank") if isinstance(raw.get("rerank"), dict) else {}
  304 + spu_cfg = raw.get("spu_config") if isinstance(raw.get("spu_config"), dict) else {}
  305 +
  306 + return SearchConfig(
  307 + field_boosts={str(key): float(value) for key, value in field_boosts.items()},
  308 + indexes=indexes,
  309 + query_config=query_config,
  310 + function_score=FunctionScoreConfig(
  311 + score_mode=str(function_score_cfg.get("score_mode") or "sum"),
  312 + boost_mode=str(function_score_cfg.get("boost_mode") or "multiply"),
  313 + functions=list(function_score_cfg.get("functions") or []),
  314 + ),
  315 + rerank=RerankConfig(
  316 + enabled=bool(rerank_cfg.get("enabled", True)),
  317 + rerank_window=int(rerank_cfg.get("rerank_window", 384)),
  318 + timeout_sec=float(rerank_cfg.get("timeout_sec", 15.0)),
  319 + weight_es=float(rerank_cfg.get("weight_es", 0.4)),
  320 + weight_ai=float(rerank_cfg.get("weight_ai", 0.6)),
  321 + rerank_query_template=str(rerank_cfg.get("rerank_query_template") or "{query}"),
  322 + rerank_doc_template=str(rerank_cfg.get("rerank_doc_template") or "{title}"),
  323 + ),
  324 + spu_config=SPUConfig(
  325 + enabled=bool(spu_cfg.get("enabled", False)),
  326 + spu_field=spu_cfg.get("spu_field"),
  327 + inner_hits_size=int(spu_cfg.get("inner_hits_size", 3)),
  328 + searchable_option_dimensions=list(
  329 + spu_cfg.get("searchable_option_dimensions") or ["option1", "option2", "option3"]
  330 + ),
  331 + ),
  332 + es_index_name=str(raw.get("es_index_name") or "search_products"),
  333 + es_settings=dict(raw.get("es_settings") or {}),
  334 + )
  335 +
  336 + def _build_services_config(self, raw: Dict[str, Any]) -> ServicesConfig:
  337 + if not isinstance(raw, dict):
  338 + raise ConfigurationError("services must be a mapping")
  339 +
  340 + translation_raw = raw.get("translation") if isinstance(raw.get("translation"), dict) else {}
  341 + normalized_translation = build_translation_config(translation_raw)
  342 + translation_config = TranslationServiceConfig(
  343 + endpoint=str(normalized_translation["service_url"]).rstrip("/"),
  344 + timeout_sec=float(normalized_translation["timeout_sec"]),
  345 + default_model=str(normalized_translation["default_model"]),
  346 + default_scene=str(normalized_translation["default_scene"]),
  347 + cache=dict(normalized_translation["cache"]),
  348 + capabilities={str(key): dict(value) for key, value in normalized_translation["capabilities"].items()},
  349 + )
  350 +
  351 + embedding_raw = raw.get("embedding") if isinstance(raw.get("embedding"), dict) else {}
  352 + embedding_provider = str(embedding_raw.get("provider") or "http").strip().lower()
  353 + embedding_providers = dict(embedding_raw.get("providers") or {})
  354 + if embedding_provider not in embedding_providers:
  355 + raise ConfigurationError(f"services.embedding.providers.{embedding_provider} must be configured")
  356 + embedding_backend = str(embedding_raw.get("backend") or "").strip().lower()
  357 + embedding_backends = {
  358 + str(key).strip().lower(): dict(value)
  359 + for key, value in dict(embedding_raw.get("backends") or {}).items()
  360 + }
  361 + if embedding_backend not in embedding_backends:
  362 + raise ConfigurationError(f"services.embedding.backends.{embedding_backend} must be configured")
  363 + image_backend = str(embedding_raw.get("image_backend") or "clip_as_service").strip().lower()
  364 + image_backends = {
  365 + str(key).strip().lower(): dict(value)
  366 + for key, value in dict(embedding_raw.get("image_backends") or {}).items()
  367 + }
  368 + if not image_backends:
  369 + image_backends = {
  370 + "clip_as_service": {
  371 + "server": "grpc://127.0.0.1:51000",
  372 + "model_name": "CN-CLIP/ViT-L-14",
  373 + "batch_size": 8,
  374 + "normalize_embeddings": True,
  375 + },
  376 + "local_cnclip": {
  377 + "model_name": "ViT-L-14",
  378 + "device": None,
  379 + "batch_size": 8,
  380 + "normalize_embeddings": True,
  381 + },
  382 + }
  383 + if image_backend not in image_backends:
  384 + raise ConfigurationError(f"services.embedding.image_backends.{image_backend} must be configured")
  385 +
  386 + embedding_config = EmbeddingServiceConfig(
  387 + provider=embedding_provider,
  388 + providers=embedding_providers,
  389 + backend=embedding_backend,
  390 + backends=embedding_backends,
  391 + image_backend=image_backend,
  392 + image_backends=image_backends,
  393 + )
  394 +
  395 + rerank_raw = raw.get("rerank") if isinstance(raw.get("rerank"), dict) else {}
  396 + rerank_provider = str(rerank_raw.get("provider") or "http").strip().lower()
  397 + rerank_providers = dict(rerank_raw.get("providers") or {})
  398 + if rerank_provider not in rerank_providers:
  399 + raise ConfigurationError(f"services.rerank.providers.{rerank_provider} must be configured")
  400 + rerank_backend = str(rerank_raw.get("backend") or "").strip().lower()
  401 + rerank_backends = {
  402 + str(key).strip().lower(): dict(value)
  403 + for key, value in dict(rerank_raw.get("backends") or {}).items()
  404 + }
  405 + if rerank_backend not in rerank_backends:
  406 + raise ConfigurationError(f"services.rerank.backends.{rerank_backend} must be configured")
  407 + rerank_request = dict(rerank_raw.get("request") or {})
  408 + rerank_request.setdefault("max_docs", 1000)
  409 + rerank_request.setdefault("normalize", True)
  410 +
  411 + rerank_config = RerankServiceConfig(
  412 + provider=rerank_provider,
  413 + providers=rerank_providers,
  414 + backend=rerank_backend,
  415 + backends=rerank_backends,
  416 + request=rerank_request,
  417 + )
  418 +
  419 + return ServicesConfig(
  420 + translation=translation_config,
  421 + embedding=embedding_config,
  422 + rerank=rerank_config,
  423 + )
  424 +
  425 + def _build_tenants_config(self, raw: Dict[str, Any]) -> TenantCatalogConfig:
  426 + if not isinstance(raw, dict):
  427 + raise ConfigurationError("tenant_config must be a mapping")
  428 + default_cfg = raw.get("default") if isinstance(raw.get("default"), dict) else {}
  429 + tenants_cfg = raw.get("tenants") if isinstance(raw.get("tenants"), dict) else {}
  430 + return TenantCatalogConfig(
  431 + default=dict(default_cfg),
  432 + tenants={str(key): dict(value) for key, value in tenants_cfg.items()},
  433 + )
  434 +
  435 + def _build_runtime_config(self) -> RuntimeConfig:
  436 + environment = (os.getenv("APP_ENV") or os.getenv("RUNTIME_ENV") or "prod").strip().lower() or "prod"
  437 + namespace = os.getenv("ES_INDEX_NAMESPACE")
  438 + if namespace is None:
  439 + namespace = "" if environment == "prod" else f"{environment}_"
  440 +
  441 + return RuntimeConfig(
  442 + environment=environment,
  443 + index_namespace=namespace,
  444 + api_host=os.getenv("API_HOST", "0.0.0.0"),
  445 + api_port=int(os.getenv("API_PORT", 6002)),
  446 + indexer_host=os.getenv("INDEXER_HOST", "0.0.0.0"),
  447 + indexer_port=int(os.getenv("INDEXER_PORT", 6004)),
  448 + embedding_host=os.getenv("EMBEDDING_HOST", "127.0.0.1"),
  449 + embedding_port=int(os.getenv("EMBEDDING_PORT", 6005)),
  450 + embedding_text_port=int(os.getenv("EMBEDDING_TEXT_PORT", 6005)),
  451 + embedding_image_port=int(os.getenv("EMBEDDING_IMAGE_PORT", 6008)),
  452 + translator_host=os.getenv("TRANSLATION_HOST", "127.0.0.1"),
  453 + translator_port=int(os.getenv("TRANSLATION_PORT", 6006)),
  454 + reranker_host=os.getenv("RERANKER_HOST", "127.0.0.1"),
  455 + reranker_port=int(os.getenv("RERANKER_PORT", 6007)),
  456 + )
  457 +
  458 + def _build_infrastructure_config(self, environment: str) -> InfrastructureConfig:
  459 + del environment
  460 + return InfrastructureConfig(
  461 + elasticsearch=ElasticsearchSettings(
  462 + host=os.getenv("ES_HOST", "http://localhost:9200"),
  463 + username=os.getenv("ES_USERNAME"),
  464 + password=os.getenv("ES_PASSWORD"),
  465 + ),
  466 + redis=RedisSettings(
  467 + host=os.getenv("REDIS_HOST", "localhost"),
  468 + port=int(os.getenv("REDIS_PORT", 6479)),
  469 + snapshot_db=int(os.getenv("REDIS_SNAPSHOT_DB", 0)),
  470 + password=os.getenv("REDIS_PASSWORD"),
  471 + socket_timeout=int(os.getenv("REDIS_SOCKET_TIMEOUT", 1)),
  472 + socket_connect_timeout=int(os.getenv("REDIS_SOCKET_CONNECT_TIMEOUT", 1)),
  473 + retry_on_timeout=os.getenv("REDIS_RETRY_ON_TIMEOUT", "false").strip().lower() == "true",
  474 + cache_expire_days=int(os.getenv("REDIS_CACHE_EXPIRE_DAYS", 360 * 2)),
  475 + embedding_cache_prefix=os.getenv("REDIS_EMBEDDING_CACHE_PREFIX", "embedding"),
  476 + anchor_cache_prefix=os.getenv("REDIS_ANCHOR_CACHE_PREFIX", "product_anchors"),
  477 + anchor_cache_expire_days=int(os.getenv("REDIS_ANCHOR_CACHE_EXPIRE_DAYS", 30)),
  478 + ),
  479 + database=DatabaseSettings(
  480 + host=os.getenv("DB_HOST"),
  481 + port=int(os.getenv("DB_PORT", 3306)) if os.getenv("DB_PORT") else 3306,
  482 + database=os.getenv("DB_DATABASE"),
  483 + username=os.getenv("DB_USERNAME"),
  484 + password=os.getenv("DB_PASSWORD"),
  485 + ),
  486 + secrets=SecretsConfig(
  487 + dashscope_api_key=os.getenv("DASHSCOPE_API_KEY"),
  488 + deepl_auth_key=os.getenv("DEEPL_AUTH_KEY"),
  489 + ),
  490 + )
  491 +
  492 + def _validate(self, app_config: AppConfig) -> None:
  493 + errors: List[str] = []
  494 +
  495 + if not app_config.search.es_index_name:
  496 + errors.append("search.es_index_name is required")
  497 +
  498 + if not app_config.search.field_boosts:
  499 + errors.append("search.field_boosts cannot be empty")
  500 + else:
  501 + for field_name, boost in app_config.search.field_boosts.items():
  502 + if boost < 0:
  503 + errors.append(f"field_boosts.{field_name} must be non-negative")
  504 +
  505 + query_config = app_config.search.query_config
  506 + if not query_config.supported_languages:
  507 + errors.append("query_config.supported_languages must not be empty")
  508 + if query_config.default_language not in query_config.supported_languages:
  509 + errors.append("query_config.default_language must be included in supported_languages")
  510 + for name, values in (
  511 + ("multilingual_fields", query_config.multilingual_fields),
  512 + ("shared_fields", query_config.shared_fields),
  513 + ("core_multilingual_fields", query_config.core_multilingual_fields),
  514 + ):
  515 + if not values:
  516 + errors.append(f"query_config.{name} must not be empty")
  517 +
  518 + if not set(query_config.core_multilingual_fields).issubset(set(query_config.multilingual_fields)):
  519 + errors.append("query_config.core_multilingual_fields must be a subset of multilingual_fields")
  520 +
  521 + if app_config.search.spu_config.enabled and not app_config.search.spu_config.spu_field:
  522 + errors.append("spu_config.spu_field is required when spu_config.enabled is true")
  523 +
  524 + if not app_config.tenants.default or not app_config.tenants.default.get("index_languages"):
  525 + errors.append("tenant_config.default.index_languages must be configured")
  526 +
  527 + if app_config.metadata.deprecated_keys:
  528 + errors.append(
  529 + "Deprecated tenant config keys are not supported: "
  530 + + ", ".join(app_config.metadata.deprecated_keys)
  531 + )
  532 +
  533 + embedding_provider_cfg = app_config.services.embedding.get_provider_config()
  534 + if not embedding_provider_cfg.get("text_base_url"):
  535 + errors.append("services.embedding.providers.<provider>.text_base_url is required")
  536 + if not embedding_provider_cfg.get("image_base_url"):
  537 + errors.append("services.embedding.providers.<provider>.image_base_url is required")
  538 +
  539 + rerank_provider_cfg = app_config.services.rerank.get_provider_config()
  540 + if not rerank_provider_cfg.get("service_url") and not rerank_provider_cfg.get("base_url"):
  541 + errors.append("services.rerank.providers.<provider>.service_url or base_url is required")
  542 +
  543 + if errors:
  544 + raise ConfigurationError("Configuration validation failed:\n" + "\n".join(f" - {err}" for err in errors))
  545 +
  546 + def _compute_hash(self, app_config: AppConfig) -> str:
  547 + payload = asdict(app_config)
  548 + payload["metadata"]["config_hash"] = ""
  549 + payload["infrastructure"]["elasticsearch"]["password"] = "***" if payload["infrastructure"]["elasticsearch"].get("password") else None
  550 + payload["infrastructure"]["database"]["password"] = "***" if payload["infrastructure"]["database"].get("password") else None
  551 + payload["infrastructure"]["redis"]["password"] = "***" if payload["infrastructure"]["redis"].get("password") else None
  552 + payload["infrastructure"]["secrets"]["dashscope_api_key"] = "***" if payload["infrastructure"]["secrets"].get("dashscope_api_key") else None
  553 + payload["infrastructure"]["secrets"]["deepl_auth_key"] = "***" if payload["infrastructure"]["secrets"].get("deepl_auth_key") else None
  554 + blob = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str)
  555 + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:16]
  556 +
  557 + def _detect_deprecated_keys(self, raw: Dict[str, Any]) -> Iterable[str]:
  558 + # Translation-era legacy flags have been removed; keep the hook for future
  559 + # deprecations, but currently no deprecated keys are detected.
  560 + return ()
  561 +
  562 +
  563 +@lru_cache(maxsize=1)
  564 +def get_app_config() -> AppConfig:
  565 + """Return the process-global application configuration."""
  566 +
  567 + return AppConfigLoader().load()
  568 +
  569 +
  570 +def reload_app_config() -> AppConfig:
  571 + """Clear the cached configuration and reload it."""
  572 +
  573 + get_app_config.cache_clear()
  574 + return get_app_config()
  575 +
  576 +
  577 +def _load_env_file_fallback(path: Path) -> None:
  578 + if not path.exists():
  579 + return
  580 + with open(path, "r", encoding="utf-8") as handle:
  581 + for raw_line in handle:
  582 + line = raw_line.strip()
  583 + if not line or line.startswith("#") or "=" not in line:
  584 + continue
  585 + key, value = line.split("=", 1)
  586 + key = key.strip()
  587 + value = value.strip().strip('"').strip("'")
  588 + if key and key not in os.environ:
  589 + os.environ[key] = value
... ...
config/schema.py 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,315 @@
  1 +"""
  2 +Typed configuration schema for the unified application configuration.
  3 +
  4 +This module defines the normalized in-memory structure used by all services.
  5 +"""
  6 +
  7 +from __future__ import annotations
  8 +
  9 +from dataclasses import asdict, dataclass, field
  10 +from pathlib import Path
  11 +from typing import Any, Dict, List, Optional, Tuple
  12 +
  13 +
  14 +@dataclass(frozen=True)
  15 +class IndexConfig:
  16 + """Deprecated compatibility shape for legacy diagnostics/tests."""
  17 +
  18 + name: str
  19 + label: str
  20 + fields: List[str]
  21 + boost: float = 1.0
  22 + example: Optional[str] = None
  23 +
  24 +
  25 +@dataclass(frozen=True)
  26 +class QueryConfig:
  27 + """Configuration for query processing."""
  28 +
  29 + supported_languages: List[str] = field(default_factory=lambda: ["zh", "en"])
  30 + default_language: str = "en"
  31 + enable_text_embedding: bool = True
  32 + enable_query_rewrite: bool = True
  33 + rewrite_dictionary: Dict[str, str] = field(default_factory=dict)
  34 + text_embedding_field: Optional[str] = "title_embedding"
  35 + image_embedding_field: Optional[str] = None
  36 + source_fields: Optional[List[str]] = None
  37 + knn_boost: float = 0.25
  38 + multilingual_fields: List[str] = field(
  39 + default_factory=lambda: [
  40 + "title",
  41 + "brief",
  42 + "description",
  43 + "vendor",
  44 + "category_path",
  45 + "category_name_text",
  46 + ]
  47 + )
  48 + shared_fields: List[str] = field(
  49 + default_factory=lambda: ["tags", "option1_values", "option2_values", "option3_values"]
  50 + )
  51 + core_multilingual_fields: List[str] = field(
  52 + default_factory=lambda: ["title", "brief", "vendor", "category_name_text"]
  53 + )
  54 + base_minimum_should_match: str = "75%"
  55 + translation_minimum_should_match: str = "75%"
  56 + translation_boost: float = 0.4
  57 + translation_boost_when_source_missing: float = 1.0
  58 + source_boost_when_missing: float = 0.6
  59 + original_query_fallback_boost_when_translation_missing: float = 0.2
  60 + tie_breaker_base_query: float = 0.9
  61 + zh_to_en_model: str = "opus-mt-zh-en"
  62 + en_to_zh_model: str = "opus-mt-en-zh"
  63 + default_translation_model: str = "nllb-200-distilled-600m"
  64 +
  65 +
  66 +@dataclass(frozen=True)
  67 +class SPUConfig:
  68 + """SPU aggregation/search configuration."""
  69 +
  70 + enabled: bool = False
  71 + spu_field: Optional[str] = None
  72 + inner_hits_size: int = 3
  73 + searchable_option_dimensions: List[str] = field(
  74 + default_factory=lambda: ["option1", "option2", "option3"]
  75 + )
  76 +
  77 +
  78 +@dataclass(frozen=True)
  79 +class FunctionScoreConfig:
  80 + """Function score configuration."""
  81 +
  82 + score_mode: str = "sum"
  83 + boost_mode: str = "multiply"
  84 + functions: List[Dict[str, Any]] = field(default_factory=list)
  85 +
  86 +
  87 +@dataclass(frozen=True)
  88 +class RerankConfig:
  89 + """Search-time rerank configuration."""
  90 +
  91 + enabled: bool = True
  92 + rerank_window: int = 384
  93 + timeout_sec: float = 15.0
  94 + weight_es: float = 0.4
  95 + weight_ai: float = 0.6
  96 + rerank_query_template: str = "{query}"
  97 + rerank_doc_template: str = "{title}"
  98 +
  99 +
  100 +@dataclass(frozen=True)
  101 +class SearchConfig:
  102 + """Search behavior configuration shared by backend and indexer."""
  103 +
  104 + field_boosts: Dict[str, float]
  105 + indexes: List[IndexConfig] = field(default_factory=list)
  106 + query_config: QueryConfig = field(default_factory=QueryConfig)
  107 + function_score: FunctionScoreConfig = field(default_factory=FunctionScoreConfig)
  108 + rerank: RerankConfig = field(default_factory=RerankConfig)
  109 + spu_config: SPUConfig = field(default_factory=SPUConfig)
  110 + es_index_name: str = "search_products"
  111 + es_settings: Dict[str, Any] = field(default_factory=dict)
  112 +
  113 +
  114 +@dataclass(frozen=True)
  115 +class TranslationServiceConfig:
  116 + """Translator service configuration."""
  117 +
  118 + endpoint: str
  119 + timeout_sec: float
  120 + default_model: str
  121 + default_scene: str
  122 + cache: Dict[str, Any]
  123 + capabilities: Dict[str, Dict[str, Any]]
  124 +
  125 + def as_dict(self) -> Dict[str, Any]:
  126 + return {
  127 + "service_url": self.endpoint,
  128 + "timeout_sec": self.timeout_sec,
  129 + "default_model": self.default_model,
  130 + "default_scene": self.default_scene,
  131 + "cache": self.cache,
  132 + "capabilities": self.capabilities,
  133 + }
  134 +
  135 +
  136 +@dataclass(frozen=True)
  137 +class EmbeddingServiceConfig:
  138 + """Embedding service configuration."""
  139 +
  140 + provider: str
  141 + providers: Dict[str, Any]
  142 + backend: str
  143 + backends: Dict[str, Dict[str, Any]]
  144 + image_backend: str
  145 + image_backends: Dict[str, Dict[str, Any]]
  146 +
  147 + def get_provider_config(self) -> Dict[str, Any]:
  148 + return dict(self.providers.get(self.provider, {}) or {})
  149 +
  150 + def get_backend_config(self) -> Dict[str, Any]:
  151 + return dict(self.backends.get(self.backend, {}) or {})
  152 +
  153 + def get_image_backend_config(self) -> Dict[str, Any]:
  154 + return dict(self.image_backends.get(self.image_backend, {}) or {})
  155 +
  156 +
  157 +@dataclass(frozen=True)
  158 +class RerankServiceConfig:
  159 + """Reranker service configuration."""
  160 +
  161 + provider: str
  162 + providers: Dict[str, Any]
  163 + backend: str
  164 + backends: Dict[str, Dict[str, Any]]
  165 + request: Dict[str, Any]
  166 +
  167 + def get_provider_config(self) -> Dict[str, Any]:
  168 + return dict(self.providers.get(self.provider, {}) or {})
  169 +
  170 + def get_backend_config(self) -> Dict[str, Any]:
  171 + return dict(self.backends.get(self.backend, {}) or {})
  172 +
  173 +
  174 +@dataclass(frozen=True)
  175 +class ServicesConfig:
  176 + """All service-level configuration."""
  177 +
  178 + translation: TranslationServiceConfig
  179 + embedding: EmbeddingServiceConfig
  180 + rerank: RerankServiceConfig
  181 +
  182 +
  183 +@dataclass(frozen=True)
  184 +class TenantCatalogConfig:
  185 + """Tenant catalog configuration."""
  186 +
  187 + default: Dict[str, Any]
  188 + tenants: Dict[str, Dict[str, Any]]
  189 +
  190 + def get_raw(self) -> Dict[str, Any]:
  191 + return {
  192 + "default": dict(self.default),
  193 + "tenants": {str(key): dict(value) for key, value in self.tenants.items()},
  194 + }
  195 +
  196 +
  197 +@dataclass(frozen=True)
  198 +class ElasticsearchSettings:
  199 + host: str = "http://localhost:9200"
  200 + username: Optional[str] = None
  201 + password: Optional[str] = None
  202 +
  203 +
  204 +@dataclass(frozen=True)
  205 +class RedisSettings:
  206 + host: str = "localhost"
  207 + port: int = 6479
  208 + snapshot_db: int = 0
  209 + password: Optional[str] = None
  210 + socket_timeout: int = 1
  211 + socket_connect_timeout: int = 1
  212 + retry_on_timeout: bool = False
  213 + cache_expire_days: int = 720
  214 + embedding_cache_prefix: str = "embedding"
  215 + anchor_cache_prefix: str = "product_anchors"
  216 + anchor_cache_expire_days: int = 30
  217 +
  218 +
  219 +@dataclass(frozen=True)
  220 +class DatabaseSettings:
  221 + host: Optional[str] = None
  222 + port: int = 3306
  223 + database: Optional[str] = None
  224 + username: Optional[str] = None
  225 + password: Optional[str] = None
  226 +
  227 +
  228 +@dataclass(frozen=True)
  229 +class SecretsConfig:
  230 + dashscope_api_key: Optional[str] = None
  231 + deepl_auth_key: Optional[str] = None
  232 +
  233 +
  234 +@dataclass(frozen=True)
  235 +class InfrastructureConfig:
  236 + elasticsearch: ElasticsearchSettings
  237 + redis: RedisSettings
  238 + database: DatabaseSettings
  239 + secrets: SecretsConfig
  240 +
  241 +
  242 +@dataclass(frozen=True)
  243 +class ProductEnrichConfig:
  244 + """Configuration for LLM-based product content understanding (enrich-content)."""
  245 +
  246 + max_workers: int = 40
  247 +
  248 +
  249 +@dataclass(frozen=True)
  250 +class RuntimeConfig:
  251 + environment: str = "prod"
  252 + index_namespace: str = ""
  253 + api_host: str = "0.0.0.0"
  254 + api_port: int = 6002
  255 + indexer_host: str = "0.0.0.0"
  256 + indexer_port: int = 6004
  257 + embedding_host: str = "127.0.0.1"
  258 + embedding_port: int = 6005
  259 + embedding_text_port: int = 6005
  260 + embedding_image_port: int = 6008
  261 + translator_host: str = "127.0.0.1"
  262 + translator_port: int = 6006
  263 + reranker_host: str = "127.0.0.1"
  264 + reranker_port: int = 6007
  265 +
  266 +
  267 +@dataclass(frozen=True)
  268 +class AssetsConfig:
  269 + query_rewrite_dictionary_path: Path
  270 +
  271 +
  272 +@dataclass(frozen=True)
  273 +class ConfigMetadata:
  274 + loaded_files: Tuple[str, ...]
  275 + config_hash: str
  276 + deprecated_keys: Tuple[str, ...] = field(default_factory=tuple)
  277 +
  278 +
  279 +@dataclass(frozen=True)
  280 +class AppConfig:
  281 + """Root application configuration."""
  282 +
  283 + runtime: RuntimeConfig
  284 + infrastructure: InfrastructureConfig
  285 + product_enrich: ProductEnrichConfig
  286 + search: SearchConfig
  287 + services: ServicesConfig
  288 + tenants: TenantCatalogConfig
  289 + assets: AssetsConfig
  290 + metadata: ConfigMetadata
  291 +
  292 + def sanitized_dict(self) -> Dict[str, Any]:
  293 + data = asdict(self)
  294 + data["infrastructure"]["elasticsearch"]["password"] = _mask_secret(
  295 + data["infrastructure"]["elasticsearch"].get("password")
  296 + )
  297 + data["infrastructure"]["database"]["password"] = _mask_secret(
  298 + data["infrastructure"]["database"].get("password")
  299 + )
  300 + data["infrastructure"]["redis"]["password"] = _mask_secret(
  301 + data["infrastructure"]["redis"].get("password")
  302 + )
  303 + data["infrastructure"]["secrets"]["dashscope_api_key"] = _mask_secret(
  304 + data["infrastructure"]["secrets"].get("dashscope_api_key")
  305 + )
  306 + data["infrastructure"]["secrets"]["deepl_auth_key"] = _mask_secret(
  307 + data["infrastructure"]["secrets"].get("deepl_auth_key")
  308 + )
  309 + return data
  310 +
  311 +
  312 +def _mask_secret(value: Optional[str]) -> Optional[str]:
  313 + if not value:
  314 + return value
  315 + return "***"
... ...
config/services_config.py
  Show/Hide comments   View file @0342d89
1 1 """
2   -Services configuration - single source for translation, embedding, rerank.
  2 +Unified service configuration accessors.
3 3  
4   -Translation is modeled as:
5   -- one translator service endpoint used by business callers
6   -- multiple translation capabilities loaded inside the translator service
  4 +This module is now a thin adapter over ``config.loader.get_app_config`` and
  5 +contains no independent parsing or precedence logic.
7 6 """
8 7  
9 8 from __future__ import annotations
10 9  
11   -import os
12   -from dataclasses import dataclass, field
13   -from functools import lru_cache
14   -from pathlib import Path
15   -from typing import Any, Dict, List, Optional
16   -
17   -import yaml
18   -from translation.settings import TranslationConfig, build_translation_config, get_translation_cache
19   -
20   -
21   -@dataclass
22   -class ServiceConfig:
23   - """Config for one capability (embedding/rerank)."""
24   -
25   - provider: str
26   - providers: Dict[str, Any] = field(default_factory=dict)
27   -
28   - def get_provider_cfg(self) -> Dict[str, Any]:
29   - p = (self.provider or "").strip().lower()
30   - return self.providers.get(p, {}) if isinstance(self.providers, dict) else {}
31   -
32   -
33   -def _load_services_raw(config_path: Optional[Path] = None) -> Dict[str, Any]:
34   - if config_path is None:
35   - config_path = Path(__file__).parent / "config.yaml"
36   - path = Path(config_path)
37   - if not path.exists():
38   - raise FileNotFoundError(f"services config file not found: {path}")
39   - try:
40   - with open(path, "r", encoding="utf-8") as f:
41   - data = yaml.safe_load(f)
42   - except Exception as exc:
43   - raise RuntimeError(f"failed to parse services config from {path}: {exc}") from exc
44   - if not isinstance(data, dict):
45   - raise RuntimeError(f"invalid config format in {path}: expected mapping root")
46   - services = data.get("services")
47   - if not isinstance(services, dict):
48   - raise RuntimeError("config.yaml must contain a valid 'services' mapping")
49   - return services
50   -
51   -
52   -def _resolve_provider_name(env_name: str, config_provider: Any, capability: str) -> str:
53   - provider = os.getenv(env_name) or config_provider
54   - if not provider:
55   - raise ValueError(
56   - f"services.{capability}.provider is required "
57   - f"(or set env override {env_name})"
58   - )
59   - return str(provider).strip().lower()
60   -
61   -
62   -def _resolve_translation() -> TranslationConfig:
63   - raw = _load_services_raw()
64   - cfg = raw.get("translation", {}) if isinstance(raw.get("translation"), dict) else {}
65   - return build_translation_config(cfg)
66   -
67   -
68   -def _resolve_embedding() -> ServiceConfig:
69   - raw = _load_services_raw()
70   - cfg = raw.get("embedding", {}) if isinstance(raw.get("embedding"), dict) else {}
71   - providers = cfg.get("providers", {}) if isinstance(cfg.get("providers"), dict) else {}
72   -
73   - provider = _resolve_provider_name(
74   - env_name="EMBEDDING_PROVIDER",
75   - config_provider=cfg.get("provider"),
76   - capability="embedding",
77   - )
78   - if provider != "http":
79   - raise ValueError(f"Unsupported embedding provider: {provider}")
80   -
81   - env_text_url = os.getenv("EMBEDDING_TEXT_SERVICE_URL")
82   - env_image_url = os.getenv("EMBEDDING_IMAGE_SERVICE_URL")
83   - if provider == "http":
84   - providers = dict(providers)
85   - http_cfg = dict(providers.get("http", {}))
86   - if env_text_url:
87   - http_cfg["text_base_url"] = env_text_url.rstrip("/")
88   - if env_image_url:
89   - http_cfg["image_base_url"] = env_image_url.rstrip("/")
90   - if not http_cfg.get("text_base_url"):
91   - raise ValueError("services.embedding.providers.http.text_base_url is required")
92   - if not http_cfg.get("image_base_url"):
93   - raise ValueError("services.embedding.providers.http.image_base_url is required")
94   - providers["http"] = http_cfg
95   -
96   - return ServiceConfig(provider=provider, providers=providers)
97   -
98   -
99   -def _resolve_rerank() -> ServiceConfig:
100   - raw = _load_services_raw()
101   - cfg = raw.get("rerank", {}) if isinstance(raw.get("rerank"), dict) else {}
102   - providers = cfg.get("providers", {}) if isinstance(cfg.get("providers"), dict) else {}
103   -
104   - provider = _resolve_provider_name(
105   - env_name="RERANK_PROVIDER",
106   - config_provider=cfg.get("provider"),
107   - capability="rerank",
108   - )
109   - if provider != "http":
110   - raise ValueError(f"Unsupported rerank provider: {provider}")
111   -
112   - env_url = os.getenv("RERANKER_SERVICE_URL")
113   - if env_url:
114   - url = env_url.rstrip("/")
115   - if not url.endswith("/rerank"):
116   - url = f"{url}/rerank" if "/rerank" not in url else url
117   - providers = dict(providers)
118   - providers["http"] = dict(providers.get("http", {}))
119   - providers["http"]["base_url"] = url.replace("/rerank", "")
120   - providers["http"]["service_url"] = url
121   -
122   - return ServiceConfig(provider=provider, providers=providers)
123   -
124   -
125   -def get_rerank_backend_config() -> tuple[str, dict]:
126   - raw = _load_services_raw()
127   - cfg = raw.get("rerank", {}) if isinstance(raw.get("rerank"), dict) else {}
128   - backends = cfg.get("backends", {}) if isinstance(cfg.get("backends"), dict) else {}
129   - name = os.getenv("RERANK_BACKEND") or cfg.get("backend")
130   - if not name:
131   - raise ValueError("services.rerank.backend is required (or env RERANK_BACKEND)")
132   - name = str(name).strip().lower()
133   - backend_cfg = backends.get(name, {}) if isinstance(backends.get(name), dict) else {}
134   - if not backend_cfg:
135   - raise ValueError(f"services.rerank.backends.{name} is required")
136   - return name, backend_cfg
137   -
138   -
139   -def get_embedding_backend_config() -> tuple[str, dict]:
140   - raw = _load_services_raw()
141   - cfg = raw.get("embedding", {}) if isinstance(raw.get("embedding"), dict) else {}
142   - backends = cfg.get("backends", {}) if isinstance(cfg.get("backends"), dict) else {}
143   - name = os.getenv("EMBEDDING_BACKEND") or cfg.get("backend")
144   - if not name:
145   - raise ValueError("services.embedding.backend is required (or env EMBEDDING_BACKEND)")
146   - name = str(name).strip().lower()
147   - backend_cfg = backends.get(name, {}) if isinstance(backends.get(name), dict) else {}
148   - if not backend_cfg:
149   - raise ValueError(f"services.embedding.backends.{name} is required")
150   - return name, backend_cfg
151   -
152   -
153   -@lru_cache(maxsize=1)
154   -def get_translation_config() -> TranslationConfig:
155   - return _resolve_translation()
156   -
157   -
158   -@lru_cache(maxsize=1)
159   -def get_embedding_config() -> ServiceConfig:
160   - return _resolve_embedding()
161   -
162   -
163   -@lru_cache(maxsize=1)
164   -def get_rerank_config() -> ServiceConfig:
165   - return _resolve_rerank()
  10 +from typing import Any, Dict, Tuple
  11 +
  12 +from config.loader import get_app_config
  13 +from config.schema import EmbeddingServiceConfig, RerankServiceConfig, TranslationServiceConfig
  14 +
  15 +
  16 +def get_translation_config() -> Dict[str, Any]:
  17 + return get_app_config().services.translation.as_dict()
  18 +
  19 +
  20 +def get_embedding_config() -> EmbeddingServiceConfig:
  21 + return get_app_config().services.embedding
  22 +
  23 +
  24 +def get_rerank_config() -> RerankServiceConfig:
  25 + return get_app_config().services.rerank
166 26  
167 27  
168 28 def get_translation_base_url() -> str:
169   - return str(get_translation_config()["service_url"])
  29 + return get_app_config().services.translation.endpoint
170 30  
171 31  
172 32 def get_translation_cache_config() -> Dict[str, Any]:
173   - return get_translation_cache(get_translation_config())
  33 + return dict(get_app_config().services.translation.cache)
174 34  
175 35  
176 36 def get_embedding_text_base_url() -> str:
177   - provider_cfg = get_embedding_config().providers.get("http", {})
178   - base = os.getenv("EMBEDDING_TEXT_SERVICE_URL") or provider_cfg.get("text_base_url")
  37 + provider_cfg = get_app_config().services.embedding.get_provider_config()
  38 + base = provider_cfg.get("text_base_url")
179 39 if not base:
180   - raise ValueError("Embedding text HTTP base_url is not configured")
  40 + raise ValueError("Embedding text base_url is not configured")
181 41 return str(base).rstrip("/")
182 42  
183 43  
184 44 def get_embedding_image_base_url() -> str:
185   - provider_cfg = get_embedding_config().providers.get("http", {})
186   - base = os.getenv("EMBEDDING_IMAGE_SERVICE_URL") or provider_cfg.get("image_base_url")
  45 + provider_cfg = get_app_config().services.embedding.get_provider_config()
  46 + base = provider_cfg.get("image_base_url")
187 47 if not base:
188   - raise ValueError("Embedding image HTTP base_url is not configured")
  48 + raise ValueError("Embedding image base_url is not configured")
189 49 return str(base).rstrip("/")
190 50  
191 51  
  52 +def get_embedding_backend_config() -> Tuple[str, Dict[str, Any]]:
  53 + cfg = get_app_config().services.embedding
  54 + return cfg.backend, cfg.get_backend_config()
  55 +
  56 +
  57 +def get_embedding_image_backend_config() -> Tuple[str, Dict[str, Any]]:
  58 + cfg = get_app_config().services.embedding
  59 + return cfg.image_backend, cfg.get_image_backend_config()
  60 +
  61 +
  62 +def get_rerank_backend_config() -> Tuple[str, Dict[str, Any]]:
  63 + cfg = get_app_config().services.rerank
  64 + return cfg.backend, cfg.get_backend_config()
  65 +
  66 +
192 67 def get_rerank_base_url() -> str:
193   - base = (
194   - os.getenv("RERANKER_SERVICE_URL")
195   - or get_rerank_config().providers.get("http", {}).get("service_url")
196   - or get_rerank_config().providers.get("http", {}).get("base_url")
197   - )
  68 + provider_cfg = get_app_config().services.rerank.get_provider_config()
  69 + base = provider_cfg.get("service_url") or provider_cfg.get("base_url")
198 70 if not base:
199   - raise ValueError("Rerank HTTP base_url is not configured")
  71 + raise ValueError("Rerank service URL is not configured")
200 72 return str(base).rstrip("/")
201 73  
202 74  
203 75 def get_rerank_service_url() -> str:
204   - """Backward-compatible alias."""
205 76 return get_rerank_base_url()
... ...
config/tenant_config_loader.py
  Show/Hide comments   View file @0342d89
... ... @@ -2,12 +2,13 @@
2 2 租户配置加载器。
3 3  
4 4 从统一配置文件（config.yaml）加载租户配置，包括主语言和索引语言（index_languages）。
5   -支持旧配置 translate_to_en / translate_to_zh 的兼容解析。
6 5 """
7 6  
8 7 import logging
9 8 from typing import Dict, Any, Optional, List
10 9  
  10 +from config.loader import get_app_config
  11 +
11 12 logger = logging.getLogger(__name__)
12 13  
13 14 # 支持的索引语言：code -> display name（供商家勾选主市场语言等场景使用）
... ... @@ -83,25 +84,13 @@ def resolve_index_languages(
83 84 ) -> List[str]:
84 85 """
85 86 从租户配置解析 index_languages。
86   - 若存在 index_languages 则用之；否则按旧配置 translate_to_en / translate_to_zh 推导。
  87 + 若配置缺失或非法，则回退到默认配置。
87 88 """
88   - if "index_languages" in tenant_config:
89   - normalized = normalize_index_languages(
90   - tenant_config["index_languages"],
91   - tenant_config.get("primary_language") or "en",
92   - )
93   - return normalized if normalized else list(default_index_languages)
94   - primary = (tenant_config.get("primary_language") or "en").strip().lower()
95   - to_en = bool(tenant_config.get("translate_to_en"))
96   - to_zh = bool(tenant_config.get("translate_to_zh"))
97   - langs: List[str] = []
98   - if primary and primary in SOURCE_LANG_CODE_MAP:
99   - langs.append(primary)
100   - for code in ("en", "zh"):
101   - if code not in langs and ((code == "en" and to_en) or (code == "zh" and to_zh)):
102   - if code in SOURCE_LANG_CODE_MAP:
103   - langs.append(code)
104   - return langs if langs else list(default_index_languages)
  89 + normalized = normalize_index_languages(
  90 + tenant_config.get("index_languages"),
  91 + tenant_config.get("primary_language") or "en",
  92 + )
  93 + return normalized if normalized else list(default_index_languages)
105 94  
106 95  
107 96 class TenantConfigLoader:
... ... @@ -122,15 +111,8 @@ class TenantConfigLoader:
122 111 return self._config
123 112  
124 113 try:
125   - from config import ConfigLoader
126   -
127   - config_loader = ConfigLoader()
128   - search_config = config_loader.load_config()
129   - tenant_cfg = search_config.tenant_config
130   - if not isinstance(tenant_cfg, dict):
131   - raise RuntimeError("tenant_config must be an object")
132   -
133   - default_cfg = tenant_cfg.get("default")
  114 + tenant_cfg = get_app_config().tenants
  115 + default_cfg = tenant_cfg.default
134 116 if not isinstance(default_cfg, dict):
135 117 raise RuntimeError("tenant_config.default must be configured in config.yaml")
136 118 default_primary = (default_cfg.get("primary_language") or "en").strip().lower()
... ... @@ -143,7 +125,7 @@ class TenantConfigLoader:
143 125 "tenant_config.default.index_languages must include at least one supported language"
144 126 )
145 127  
146   - tenants_cfg = tenant_cfg.get("tenants", {})
  128 + tenants_cfg = tenant_cfg.tenants
147 129 if not isinstance(tenants_cfg, dict):
148 130 raise RuntimeError("tenant_config.tenants must be an object")
149 131  
... ...
config/utils.py
  Show/Hide comments   View file @0342d89
1 1 """Configuration helper functions for dynamic multi-language search fields."""
2 2  
3 3 from typing import Dict, List
4   -from .config_loader import SearchConfig
  4 +
  5 +from config.schema import SearchConfig
5 6  
6 7  
7 8 def _format_field_with_boost(field_name: str, boost: float) -> str:
... ...
docs/config-system-review-and-redesign.md 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,738 @@
  1 +# Configuration System Review And Redesign
  2 +
  3 +## 1. Goal
  4 +
  5 +This document reviews the current configuration system and proposes a practical redesign for long-term maintainability.
  6 +
  7 +The target is a configuration system that is:
  8 +
  9 +- unified in loading and ownership
  10 +- clear in boundaries and precedence
  11 +- visible in effective behavior
  12 +- easy to evolve across development, deployment, and operations
  13 +
  14 +This review is based on the current implementation, not only on the intended architecture in docs.
  15 +
  16 +## 2. Project Context
  17 +
  18 +The repo already defines the right architectural direction:
  19 +
  20 +- `config/config.yaml` should be the main configuration source for search behavior and service wiring
  21 +- `.env` should mainly carry deployment-specific values and secrets
  22 +- provider/backend expansion should stay centralized instead of spreading through business code
  23 +
  24 +That direction is described in:
  25 +
  26 +- [`README.md`](/data/saas-search/README.md)
  27 +- [`docs/DEVELOPER_GUIDE.md`](/data/saas-search/docs/DEVELOPER_GUIDE.md)
  28 +- [`docs/QUICKSTART.md`](/data/saas-search/docs/QUICKSTART.md)
  29 +- [`translation/README.md`](/data/saas-search/translation/README.md)
  30 +
  31 +The problem is not the architectural intent. The problem is that the current implementation only partially follows it.
  32 +
  33 +## 3. Current-State Review
  34 +
  35 +### 3.1 What exists today
  36 +
  37 +The current system effectively has several configuration channels:
  38 +
  39 +- `config/config.yaml`
  40 + - search behavior
  41 + - rerank behavior
  42 + - services registry
  43 + - tenant config
  44 +- `config/config_loader.py`
  45 + - parses search behavior and tenant config into `SearchConfig`
  46 + - also injects some defaults from code
  47 +- `config/services_config.py`
  48 + - reparses `config/config.yaml` again, independently
  49 + - resolves translation, embedding, rerank service config
  50 + - also applies env overrides
  51 +- `config/env_config.py`
  52 + - loads `.env`
  53 + - defines ES, Redis, DB, host/port, service URLs, namespace, model path defaults
  54 +- service-local config modules
  55 + - [`embeddings/config.py`](/data/saas-search/embeddings/config.py)
  56 + - [`reranker/config.py`](/data/saas-search/reranker/config.py)
  57 +- startup scripts
  58 + - derive defaults from shell env, Python config, and YAML in different combinations
  59 +- inline fallbacks in business logic
  60 + - query parsing
  61 + - indexing
  62 + - service startup
  63 +
  64 +### 3.2 Main findings
  65 +
  66 +#### Finding A: there is no single loader for the full effective configuration
  67 +
  68 +`ConfigLoader` and `services_config` both parse `config/config.yaml`, but they do so separately and with different responsibilities.
  69 +
  70 +- [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L148)
  71 +- [`config/services_config.py`](/data/saas-search/config/services_config.py#L33)
  72 +
  73 +Impact:
  74 +
  75 +- the same file is loaded twice through different code paths
  76 +- search config and services config can drift in interpretation
  77 +- alternative config paths are hard to support cleanly
  78 +- tests and tools cannot ask one place for the full effective config tree
  79 +
  80 +#### Finding B: precedence is not explicit, stable, or globally enforced
  81 +
  82 +Current precedence differs by subsystem:
  83 +
  84 +- search behavior mostly comes from YAML plus code defaults
  85 +- embedding and rerank allow env overrides for provider/backend/url
  86 +- translation intentionally blocks some env overrides
  87 +- startup scripts still choose host/port and mode via env
  88 +- some values are reconstructed from other env vars
  89 +
  90 +Examples:
  91 +
  92 +- env override for embedding provider/url/backend:
  93 + - [`config/services_config.py`](/data/saas-search/config/services_config.py#L52)
  94 + - [`config/services_config.py`](/data/saas-search/config/services_config.py#L68)
  95 + - [`config/services_config.py`](/data/saas-search/config/services_config.py#L139)
  96 +- host/port and service URL reconstruction:
  97 + - [`config/env_config.py`](/data/saas-search/config/env_config.py#L55)
  98 + - [`config/env_config.py`](/data/saas-search/config/env_config.py#L75)
  99 +- translator host/port still driven by startup env:
  100 + - [`scripts/start_translator.sh`](/data/saas-search/scripts/start_translator.sh#L28)
  101 +
  102 +Impact:
  103 +
  104 +- operators cannot reliably predict the effective configuration by reading one file
  105 +- the same setting category behaves differently across services
  106 +- incidents become harder to debug because source-of-truth depends on the code path
  107 +
  108 +#### Finding C: defaults are duplicated across YAML and code
  109 +
  110 +There are several layers of default values:
  111 +
  112 +- dataclass defaults in `QueryConfig`
  113 +- fallback defaults in `ConfigLoader._parse_config`
  114 +- defaults in `config.yaml`
  115 +- defaults in `env_config.py`
  116 +- defaults in `embeddings/config.py`
  117 +- defaults in `reranker/config.py`
  118 +- defaults in startup scripts
  119 +
  120 +Examples:
  121 +
  122 +- query defaults duplicated in dataclass and parser:
  123 + - [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L24)
  124 + - [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L240)
  125 +- embedding defaults duplicated in YAML, `services_config`, `embeddings/config.py`, and startup script:
  126 + - [`config/config.yaml`](/data/saas-search/config/config.yaml#L196)
  127 + - [`embeddings/config.py`](/data/saas-search/embeddings/config.py#L14)
  128 + - [`scripts/start_embedding_service.sh`](/data/saas-search/scripts/start_embedding_service.sh#L29)
  129 +- reranker defaults duplicated in YAML and `reranker/config.py`:
  130 + - [`config/config.yaml`](/data/saas-search/config/config.yaml#L214)
  131 + - [`reranker/config.py`](/data/saas-search/reranker/config.py#L6)
  132 +
  133 +Impact:
  134 +
  135 +- changing a default is risky because there may be multiple hidden copies
  136 +- code review cannot easily tell whether a value is authoritative or dead legacy
  137 +- “same config” may behave differently across processes
  138 +
  139 +#### Finding D: config is still embedded in runtime logic
  140 +
  141 +Some important behavior remains encoded as inline fallback logic rather than declared config.
  142 +
  143 +Examples:
  144 +
  145 +- query-time translation target languages fallback to `["en", "zh"]`:
  146 + - [`query/query_parser.py`](/data/saas-search/query/query_parser.py#L339)
  147 +- indexer text handling and LLM enrichment also fallback to `["en", "zh"]`:
  148 + - [`indexer/document_transformer.py`](/data/saas-search/indexer/document_transformer.py#L216)
  149 + - [`indexer/document_transformer.py`](/data/saas-search/indexer/document_transformer.py#L310)
  150 + - [`indexer/document_transformer.py`](/data/saas-search/indexer/document_transformer.py#L649)
  151 +
  152 +Impact:
  153 +
  154 +- configuration is not fully visible in config files
  155 +- behavior can silently change when tenant config is missing or malformed
  156 +- “default behavior” is spread across business modules
  157 +
  158 +#### Finding E: some configuration assets are not managed as first-class config
  159 +
  160 +Query rewrite is configured through an external file, but the file path is hardcoded and currently inconsistent with the repository content.
  161 +
  162 +- loader expects:
  163 + - [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L162)
  164 +- repo currently contains:
  165 + - [`config/query_rewrite.dict`](/data/saas-search/config/query_rewrite.dict)
  166 +
  167 +There is also an admin API that mutates rewrite rules in memory only:
  168 +
  169 +- [`api/routes/admin.py`](/data/saas-search/api/routes/admin.py#L68)
  170 +- [`query/query_parser.py`](/data/saas-search/query/query_parser.py#L622)
  171 +
  172 +Impact:
  173 +
  174 +- rewrite rules are neither cleanly file-backed nor fully runtime-managed
  175 +- restart behavior is unclear
  176 +- configuration visibility and persistence are weak
  177 +
  178 +#### Finding F: visibility is limited
  179 +
  180 +The system exposes only a small sanitized subset at `/admin/config`.
  181 +
  182 +- [`api/routes/admin.py`](/data/saas-search/api/routes/admin.py#L42)
  183 +
  184 +At the same time, the true effective config includes:
  185 +
  186 +- tenant overlays
  187 +- env overrides
  188 +- service backend selections
  189 +- script-selected modes
  190 +- hidden defaults in code
  191 +
  192 +Impact:
  193 +
  194 +- there is no authoritative “effective config” view
  195 +- debugging configuration mismatches requires source reading
  196 +- operators cannot easily verify what each process actually started with
  197 +
  198 +#### Finding G: the indexer does not really consume the unified config as a first-class dependency
  199 +
  200 +Indexer startup explicitly says config is loaded only for parity/logging and routes do not depend on it.
  201 +
  202 +- [`api/indexer_app.py`](/data/saas-search/api/indexer_app.py#L76)
  203 +
  204 +Impact:
  205 +
  206 +- configuration is not truly system-wide
  207 +- search-side and indexer-side behavior can drift
  208 +- the current “unified config” is only partially unified
  209 +
  210 +#### Finding H: docs still carry legacy and mixed mental models
  211 +
  212 +Most high-level docs describe the desired centralized model, but some implementation/docs still expose legacy concepts such as `translate_to_en` and `translate_to_zh`.
  213 +
  214 +- desired model:
  215 + - [`README.md`](/data/saas-search/README.md#L78)
  216 + - [`docs/DEVELOPER_GUIDE.md`](/data/saas-search/docs/DEVELOPER_GUIDE.md#L207)
  217 + - [`translation/README.md`](/data/saas-search/translation/README.md#L161)
  218 +- legacy tenant translation flags still documented:
  219 + - [`indexer/README.md`](/data/saas-search/indexer/README.md#L39)
  220 +
  221 +Impact:
  222 +
  223 +- new developers may follow old mental models
  224 +- cleanup work keeps getting deferred because old and new systems appear both “supported”
  225 +
  226 +## 4. Design Principles For The Redesign
  227 +
  228 +The redesign should follow these rules.
  229 +
  230 +### 4.1 One logical configuration system
  231 +
  232 +It is acceptable to have multiple files, but not multiple loaders with overlapping ownership.
  233 +
  234 +There must be one loader pipeline that produces one typed `AppConfig`.
  235 +
  236 +### 4.2 Configuration files declare, parser code interprets, env provides runtime injection
  237 +
  238 +Responsibilities should be:
  239 +
  240 +- configuration files
  241 + - declare non-secret desired behavior and non-secret deployable settings
  242 +- parsing logic
  243 + - load, merge, validate, normalize, and expose typed config
  244 + - never invent hidden business behavior
  245 +- environment variables
  246 + - carry secrets and a small set of runtime/process values
  247 + - do not redefine business behavior casually
  248 +
  249 +### 4.3 One precedence rule for the whole system
  250 +
  251 +Every config category should follow the same merge model unless explicitly exempted.
  252 +
  253 +### 4.4 No silent implicit fallback for business behavior
  254 +
  255 +Fail fast at startup when required config is missing or invalid.
  256 +
  257 +Do not silently fall back to legacy behavior such as hardcoded language lists.
  258 +
  259 +### 4.5 Effective configuration must be observable
  260 +
  261 +Every service should be able to show:
  262 +
  263 +- config version or hash
  264 +- source files loaded
  265 +- environment name
  266 +- sanitized effective configuration
  267 +
  268 +## 5. Recommended Target Design
  269 +
  270 +## 5.1 Boundary model
  271 +
  272 +Use three clear layers.
  273 +
  274 +### Layer 1: repository-managed static config
  275 +
  276 +Purpose:
  277 +
  278 +- search behavior
  279 +- tenant behavior
  280 +- provider/backend registry
  281 +- non-secret service topology defaults
  282 +- feature switches
  283 +
  284 +Examples:
  285 +
  286 +- field boosts
  287 +- query strategy
  288 +- rerank fusion parameters
  289 +- tenant language plans
  290 +- translation capability registry
  291 +- embedding backend selection default
  292 +
  293 +### Layer 2: environment-specific overlays
  294 +
  295 +Purpose:
  296 +
  297 +- per-environment non-secret differences
  298 +- service endpoints by environment
  299 +- resource sizing defaults by environment
  300 +- dev/test/prod operational differences
  301 +
  302 +Examples:
  303 +
  304 +- local embedding URL vs production URL
  305 +- dev rerank backend vs prod rerank backend
  306 +- lower concurrency in local development
  307 +
  308 +### Layer 3: environment variables
  309 +
  310 +Purpose:
  311 +
  312 +- secrets
  313 +- bind host/port
  314 +- external infrastructure credentials
  315 +- container-orchestrator last-mile injection
  316 +
  317 +Examples:
  318 +
  319 +- `ES_HOST`, `ES_USERNAME`, `ES_PASSWORD`
  320 +- `DB_HOST`, `DB_USERNAME`, `DB_PASSWORD`
  321 +- `REDIS_HOST`, `REDIS_PASSWORD`
  322 +- `DASHSCOPE_API_KEY`, `DEEPL_AUTH_KEY`
  323 +- `API_HOST`, `API_PORT`, `INDEXER_PORT`, `TRANSLATION_PORT`
  324 +
  325 +Rule:
  326 +
  327 +- environment variables should not be the normal path for choosing business behavior such as translation model, embedding backend, or tenant language policy
  328 +- if an env override is allowed for a non-secret field, it must be explicitly listed and documented as an operational override, not a hidden convention
  329 +
  330 +## 5.2 Unified precedence
  331 +
  332 +Recommended precedence:
  333 +
  334 +1. schema defaults in code
  335 +2. `config/base.yaml`
  336 +3. `config/environments/<env>.yaml`
  337 +4. tenant overlay from `config/tenants/`
  338 +5. environment variables for the explicitly allowed runtime keys
  339 +6. CLI flags for the current process only
  340 +
  341 +Important rule:
  342 +
  343 +- only one module may implement this merge logic
  344 +- no business module may call `os.getenv()` directly for configuration
  345 +
  346 +## 5.3 Recommended directory structure
  347 +
  348 +```text
  349 +config/
  350 + schema.py
  351 + loader.py
  352 + sources.py
  353 + base.yaml
  354 + environments/
  355 + dev.yaml
  356 + test.yaml
  357 + prod.yaml
  358 + tenants/
  359 + _default.yaml
  360 + 1.yaml
  361 + 162.yaml
  362 + 170.yaml
  363 + dictionaries/
  364 + query_rewrite.dict
  365 + README.md
  366 +.env.example
  367 +```
  368 +
  369 +Notes:
  370 +
  371 +- `base.yaml` contains shared defaults and feature behavior
  372 +- `environments/*.yaml` contains environment-specific non-secret overrides
  373 +- `tenants/*.yaml` contains tenant-specific overrides only
  374 +- `dictionaries/` stores first-class config assets such as rewrite dictionaries
  375 +- `schema.py` defines the typed config model
  376 +- `loader.py` is the only entry point that loads and merges config
  377 +
  378 +If the team prefers fewer files, `tenants.yaml` is also acceptable. The key requirement is not “one file”, but “one loading model with clear ownership”.
  379 +
  380 +## 5.4 Typed configuration model
  381 +
  382 +Introduce one root object, for example:
  383 +
  384 +```python
  385 +class AppConfig(BaseModel):
  386 + runtime: RuntimeConfig
  387 + infrastructure: InfrastructureConfig
  388 + search: SearchConfig
  389 + services: ServicesConfig
  390 + tenants: TenantCatalogConfig
  391 + assets: ConfigAssets
  392 +```
  393 +
  394 +Suggested subtrees:
  395 +
  396 +- `runtime`
  397 + - environment name
  398 + - config revision/hash
  399 + - bind addresses/ports
  400 +- `infrastructure`
  401 + - ES
  402 + - DB
  403 + - Redis
  404 + - index namespace
  405 +- `search`
  406 + - field boosts
  407 + - query config
  408 + - function score
  409 + - rerank behavior
  410 + - spu config
  411 +- `services`
  412 + - translation
  413 + - embedding
  414 + - rerank
  415 +- `tenants`
  416 + - default tenant config
  417 + - tenant overrides
  418 +- `assets`
  419 + - rewrite dictionary path
  420 +
  421 +Benefits:
  422 +
  423 +- one validated object shared by backend, indexer, translator, embedding, reranker
  424 +- one place for defaults
  425 +- one place for schema evolution
  426 +
  427 +## 5.5 Loading flow
  428 +
  429 +Recommended loading flow:
  430 +
  431 +1. determine `APP_ENV` or `RUNTIME_ENV`
  432 +2. load schema defaults
  433 +3. load `config/base.yaml`
  434 +4. load `config/environments/<env>.yaml` if present
  435 +5. load tenant files
  436 +6. inject first-class assets such as rewrite dictionary
  437 +7. apply allowed env overrides
  438 +8. validate the final `AppConfig`
  439 +9. freeze and cache the config object
  440 +10. expose a sanitized effective-config view
  441 +
  442 +Important:
  443 +
  444 +- every process should call the same loader
  445 +- services should receive a resolved `AppConfig`, not re-open YAML independently
  446 +
  447 +## 5.6 Clear responsibility split
  448 +
  449 +### Configuration files are responsible for
  450 +
  451 +- what the system should do
  452 +- what providers/backends are available
  453 +- which features are enabled
  454 +- tenant language/index policies
  455 +- non-secret service topology
  456 +
  457 +### Parser/loader code is responsible for
  458 +
  459 +- locating sources
  460 +- merge precedence
  461 +- type validation
  462 +- normalization
  463 +- deprecation warnings
  464 +- producing the final immutable config object
  465 +
  466 +### Environment variables are responsible for
  467 +
  468 +- secrets
  469 +- bind addresses/ports
  470 +- infrastructure endpoints when the deployment platform injects them
  471 +- a very small set of documented operational overrides
  472 +
  473 +### Business code is not responsible for
  474 +
  475 +- inventing defaults for missing config
  476 +- loading YAML directly
  477 +- calling `os.getenv()` for normal application behavior
  478 +
  479 +## 5.7 How to handle service config
  480 +
  481 +Unify all service-facing config under one structure:
  482 +
  483 +```yaml
  484 +services:
  485 + translation:
  486 + endpoint: "http://translator:6006"
  487 + timeout_sec: 10
  488 + default_model: "llm"
  489 + default_scene: "general"
  490 + capabilities: ...
  491 + embedding:
  492 + endpoint:
  493 + text: "http://embedding:6005"
  494 + image: "http://embedding-image:6008"
  495 + backend: "tei"
  496 + backends: ...
  497 + rerank:
  498 + endpoint: "http://reranker:6007/rerank"
  499 + backend: "qwen3_vllm"
  500 + backends: ...
  501 +```
  502 +
  503 +Rules:
  504 +
  505 +- `endpoint` is how callers reach the service
  506 +- `backend` is how the service itself is implemented
  507 +- only the service process cares about `backend`
  508 +- only callers care about `endpoint`
  509 +- both still belong to the same config tree, because they are part of one system
  510 +
  511 +## 5.8 How to handle tenant config
  512 +
  513 +Tenant config should become explicit policy, not translation-era leftovers.
  514 +
  515 +Recommended tenant fields:
  516 +
  517 +- `primary_language`
  518 +- `index_languages`
  519 +- `search_languages`
  520 +- `translation_policy`
  521 +- `facet_policy`
  522 +- optional tenant-specific ranking overrides
  523 +
  524 +Avoid keeping `translate_to_en` and `translate_to_zh` as active concepts in the long-term model.
  525 +
  526 +If compatibility is needed, support them only in the loader as deprecated aliases and emit warnings.
  527 +
  528 +## 5.9 How to handle rewrite rules and similar assets
  529 +
  530 +Treat them as declared config assets.
  531 +
  532 +Recommended rules:
  533 +
  534 +- file path declared in config
  535 +- one canonical location under `config/dictionaries/`
  536 +- loader validates presence and format
  537 +- admin runtime updates either:
  538 + - are removed, or
  539 + - write back through a controlled persistence path
  540 +
  541 +Do not keep a hybrid model where startup loads one file and admin mutates only in memory.
  542 +
  543 +## 5.10 Observability improvements
  544 +
  545 +Add the following:
  546 +
  547 +- `config dump` CLI that prints sanitized effective config
  548 +- startup log with config hash, environment, and config file list
  549 +- `/admin/config/effective` endpoint returning sanitized effective config
  550 +- `/admin/config/meta` endpoint returning:
  551 + - environment
  552 + - config hash
  553 + - loaded source files
  554 + - deprecated keys in use
  555 +
  556 +This is important for operations and for multi-service debugging.
  557 +
  558 +## 6. Practical Refactor Plan
  559 +
  560 +The refactor should be incremental.
  561 +
  562 +### Phase 1: establish the new config core without changing behavior
  563 +
  564 +- create `config/schema.py`
  565 +- create `config/loader.py`
  566 +- move all current defaults into schema models
  567 +- make loader read current `config/config.yaml`
  568 +- make loader read `.env` only for approved keys
  569 +- expose one `get_app_config()`
  570 +
  571 +Result:
  572 +
  573 +- same behavior, but one typed root config becomes available
  574 +
  575 +### Phase 2: remove duplicate readers
  576 +
  577 +- make `services_config.py` a thin adapter over `get_app_config()`
  578 +- make `tenant_config_loader.py` read from `get_app_config()`
  579 +- stop reparsing YAML in `services_config.py`
  580 +- stop service modules from depending on legacy local config modules for behavior
  581 +
  582 +Result:
  583 +
  584 +- one parsing path
  585 +- fewer divergence risks
  586 +
  587 +### Phase 3: move hidden defaults out of business logic
  588 +
  589 +- remove hardcoded fallback language lists from query/indexer modules
  590 +- require tenant defaults to come from config schema only
  591 +- remove duplicate behavior defaults from service code
  592 +
  593 +Result:
  594 +
  595 +- behavior becomes visible and reviewable
  596 +
  597 +### Phase 4: clean service startup configuration
  598 +
  599 +- make startup scripts ask the unified loader for resolved values
  600 +- keep only bind host/port and secret injection in shell env
  601 +- retire or reduce `embeddings/config.py` and `reranker/config.py`
  602 +
  603 +Result:
  604 +
  605 +- startup behavior matches runtime config model
  606 +
  607 +### Phase 5: split config files by responsibility
  608 +
  609 +- keep a single root loader
  610 +- split current giant `config.yaml` into:
  611 + - `base.yaml`
  612 + - `environments/<env>.yaml`
  613 + - `tenants/*.yaml`
  614 + - `dictionaries/query_rewrite.dict`
  615 +
  616 +Result:
  617 +
  618 +- config remains unified logically, but is easier to read and maintain physically
  619 +
  620 +### Phase 6: deprecate legacy compatibility
  621 +
  622 +- deprecate `translate_to_en` and `translate_to_zh`
  623 +- deprecate env-based backend/provider selection except for explicitly approved keys
  624 +- remove old code paths after one or two release cycles
  625 +
  626 +Result:
  627 +
  628 +- the system becomes simpler instead of carrying two generations forever
  629 +
  630 +## 7. Concrete Rules To Adopt
  631 +
  632 +These rules should be documented and enforced in code review.
  633 +
  634 +### Rule 1
  635 +
  636 +Only `config/loader.py` may load config files or `.env`.
  637 +
  638 +### Rule 2
  639 +
  640 +Only `config/loader.py` may read `os.getenv()` for application config.
  641 +
  642 +### Rule 3
  643 +
  644 +Business modules receive typed config objects and do not read files or env directly.
  645 +
  646 +### Rule 4
  647 +
  648 +Each config key has one owner.
  649 +
  650 +Examples:
  651 +
  652 +- `search.query.knn_boost` belongs to search behavior config
  653 +- `services.embedding.backend` belongs to service implementation config
  654 +- `infrastructure.redis.password` belongs to env/secrets
  655 +
  656 +### Rule 5
  657 +
  658 +Every fallback must be either:
  659 +
  660 +- declared in schema defaults, or
  661 +- rejected at startup
  662 +
  663 +No hidden fallback in runtime logic.
  664 +
  665 +### Rule 6
  666 +
  667 +Every configuration asset must be visible in one of these places only:
  668 +
  669 +- config file
  670 +- env var
  671 +- generated runtime metadata
  672 +
  673 +Not inside parser code as an implicit constant.
  674 +
  675 +## 8. Recommended Naming Conventions
  676 +
  677 +Suggested conventions:
  678 +
  679 +- config keys use noun-based hierarchical names
  680 +- avoid mixing transport and implementation concepts in one field
  681 +- use `endpoint` for caller-facing addresses
  682 +- use `backend` for service-internal implementation choice
  683 +- use `enabled` only for true feature toggles
  684 +- use `default_*` only when a real selection happens at runtime
  685 +
  686 +Examples:
  687 +
  688 +- good: `services.rerank.endpoint`
  689 +- good: `services.rerank.backend`
  690 +- good: `tenants.default.index_languages`
  691 +- avoid: `service_url`, `base_url`, `provider`, `backend`, and script env all meaning slightly different things without a common model
  692 +
  693 +## 9. Highest-Priority Cleanup Items
  694 +
  695 +If the team wants the shortest path to improvement, start here:
  696 +
  697 +1. build one root `AppConfig`
  698 +2. make `services_config.py` stop reparsing YAML
  699 +3. declare rewrite dictionary path explicitly and fix the current mismatch
  700 +4. remove hardcoded `["en", "zh"]` fallbacks from query/indexer logic
  701 +5. replace `/admin/config` with an effective-config endpoint
  702 +6. retire `embeddings/config.py` and `reranker/config.py` as behavior sources
  703 +7. deprecate legacy tenant translation flags
  704 +
  705 +## 10. Expected Outcome
  706 +
  707 +After the redesign:
  708 +
  709 +- developers can answer “where does this setting come from?” in one step
  710 +- operators can see effective config without reading source code
  711 +- backend, indexer, translator, embedding, and reranker all share one model
  712 +- tenant behavior is explicit instead of partially implicit
  713 +- migration becomes safer because defaults and precedence are centralized
  714 +- adding a new provider/backend becomes configuration extension, not configuration archaeology
  715 +
  716 +## 11. Summary
  717 +
  718 +The current system has the right intent but not yet the right implementation shape.
  719 +
  720 +Today the main problems are:
  721 +
  722 +- duplicate config loaders
  723 +- inconsistent precedence
  724 +- duplicated defaults
  725 +- config hidden in runtime logic
  726 +- weak effective-config visibility
  727 +- leftover legacy concepts
  728 +
  729 +The recommended direction is:
  730 +
  731 +- one root typed config
  732 +- one loader pipeline
  733 +- explicit layered sources
  734 +- narrow env responsibility
  735 +- no hidden business fallbacks
  736 +- observable effective config
  737 +
  738 +That design is practical to implement incrementally in this repository and aligns well with the project's multi-tenant, multi-service, provider/backend-based architecture.
... ...
docs/搜索API对接指南-00-总览与快速开始.md 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,110 @@
  1 +# 搜索API对接指南-00-总览与快速开始
  2 +
  3 +本文档旨在为搜索服务的使用方提供完整的API对接指南，包括接口说明、请求参数、响应格式和使用示例。
  4 +拆分目录：
  5 +- `-01-搜索接口（POST /search/ 与响应）`
  6 +- `-02-搜索建议与即时搜索`
  7 +- `-03-获取文档（GET /search/{doc_id}）`
  8 +- `-05-索引接口（Indexer）`
  9 +- `-06-管理接口（Admin）`
  10 +- `-07-微服务接口（Embedding/Reranker/Translation）`
  11 +- `-08-数据模型与字段速查`
  12 +- `-10-接口级压测脚本`
  13 +
  14 +## 快速开始
  15 +
  16 +### 1.1 基础信息
  17 +
  18 +- **Base URL**: `http://43.166.252.75:6002`
  19 +- **协议**: HTTP/HTTPS
  20 +- **数据格式**: JSON
  21 +- **字符编码**: UTF-8
  22 +- **请求方法**: POST（搜索接口）
  23 +
  24 +**重要提示**: `tenant_id` 通过 HTTP Header `X-Tenant-ID` 传递，不在请求体中。
  25 +
  26 +**环境与凭证**：MySQL、Redis、Elasticsearch 等外部服务的 AI 生产地址与凭证见 [QUICKSTART.md §1.6](./QUICKSTART.md#16-外部服务与-env含生产凭证)。
  27 +
  28 +### 1.2 最简单的搜索请求
  29 +
  30 +```bash
  31 +curl -X POST "http://43.166.252.75:6002/search/" \
  32 + -H "Content-Type: application/json" \
  33 + -H "X-Tenant-ID: 162" \
  34 + -d '{"query": "芭比娃娃"}'
  35 +```
  36 +
  37 +### 1.3 带过滤与分页的搜索
  38 +
  39 +```bash
  40 +curl -X POST "http://43.166.252.75:6002/search/" \
  41 + -H "Content-Type: application/json" \
  42 + -H "X-Tenant-ID: 162" \
  43 + -d '{
  44 + "query": "芭比娃娃",
  45 + "size": 5,
  46 + "from": 10,
  47 + "range_filters": {
  48 + "min_price": {
  49 + "gte": 50,
  50 + "lte": 200
  51 + },
  52 + "create_time": {
  53 + "gte": "2020-01-01T00:00:00Z"
  54 + }
  55 + },
  56 + "sort_by": "price",
  57 + "sort_order": "asc"
  58 + }'
  59 +```
  60 +
  61 +### 1.4 开启分面的搜索
  62 +
  63 +```bash
  64 +curl -X POST "http://43.166.252.75:6002/search/" \
  65 + -H "Content-Type: application/json" \
  66 + -H "X-Tenant-ID: 162" \
  67 + -d '{
  68 + "query": "芭比娃娃",
  69 + "facets": [
  70 + {"field": "category1_name", "size": 10, "type": "terms"},
  71 + {"field": "specifications.color", "size": 10, "type": "terms"},
  72 + {"field": "specifications.size", "size": 10, "type": "terms"}
  73 + ],
  74 + "min_score": 0.2
  75 + }'
  76 +```
  77 +
  78 +---
  79 +
  80 +## 接口概览
  81 +
  82 +| 接口 | HTTP Method | Endpoint | 说明 |
  83 +|------|------|------|------|
  84 +| 搜索 | POST | `/search/` | 执行搜索查询 |
  85 +| 搜索建议 | GET | `/search/suggestions` | 搜索建议（自动补全/热词，多语言） |
  86 +| 即时搜索 | GET | `/search/instant` | 即时搜索预留接口（当前返回 `501 Not Implemented`） |
  87 +| 获取文档 | GET | `/search/{doc_id}` | 获取单个文档 |
  88 +| 全量索引 | POST | `/indexer/reindex` | 全量索引接口（导入数据，不删除索引，仅推荐自测使用） |
  89 +| 增量索引 | POST | `/indexer/index` | 增量索引接口（指定SPU ID列表进行索引，支持自动检测删除和显式删除，仅推荐自测使用） |
  90 +| 查询文档 | POST | `/indexer/documents` | 查询SPU文档数据（不写入ES） |
  91 +| 构建ES文档（正式对接） | POST | `/indexer/build-docs` | 基于上游提供的 MySQL 行数据构建 ES doc，不写入 ES，供 Java 等调用后自行写入 |
  92 +| 构建ES文档（测试用） | POST | `/indexer/build-docs-from-db` | 仅在测试/调试时使用，根据 `tenant_id + spu_ids` 内部查库并构建 ES doc |
  93 +| 内容理解字段生成 | POST | `/indexer/enrich-content` | 根据商品标题批量生成 qanchors、semantic_attributes、tags，供微服务组合方式使用 |
  94 +| 索引健康检查 | GET | `/indexer/health` | 检查索引服务状态 |
  95 +| 健康检查 | GET | `/admin/health` | 服务健康检查 |
  96 +| 获取配置 | GET | `/admin/config` | 获取租户配置 |
  97 +| 索引统计 | GET | `/admin/stats` | 获取租户索引统计信息（需 tenant_id） |
  98 +
  99 +**微服务（独立端口或 Indexer 内，外部可直连）**：
  100 +
  101 +| 服务 | 端口 | 接口 | 说明 |
  102 +|------|------|------|------|
  103 +| 向量服务（文本） | 6005 | `POST /embed/text` | 文本向量化 |
  104 +| 向量服务（图片） | 6008 | `POST /embed/image` | 图片向量化 |
  105 +| 翻译服务 | 6006 | `POST /translate` | 文本翻译（支持 qwen-mt / llm / deepl / 本地模型） |
  106 +| 重排服务 | 6007 | `POST /rerank` | 检索结果重排 |
  107 +| 内容理解（Indexer 内） | 6004 | `POST /indexer/enrich-content` | 根据商品标题生成 qanchors、tags 等，供 indexer 微服务组合方式使用 |
  108 +
  109 +---
  110 +
... ...
docs/搜索API对接指南-01-搜索接口.md 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,903 @@
  1 +# 搜索API对接指南-01-搜索接口（POST /search/ 与响应）
  2 +
  3 +本篇以 `POST /search/` 为主线，包含：
  4 +- 请求参数：`3.2`、过滤器：`3.3`、分面：`3.4`、SKU筛选维度：`3.5`
  5 +- 响应格式：第 `4` 章（4.1~4.5）
  6 +- 常见场景示例：第 `8` 章（示例整体并入本篇，避免散落）
  7 +
  8 +## 搜索接口
  9 +
  10 +### 3.1 接口信息
  11 +
  12 +- **端点**: `POST /search/`
  13 +- **描述**: 执行文本搜索查询，支持多语言、过滤器和分面搜索
  14 +- **租户标识**：`tenant_id` 通过 HTTP 请求头 **`X-Tenant-ID`** 传递（推荐）；也可通过 URL query 参数 **`tenant_id`** 传递。**不要放在请求体中。**
  15 +
  16 +**请求示例（推荐）**:
  17 +
  18 +```python
  19 +url = f"{base_url.rstrip('/')}/search/"
  20 +headers = {
  21 + "Content-Type": "application/json",
  22 + "X-Tenant-ID": "162", # 租户ID，必填
  23 +}
  24 +response = requests.post(url, headers=headers, json={"query": "芭比娃娃"})
  25 +```
  26 +
  27 +### 3.2 请求参数
  28 +
  29 +#### 完整请求体结构
  30 +
  31 +```json
  32 +{
  33 + "query": "string (required)",
  34 + "size": 10,
  35 + "from": 0,
  36 + "language": "zh",
  37 + "filters": {},
  38 + "range_filters": {},
  39 + "facets": [],
  40 + "sort_by": "string",
  41 + "sort_order": "desc",
  42 + "min_score": 0.0,
  43 + "sku_filter_dimension": ["string"],
  44 + "debug": false,
  45 + "enable_rerank": null,
  46 + "rerank_query_template": "{query}",
  47 + "rerank_doc_template": "{title}",
  48 + "user_id": "string",
  49 + "session_id": "string"
  50 +}
  51 +```
  52 +
  53 +#### 参数详细说明
  54 +
  55 +| 参数 | 类型 | 必填 | 默认值 | 说明 |
  56 +|------|------|------|--------|------|
  57 +| `query` | string | Y | - | 搜索查询字符串（统一文本检索策略） |
  58 +| `size` | integer | N | 10 | 返回结果数量（1-100） |
  59 +| `from` | integer | N | 0 | 分页偏移量（用于分页） |
  60 +| `language` | string | N | "zh" | 返回语言：`zh`（中文）或 `en`（英文）。后端会根据此参数选择对应的中英文字段返回 |
  61 +| `filters` | object | N | null | 精确匹配过滤器（见[过滤器详解](#33-过滤器详解)） |
  62 +| `range_filters` | object | N | null | 数值范围过滤器（见[过滤器详解](#33-过滤器详解)） |
  63 +| `facets` | array | N | null | 分面配置（见[分面配置](#34-分面配置)） |
  64 +| `sort_by` | string | N | null | 排序字段名。支持：`price`（价格）、`sales`（销量）、`create_time`（创建时间）、`update_time`（更新时间）。默认按相关性排序 |
  65 +| `sort_order` | string | N | "desc" | 排序方向：`asc`（升序）或 `desc`（降序）。注意：`price`+`asc`=价格从低到高，`price`+`desc`=价格从高到低（后端自动映射为min_price或max_price） |
  66 +| `min_score` | float | N | null | 最小相关性分数阈值 |
  67 +| `sku_filter_dimension` | array[string] | N | null | 子SKU筛选维度列表（见[SKU筛选维度](#35-sku筛选维度)） |
  68 +| `debug` | boolean | N | false | 是否返回调试信息 |
  69 +| `enable_rerank` | boolean/null | N | null | 是否开启重排（调用外部重排服务对 ES 结果进行二次排序）。不传/传 null 使用服务端 `rerank.enabled`（默认开启）。开启后会先对 ES TopN（`rerank_window`）重排，再按分页截取；若 `from+size>1000`，则不重排，直接按分页从 ES 返回 |
  70 +| `rerank_query_template` | string | N | null | 重排 query 模板（可选）。支持 `{query}` 占位符；不传则使用服务端配置 |
  71 +| `rerank_doc_template` | string | N | null | 重排 doc 模板（可选）。支持 `{title} {brief} {vendor} {description} {category_path}`；不传则使用服务端配置 |
  72 +| `user_id` | string | N | null | 用户ID（用于个性化，预留） |
  73 +| `session_id` | string | N | null | 会话ID（用于分析，预留） |
  74 +
  75 +### 3.3 过滤器详解
  76 +
  77 +#### 3.3.1 精确匹配过滤器 (filters)
  78 +
  79 +用于精确匹配或多值匹配。对于普通字段，数组表示 OR 逻辑（匹配任意一个值）；对于 specifications 字段，按维度分组处理。**任意字段名加 `_all` 后缀**表示多值 AND 逻辑（必须同时匹配所有值）。
  80 +
  81 +**格式**:
  82 +
  83 +```json
  84 +{
  85 + "filters": {
  86 + "category_name": "手机", // 可以为单值 或者 数组 匹配数组中任意一个（OR）
  87 + "category1_name": "服装", // 可以为单值 或者 数组 匹配数组中任意一个（OR）
  88 + "category2_name": "男装", // 可以为单值 或者 数组 匹配数组中任意一个（OR）
  89 + "category3_name": "衬衫", // 可以为单值 或者 数组 匹配数组中任意一个（OR）
  90 + "vendor.zh.keyword": ["奇乐", "品牌A"], // 可以为单值 或者 数组 匹配数组中任意一个（OR）
  91 + "tags": "手机", // 可以为单值 或者 数组 匹配数组中任意一个（OR）
  92 + "tags_all": ["手机", "促销", "新品"], // *_all：多值为 AND，必须同时包含所有标签
  93 + "category1_name_all": ["服装", "男装"], // 同上，适用于任意可过滤字段
  94 + // specifications 嵌套过滤（特殊格式）
  95 + "specifications": {
  96 + "name": "color",
  97 + "value": "white"
  98 + }
  99 + }
  100 +}
  101 +```
  102 +
  103 +**支持的值类型**:
  104 +- 字符串：精确匹配
  105 +- 整数：精确匹配
  106 +- 布尔值：精确匹配
  107 +- 数组：匹配任意值（OR 逻辑）；若字段名以 `_all` 结尾，则数组表示 AND 逻辑（必须同时匹配所有值）
  108 +- 对象：specifications 嵌套过滤（见下文）
  109 +
  110 +**`*_all` 语义（多值 AND）**:
  111 +- 任意过滤字段均可使用 `_all` 后缀，对应 ES 字段名为去掉 `_all` 后的名称。
  112 +- 例如：`tags_all: ["A", "B"]` 表示文档的 `tags` 必须**同时包含** A 和 B；`vendor.zh.keyword_all: ["奇乐", "品牌A"]` 表示同时匹配两个品牌（通常用于 keyword 多值场景）。
  113 +- `specifications_all`：传列表 `[{"name":"color","value":"white"},{"name":"size","value":"256GB"}]` 时，表示所有列出的规格条件都要满足（与 `specifications` 多维度时的 AND 一致；若同维度多值则要求文档同时满足多个值，一般用于嵌套多值场景）。
  114 +
  115 +**Specifications 嵌套过滤**:
  116 +
  117 +`specifications` 是嵌套字段，支持按规格名称和值进行过滤。
  118 +
  119 +**单个规格过滤**:
  120 +
  121 +```json
  122 +{
  123 + "filters": {
  124 + "specifications": {
  125 + "name": "color",
  126 + "value": "white"
  127 + }
  128 + }
  129 +}
  130 +```
  131 +
  132 +查询规格名称为"color"且值为"white"的商品。
  133 +
  134 +**多个规格过滤（按维度分组）**:
  135 +
  136 +```json
  137 +{
  138 + "filters": {
  139 + "specifications": [
  140 + {"name": "color", "value": "white"},
  141 + {"name": "size", "value": "256GB"}
  142 + ]
  143 + }
  144 +}
  145 +```
  146 +
  147 +查询同时满足所有规格的商品（color=white **且** size=256GB）。
  148 +
  149 +**相同维度的多个值（OR 逻辑）**:
  150 +
  151 +```json
  152 +{
  153 + "filters": {
  154 + "specifications": [
  155 + {"name": "size", "value": "3"},
  156 + {"name": "size", "value": "4"},
  157 + {"name": "size", "value": "5"},
  158 + {"name": "color", "value": "green"}
  159 + ]
  160 + }
  161 +}
  162 +```
  163 +
  164 +查询满足 (size=3 **或** size=4 **或** size=5) **且** color=green 的商品。
  165 +
  166 +**过滤逻辑说明**:
  167 +- **不同维度**（不同的 `name`）之间是 **AND** 关系（求交集）
  168 +- **相同维度**（相同的 `name`）的多个值之间是 **OR** 关系（求并集）
  169 +
  170 +**常用过滤字段**（详见[常用字段列表](./搜索API对接指南-08-数据模型与字段速查.md#93-常用字段列表)）:
  171 +- `category_name`: 类目名称
  172 +- `category1_name`, `category2_name`, `category3_name`: 多级类目
  173 +- `category_id`: 类目ID
  174 +- `vendor.zh.keyword`, `vendor.en.keyword`: 供应商/品牌（使用keyword子字段）
  175 +- `tags`: 标签（keyword类型，支持数组）
  176 +- `option1_name`, `option2_name`, `option3_name`: 选项名称
  177 +- `specifications`: 规格过滤（嵌套字段，格式见上文）
  178 +- 以上任意字段均可加 `_all` 后缀表示多值 AND，如 `tags_all`、`category1_name_all`。
  179 +
  180 +#### 3.3.2 范围过滤器 (range_filters)
  181 +
  182 +用于数值字段的范围过滤。
  183 +
  184 +**格式**:
  185 +
  186 +```json
  187 +{
  188 + "range_filters": {
  189 + "min_price": {
  190 + "gte": 50, // 大于等于
  191 + "lte": 200 // 小于等于
  192 + },
  193 + "max_price": {
  194 + "gt": 100 // 大于
  195 + },
  196 + "create_time": {
  197 + "gte": "2024-01-01T00:00:00Z" // 日期时间字符串
  198 + }
  199 + }
  200 +}
  201 +```
  202 +
  203 +**支持的操作符**:
  204 +- `gte`: 大于等于 (>=)
  205 +- `gt`: 大于 (>)
  206 +- `lte`: 小于等于 (<=)
  207 +- `lt`: 小于 (<)
  208 +
  209 +**注意**: 至少需要指定一个操作符。
  210 +
  211 +**常用范围字段**（详见[常用字段列表](./搜索API对接指南-08-数据模型与字段速查.md#93-常用字段列表)）:
  212 +- `min_price`: 最低价格
  213 +- `max_price`: 最高价格
  214 +- `compare_at_price`: 原价
  215 +- `create_time`: 创建时间
  216 +- `update_time`: 更新时间
  217 +
  218 +### 3.4 分面配置
  219 +
  220 +用于生成分面统计（分组聚合），常用于构建筛选器UI。
  221 +
  222 +#### 3.4.1 配置格式
  223 +
  224 +```json
  225 +{
  226 + "facets": [
  227 + {
  228 + "field": "category1_name",
  229 + "size": 15,
  230 + "type": "terms",
  231 + "disjunctive": false
  232 + },
  233 + {
  234 + "field": "brand_name",
  235 + "size": 10,
  236 + "type": "terms",
  237 + "disjunctive": true
  238 + },
  239 + {
  240 + "field": "specifications.color",
  241 + "size": 20,
  242 + "type": "terms",
  243 + "disjunctive": true
  244 + },
  245 + {
  246 + "field": "min_price",
  247 + "type": "range",
  248 + "ranges": [
  249 + {"key": "0-50", "to": 50},
  250 + {"key": "50-100", "from": 50, "to": 100},
  251 + {"key": "100-200", "from": 100, "to": 200},
  252 + {"key": "200+", "from": 200}
  253 + ]
  254 + }
  255 + ]
  256 +}
  257 +```
  258 +
  259 +#### 3.4.2 Facet 字段说明
  260 +
  261 +| 字段 | 类型 | 必填 | 默认值 | 说明 |
  262 +|------|------|------|--------|------|
  263 +| `field` | string | 是 | - | 分面字段名 |
  264 +| `size` | int | 否 | 10 | 返回的分面值数量（1-100） |
  265 +| `type` | string | 否 | "terms" | 分面类型：`terms`（词条聚合）或 `range`（范围聚合） |
  266 +| `disjunctive` | bool | 否 | false | 是否支持多选（disjunctive faceting）。启用后，选中该分面的过滤器时，仍会显示其他可选项 |
  267 +| `ranges` | array | 否 | null | 范围配置（仅 `type="range"` 时需要） |
  268 +
  269 +#### 3.4.3 disjunctive字段说明
  270 +
  271 +**重要特性**: `disjunctive` 字段控制分面的行为模式。启用后，选中该分面的过滤器时，仍会显示其他可选项
  272 +
  273 +**标准模式 (disjunctive: false)**:
  274 +- **行为**: 选中某个分面值后，该分面只显示选中的值
  275 +- **适用场景**: 层级类目、互斥选择
  276 +- **示例**: 类目下钻（玩具 > 娃娃 > 芭比）
  277 +
  278 +**Multi-Select 模式 (disjunctive: true)** ⭐:
  279 +- **行为**: 选中某个分面值后，该分面仍显示所有可选项
  280 +- **适用场景**: 颜色、品牌、尺码等可切换属性
  281 +- **示例**: 选择了"红色"后，仍能看到"蓝色"、"绿色"等选项
  282 +
  283 +**推荐配置**:
  284 +
  285 +| 分面类型 | disjunctive | 原因 |
  286 +|---------|-------------|------|
  287 +| 颜色 | `true` | 用户需要切换颜色 |
  288 +| 品牌 | `true` | 用户需要比较品牌 |
  289 +| 尺码 | `true` | 用户需要查看其他尺码 |
  290 +| 类目 | `false` | 层级下钻 |
  291 +| 价格区间 | `false` | 互斥选择 |
  292 +
  293 +#### 3.4.4 规格分面说明
  294 +
  295 +`specifications` 是嵌套字段，支持两种分面模式：
  296 +
  297 +**模式1：所有规格名称的分面**:
  298 +
  299 +```json
  300 +{
  301 + "facets": [
  302 + {
  303 + "field": "specifications",
  304 + "size": 10,
  305 + "type": "terms"
  306 + }
  307 + ]
  308 +}
  309 +```
  310 +
  311 +返回所有规格名称（name）及其对应的值（value）列表。每个 name 会生成一个独立的分面结果。
  312 +
  313 +**模式2：指定规格名称的分面**:
  314 +
  315 +```json
  316 +{
  317 + "facets": [
  318 + {
  319 + "field": "specifications.color",
  320 + "size": 20,
  321 + "type": "terms",
  322 + "disjunctive": true
  323 + },
  324 + {
  325 + "field": "specifications.size",
  326 + "size": 15,
  327 + "type": "terms",
  328 + "disjunctive": true
  329 + }
  330 + ]
  331 +}
  332 +```
  333 +
  334 +只返回指定规格名称的值列表。格式：`specifications.{name}`，其中 `{name}` 是规格名称（如"color"、"size"、"material"）。
  335 +
  336 +**返回格式示例**:
  337 +
  338 +```json
  339 +{
  340 + "facets": [
  341 + {
  342 + "field": "specifications.color",
  343 + "label": "color",
  344 + "type": "terms",
  345 + "values": [
  346 + {"value": "white", "count": 50, "selected": true}, // ✓ selected 字段由后端标记
  347 + {"value": "black", "count": 30, "selected": false},
  348 + {"value": "red", "count": 20, "selected": false}
  349 + ]
  350 + },
  351 + {
  352 + "field": "specifications.size",
  353 + "label": "size",
  354 + "type": "terms",
  355 + "values": [
  356 + {"value": "256GB", "count": 40, "selected": false},
  357 + {"value": "512GB", "count": 20, "selected": false}
  358 + ]
  359 + }
  360 + ]
  361 +}
  362 +```
  363 +
  364 +### 3.5 SKU筛选维度
  365 +
  366 +**功能说明**:
  367 +`sku_filter_dimension` 用于控制搜索列表页中 **每个 SPU 下方可切换的子款式（子 SKU）维度**，为字符串列表。
  368 +在店铺的 **主题装修配置** 中，商家可以为店铺设置一个或多个子款式筛选维度（例如 `color`、`size`），前端列表页会在每个 SPU 下展示这些维度对应的子 SKU 列表，用户可以通过点击不同维度值（如不同颜色）来切换展示的子款式。
  369 +当指定 `sku_filter_dimension` 后，后端会根据店铺的这项配置，从所有 SKU 中筛选出这些维度组合对应的子 SKU 数据：系统会按指定维度**组合**对 SKU 进行分组，每个维度组合只返回第一个 SKU（从简实现，选择该组合下的第一款），其余不在这些维度组合中的子 SKU 将不返回。
  370 +
  371 +**支持的维度值**:
  372 +1. **直接选项字段**: `option1`、`option2`、`option3`
  373 + - 直接使用对应的 `option1_value`、`option2_value`、`option3_value` 字段进行分组
  374 +
  375 +2. **规格/选项名称**: 通过 `option1_name`、`option2_name`、`option3_name` 匹配
  376 + - 例如：如果 `option1_name` 为 `"color"`，则可以使用 `sku_filter_dimension: ["color"]` 来按颜色分组
  377 +
  378 +**示例**:
  379 +
  380 +**按颜色筛选（假设 option1_name = "color"）**:
  381 +
  382 +```json
  383 +{
  384 + "query": "芭比娃娃",
  385 + "sku_filter_dimension": ["color"]
  386 +}
  387 +```
  388 +
  389 +**按选项1筛选**:
  390 +
  391 +```json
  392 +{
  393 + "query": "芭比娃娃",
  394 + "sku_filter_dimension": ["option1"]
  395 +}
  396 +```
  397 +
  398 +**按颜色 + 尺寸组合筛选（假设 option1_name = "color", option2_name = "size"）**:
  399 +
  400 +```json
  401 +{
  402 + "query": "芭比娃娃",
  403 + "sku_filter_dimension": ["color", "size"]
  404 +}
  405 +```
  406 +
  407 +## 响应格式说明
  408 +
  409 +### 4.1 标准响应结构
  410 +
  411 +```json
  412 +{
  413 + "results": [
  414 + {
  415 + "spu_id": "12345",
  416 + "title": "芭比时尚娃娃",
  417 + "brief": "高品质芭比娃娃",
  418 + "description": "详细描述...",
  419 + "vendor": "美泰",
  420 + "category": "玩具",
  421 + "category_path": "玩具/娃娃/时尚",
  422 + "category_name": "时尚",
  423 + "category_id": "cat_001",
  424 + "category_level": 3,
  425 + "category1_name": "玩具",
  426 + "category2_name": "娃娃",
  427 + "category3_name": "时尚",
  428 + "tags": ["娃娃", "玩具", "女孩"],
  429 + "price": 89.99,
  430 + "compare_at_price": 129.99,
  431 + "currency": "USD",
  432 + "image_url": "https://example.com/image.jpg",
  433 + "in_stock": true,
  434 + "sku_prices": [89.99, 99.99, 109.99],
  435 + "sku_weights": [100, 150, 200],
  436 + "sku_weight_units": ["g", "g", "g"],
  437 + "total_inventory": 500,
  438 + "option1_name": "color",
  439 + "option2_name": "size",
  440 + "option3_name": null,
  441 + "specifications": [
  442 + {"sku_id": "sku_001", "name": "color", "value": "pink"},
  443 + {"sku_id": "sku_001", "name": "size", "value": "standard"}
  444 + ],
  445 + "skus": [
  446 + {
  447 + "sku_id": "67890",
  448 + "price": 89.99,
  449 + "compare_at_price": 129.99,
  450 + "sku": "BARBIE-001",
  451 + "stock": 100,
  452 + "weight": 0.1,
  453 + "weight_unit": "kg",
  454 + "option1_value": "pink",
  455 + "option2_value": "standard",
  456 + "option3_value": null,
  457 + "image_src": "https://example.com/sku1.jpg"
  458 + }
  459 + ],
  460 + "relevance_score": 8.5
  461 + }
  462 + ],
  463 + "total": 118,
  464 + "max_score": 8.5,
  465 + "facets": [
  466 + {
  467 + "field": "category1_name",
  468 + "label": "category1_name",
  469 + "type": "terms",
  470 + "values": [
  471 + {
  472 + "value": "玩具",
  473 + "label": "玩具",
  474 + "count": 85,
  475 + "selected": false
  476 + }
  477 + ]
  478 + },
  479 + {
  480 + "field": "specifications.color",
  481 + "label": "color",
  482 + "type": "terms",
  483 + "values": [
  484 + {
  485 + "value": "pink",
  486 + "label": "pink",
  487 + "count": 30,
  488 + "selected": false
  489 + }
  490 + ]
  491 + }
  492 + ],
  493 + "query_info": {
  494 + "original_query": "芭比娃娃",
  495 + "query_normalized": "芭比娃娃",
  496 + "rewritten_query": "芭比娃娃",
  497 + "detected_language": "zh",
  498 + "translations": {
  499 + "en": "barbie doll"
  500 + },
  501 + "domain": "default"
  502 + },
  503 + "suggestions": [],
  504 + "related_searches": [],
  505 + "took_ms": 45,
  506 + "performance_info": null,
  507 + "debug_info": null
  508 +}
  509 +```
  510 +
  511 +### 4.2 响应字段说明
  512 +
  513 +| 字段 | 类型 | 说明 |
  514 +|------|------|------|
  515 +| `results` | array | 搜索结果列表（SpuResult对象数组） |
  516 +| `results[].spu_id` | string | SPU ID |
  517 +| `results[].title` | string | 商品标题 |
  518 +| `results[].price` | float | 价格（min_price） |
  519 +| `results[].skus` | array | SKU列表（如果指定了`sku_filter_dimension`，则按维度过滤后的SKU） |
  520 +| `results[].relevance_score` | float | 相关性分数 |
  521 +| `total` | integer | 匹配的总文档数 |
  522 +| `max_score` | float | 最高相关性分数 |
  523 +| `facets` | array | 分面统计结果 |
  524 +| `query_info` | object | query处理信息 |
  525 +| `took_ms` | integer | 搜索耗时（毫秒） |
  526 +| `debug_info` | object/null | 调试信息，仅当请求传 `debug=true` 时返回 |
  527 +
  528 +#### 4.2.1 query_info 说明
  529 +
  530 +`query_info` 包含本次搜索的查询解析与处理结果：
  531 +
  532 +| 子字段 | 类型 | 说明 |
  533 +|--------|------|------|
  534 +| `original_query` | string | 用户原始查询 |
  535 +| `query_normalized` | string | 归一化后的查询（去空白、大小写等预处理，用于后续解析与改写） |
  536 +| `rewritten_query` | string | 重写后的查询（同义词/词典扩展等） |
  537 +| `detected_language` | string | 检测到的查询语言（如 `zh`、`en`） |
  538 +| `translations` | object | 翻译结果，键为语言代码，值为翻译文本 |
  539 +| `domain` | string | 查询域（如 `default`、`title`、`brand` 等） |
  540 +
  541 +#### 4.2.2 debug_info 说明
  542 +
  543 +`debug_info` 主要用于检索效果评估、融合打分分析与 bad case 排查。
  544 +
  545 +`debug_info.query_analysis` 常见字段：
  546 +
  547 +| 子字段 | 类型 | 说明 |
  548 +|--------|------|------|
  549 +| `original_query` | string | 原始查询 |
  550 +| `query_normalized` | string | 归一化后的查询 |
  551 +| `rewritten_query` | string | 重写后的查询 |
  552 +| `detected_language` | string | 检测到的语言 |
  553 +| `translations` | object | 翻译结果 |
  554 +| `query_text_by_lang` | object | 实际参与检索的多语言 query 文本 |
  555 +| `search_langs` | array[string] | 实际参与检索的语言列表 |
  556 +| `supplemental_search_langs` | array[string] | 因 mixed query 补入的附加语言列表 |
  557 +| `has_vector` | boolean | 是否生成了向量 |
  558 +
  559 +`debug_info.per_result[]` 常见字段：
  560 +
  561 +| 子字段 | 类型 | 说明 |
  562 +|--------|------|------|
  563 +| `spu_id` | string | 结果 SPU ID |
  564 +| `es_score` | float | ES 原始 `_score` |
  565 +| `rerank_score` | float | 重排分数 |
  566 +| `text_score` | float | 文本相关性大分（由 `base_query` / `base_query_trans_*` / `fallback_original_query_*` 聚合而来） |
  567 +| `text_source_score` | float | `base_query` 分数 |
  568 +| `text_translation_score` | float | `base_query_trans_*` 里的最大分数 |
  569 +| `text_fallback_score` | float | `fallback_original_query_*` 里的最大分数 |
  570 +| `text_primary_score` | float | 文本大分中的主证据部分 |
  571 +| `text_support_score` | float | 文本大分中的辅助证据部分 |
  572 +| `knn_score` | float | `knn_query` 分数 |
  573 +| `fused_score` | float | 最终融合分数 |
  574 +| `matched_queries` | object/array | ES named queries 命中详情 |
  575 +
  576 +### 4.3 SpuResult字段说明
  577 +
  578 +| 字段 | 类型 | 说明 |
  579 +|------|------|------|
  580 +| `spu_id` | string | SPU ID |
  581 +| `title` | string | 商品标题（根据language参数自动选择 `title.zh` 或 `title.en`） |
  582 +| `brief` | string | 商品短描述（根据language参数自动选择） |
  583 +| `description` | string | 商品详细描述（根据language参数自动选择） |
  584 +| `vendor` | string | 供应商/品牌（根据language参数自动选择） |
  585 +| `category` | string | 类目（兼容字段，等同于category_name） |
  586 +| `category_path` | string | 类目路径（多级，用于面包屑，根据language参数自动选择） |
  587 +| `category_name` | string | 类目名称（展示用，根据language参数自动选择） |
  588 +| `category_id` | string | 类目ID |
  589 +| `category_level` | integer | 类目层级（1/2/3） |
  590 +| `category1_name` | string | 一级类目名称 |
  591 +| `category2_name` | string | 二级类目名称 |
  592 +| `category3_name` | string | 三级类目名称 |
  593 +| `tags` | array[string] | 标签列表 |
  594 +| `price` | float | 价格（min_price） |
  595 +| `compare_at_price` | float | 原价 |
  596 +| `currency` | string | 货币单位（默认USD） |
  597 +| `image_url` | string | 主图URL |
  598 +| `in_stock` | boolean | 是否有库存（任意SKU有库存即为true） |
  599 +| `sku_prices` | array[float] | 所有SKU价格列表 |
  600 +| `sku_weights` | array[integer] | 所有SKU重量列表 |
  601 +| `sku_weight_units` | array[string] | 所有SKU重量单位列表 |
  602 +| `total_inventory` | integer | 总库存 |
  603 +| `sales` | integer | 销量（展示销量） |
  604 +| `option1_name` | string | 选项1名称（如"color"） |
  605 +| `option2_name` | string | 选项2名称（如"size"） |
  606 +| `option3_name` | string | 选项3名称 |
  607 +| `specifications` | array[object] | 规格列表（与ES specifications字段对应） |
  608 +| `skus` | array | SKU 列表 |
  609 +| `relevance_score` | float | 相关性分数（默认为 ES 原始分数；当开启 AI 搜索时为融合后的最终分数） |
  610 +
  611 +### 4.4 SkuResult字段说明
  612 +
  613 +| 字段 | 类型 | 说明 |
  614 +|------|------|------|
  615 +| `sku_id` | string | SKU ID |
  616 +| `price` | float | 价格 |
  617 +| `compare_at_price` | float | 原价 |
  618 +| `sku` | string | SKU编码（sku_code） |
  619 +| `stock` | integer | 库存数量 |
  620 +| `weight` | float | 重量 |
  621 +| `weight_unit` | string | 重量单位 |
  622 +| `option1_value` | string | 选项1取值（如color值） |
  623 +| `option2_value` | string | 选项2取值（如size值） |
  624 +| `option3_value` | string | 选项3取值 |
  625 +| `image_src` | string | SKU图片地址 |
  626 +
  627 +### 4.5 多语言字段说明
  628 +
  629 +- `title`, `brief`, `description`, `vendor`, `category_path`, `category_name` 会根据请求的 `language` 参数自动选择对应的中英文字段
  630 +- `language="zh"`: 优先返回 `*_zh` 字段，如果为空则回退到 `*_en` 字段
  631 +- `language="en"`: 优先返回 `*_en` 字段，如果为空则回退到 `*_zh` 字段
  632 +
  633 +---
  634 +
  635 +## 8. 常见场景示例
  636 +
  637 +以下示例仅展示**请求体**（body）；实际调用时请加上请求头 `X-Tenant-ID: <租户ID>`（或 URL 参数 `tenant_id`），参见 [3.1 接口信息](#31-接口信息)。
  638 +
  639 +### 8.1 基础搜索与排序
  640 +
  641 +**按价格从低到高排序**:
  642 +
  643 +```json
  644 +{
  645 + "query": "玩具",
  646 + "size": 20,
  647 + "from": 0,
  648 + "sort_by": "price",
  649 + "sort_order": "asc"
  650 +}
  651 +```
  652 +
  653 +**按价格从高到低排序**:
  654 +
  655 +```json
  656 +{
  657 + "query": "玩具",
  658 + "size": 20,
  659 + "from": 0,
  660 + "sort_by": "price",
  661 + "sort_order": "desc"
  662 +}
  663 +```
  664 +
  665 +**按销量从高到低排序**:
  666 +
  667 +```json
  668 +{
  669 + "query": "玩具",
  670 + "size": 20,
  671 + "from": 0,
  672 + "sort_by": "sales",
  673 + "sort_order": "desc"
  674 +}
  675 +```
  676 +
  677 +**按默认（相关性）排序**:
  678 +
  679 +```json
  680 +{
  681 + "query": "玩具",
  682 + "size": 20,
  683 + "from": 0
  684 +}
  685 +```
  686 +
  687 +### 8.2 过滤搜索
  688 +
  689 +**需求**: 搜索"玩具"，筛选类目为"益智玩具"，价格在50-200之间
  690 +
  691 +```json
  692 +{
  693 + "query": "玩具",
  694 + "size": 20,
  695 + "language": "zh",
  696 + "filters": {
  697 + "category_name": "益智玩具"
  698 + },
  699 + "range_filters": {
  700 + "min_price": {
  701 + "gte": 50,
  702 + "lte": 200
  703 + }
  704 + }
  705 +}
  706 +```
  707 +
  708 +**需求**: 搜索"手机"，筛选多个品牌，价格范围
  709 +
  710 +```json
  711 +{
  712 + "query": "手机",
  713 + "size": 20,
  714 + "language": "zh",
  715 + "filters": {
  716 + "vendor.zh.keyword": ["品牌A", "品牌B"]
  717 + },
  718 + "range_filters": {
  719 + "min_price": {
  720 + "gte": 50,
  721 + "lte": 200
  722 + }
  723 + }
  724 +}
  725 +```
  726 +
  727 +### 8.3 分面搜索
  728 +
  729 +**需求**: 搜索"玩具"，获取类目和规格的分面统计，用于构建筛选器
  730 +
  731 +```json
  732 +{
  733 + "query": "玩具",
  734 + "size": 20,
  735 + "language": "zh",
  736 + "facets": [
  737 + {"field": "category1_name", "size": 15, "type": "terms"},
  738 + {"field": "category2_name", "size": 10, "type": "terms"},
  739 + {"field": "specifications", "size": 10, "type": "terms"}
  740 + ]
  741 +}
  742 +```
  743 +
  744 +**需求**: 搜索"手机"，获取价格区间和规格的分面统计
  745 +
  746 +```json
  747 +{
  748 + "query": "手机",
  749 + "size": 20,
  750 + "language": "zh",
  751 + "facets": [
  752 + {
  753 + "field": "min_price",
  754 + "type": "range",
  755 + "ranges": [
  756 + {"key": "0-50", "to": 50},
  757 + {"key": "50-100", "from": 50, "to": 100},
  758 + {"key": "100-200", "from": 100, "to": 200},
  759 + {"key": "200+", "from": 200}
  760 + ]
  761 + },
  762 + {
  763 + "field": "specifications",
  764 + "size": 10,
  765 + "type": "terms"
  766 + }
  767 + ]
  768 +}
  769 +```
  770 +
  771 +### 8.4 规格过滤与分面
  772 +
  773 +**需求**: 搜索"手机"，筛选color为"white"的商品
  774 +
  775 +```json
  776 +{
  777 + "query": "手机",
  778 + "size": 20,
  779 + "language": "zh",
  780 + "filters": {
  781 + "specifications": {
  782 + "name": "color",
  783 + "value": "white"
  784 + }
  785 + }
  786 +}
  787 +```
  788 +
  789 +**需求**: 搜索"手机"，筛选color为"white"且size为"256GB"的商品
  790 +
  791 +```json
  792 +{
  793 + "query": "手机",
  794 + "size": 20,
  795 + "language": "zh",
  796 + "filters": {
  797 + "specifications": [
  798 + {"name": "color", "value": "white"},
  799 + {"name": "size", "value": "256GB"}
  800 + ]
  801 + }
  802 +}
  803 +```
  804 +
  805 +**需求**: 搜索"手机"，筛选size为"3"、"4"或"5"，且color为"green"的商品
  806 +
  807 +```json
  808 +{
  809 + "query": "手机",
  810 + "size": 20,
  811 + "language": "zh",
  812 + "filters": {
  813 + "specifications": [
  814 + {"name": "size", "value": "3"},
  815 + {"name": "size", "value": "4"},
  816 + {"name": "size", "value": "5"},
  817 + {"name": "color", "value": "green"}
  818 + ]
  819 + }
  820 +}
  821 +```
  822 +
  823 +**需求**: 搜索"手机"，获取所有规格的分面统计
  824 +
  825 +```json
  826 +{
  827 + "query": "手机",
  828 + "size": 20,
  829 + "language": "zh",
  830 + "facets": [
  831 + {"field": "specifications", "size": 10, "type": "terms"}
  832 + ]
  833 +}
  834 +```
  835 +
  836 +**需求**: 只获取"color"和"size"规格的分面统计
  837 +
  838 +```json
  839 +{
  840 + "query": "手机",
  841 + "size": 20,
  842 + "language": "zh",
  843 + "facets": [
  844 + {"field": "specifications.color", "size": 20, "type": "terms"},
  845 + {"field": "specifications.size", "size": 15, "type": "terms"}
  846 + ]
  847 +}
  848 +```
  849 +
  850 +**需求**: 搜索"手机"，筛选类目和规格，并获取对应的分面统计
  851 +
  852 +```json
  853 +{
  854 + "query": "手机",
  855 + "size": 20,
  856 + "language": "zh",
  857 + "filters": {
  858 + "category_name": "手机",
  859 + "specifications": {
  860 + "name": "color",
  861 + "value": "white"
  862 + }
  863 + },
  864 + "facets": [
  865 + {"field": "category1_name", "size": 15, "type": "terms"},
  866 + {"field": "category2_name", "size": 10, "type": "terms"},
  867 + {"field": "specifications.color", "size": 20, "type": "terms"},
  868 + {"field": "specifications.size", "size": 15, "type": "terms"}
  869 + ]
  870 +}
  871 +```
  872 +
  873 +### 8.5 SKU筛选
  874 +
  875 +**需求**: 搜索"芭比娃娃"，每个SPU下按颜色筛选，每种颜色只显示一个SKU
  876 +
  877 +```json
  878 +{
  879 + "query": "芭比娃娃",
  880 + "size": 20,
  881 + "sku_filter_dimension": ["color"]
  882 +}
  883 +```
  884 +
  885 +**说明**:
  886 +- 如果 `option1_name` 为 `"color"`，则使用 `sku_filter_dimension: ["color"]` 可以按颜色分组
  887 +- 每个SPU下，每种颜色只会返回第一个SKU
  888 +- 如果维度不匹配，返回所有SKU（不进行过滤）
  889 +
  890 +### 8.6 分页查询
  891 +
  892 +**需求**: 获取第2页结果（每页20条）
  893 +
  894 +```json
  895 +{
  896 + "query": "手机",
  897 + "size": 20,
  898 + "from": 20
  899 +}
  900 +```
  901 +
  902 +---
  903 +
... ...
docs/搜索API对接指南-02-搜索建议与即时搜索.md 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,81 @@
  1 +# 搜索API对接指南-02-搜索建议与即时搜索
  2 +
  3 +本篇面向前端联想词/搜索框团队，独立阅读 `GET /search/suggestions` 与 `GET /search/instant`。
  4 +
  5 +## 搜索接口
  6 +
  7 +### 3.7 搜索建议接口
  8 +
  9 +- **端点**: `GET /search/suggestions`
  10 +- **描述**: 返回搜索建议（自动补全/热词），支持多语言。
  11 +
  12 +#### 查询参数
  13 +
  14 +| 参数 | 类型 | 必填 | 默认值 | 描述 |
  15 +|------|------|------|--------|------|
  16 +| `q` | string | Y | - | 查询字符串（至少 1 个字符） |
  17 +| `size` | integer | N | 10 | 返回建议数量（1-50） |
  18 +| `language` | string | N | `en` | 请求语言，如 `zh` / `en` / `ar` / `ru`，用于路由到对应语种 suggestion 索引 |
  19 +| `debug` | bool | N | `false` | 是否开启调试（目前主要用于排查 suggestion 排序与语言解析） |
  20 +
  21 +> **租户标识**：同 [-01-搜索接口](./搜索API对接指南-01-搜索接口.md#31-接口信息)，通过请求头 `X-Tenant-ID` 或 query 参数 `tenant_id` 传递。
  22 +
  23 +#### 响应示例
  24 +
  25 +```json
  26 +{
  27 + "query": "iph",
  28 + "language": "en",
  29 + "resolved_language": "en",
  30 + "suggestions": [
  31 + {
  32 + "text": "iphone 15",
  33 + "lang": "en",
  34 + "score": 12.37,
  35 + "rank_score": 5.1,
  36 + "sources": ["query_log", "qanchor"],
  37 + "lang_source": "log_field",
  38 + "lang_confidence": 1.0,
  39 + "lang_conflict": false
  40 + }
  41 + ],
  42 + "took_ms": 12
  43 +}
  44 +```
  45 +
  46 +#### 请求示例
  47 +
  48 +```bash
  49 +curl "http://localhost:6002/search/suggestions?q=芭&size=5&language=zh" \
  50 + -H "X-Tenant-ID: 162"
  51 +```
  52 +
  53 +### 3.8 即时搜索接口
  54 +
  55 +> ⚠️ 当前版本未开放该能力。接口会明确返回 `501 Not Implemented`，避免误用未完成实现。
  56 +
  57 +- **端点**: `GET /search/instant`
  58 +- **描述**: 即时搜索预留端点，后续会在独立实现完成后开放。
  59 +
  60 +#### 查询参数
  61 +
  62 +| 参数 | 类型 | 必填 | 默认值 | 描述 |
  63 +|------|------|------|--------|------|
  64 +| `q` | string | Y | - | 搜索查询（至少 2 个字符） |
  65 +| `size` | integer | N | 5 | 返回结果数量（1-20） |
  66 +
  67 +#### 请求示例
  68 +
  69 +```bash
  70 +curl "http://localhost:6002/search/instant?q=玩具&size=5"
  71 +```
  72 +
  73 +#### 当前响应
  74 +
  75 +```json
  76 +{
  77 + "error": "/search/instant is not implemented yet. Use POST /search/ for production traffic.",
  78 + "status_code": 501
  79 +}
  80 +```
  81 +
... ...
docs/搜索API对接指南-03-获取文档.md 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,40 @@
  1 +# 搜索API对接指南-03-获取文档（GET /search/{doc_id}）
  2 +
  3 +用于点击结果后的详情页回源，或排查某个文档在检索侧的字段情况。
  4 +
  5 +## 搜索接口
  6 +
  7 +### 3.9 获取单个文档
  8 +
  9 +- **端点**: `GET /search/{doc_id}`
  10 +- **描述**: 根据文档 ID 获取单个商品详情，用于点击结果后的详情页或排查问题。
  11 +- **租户标识**：同 [-01-搜索接口](./搜索API对接指南-01-搜索接口.md#31-接口信息)，通过请求头 `X-Tenant-ID` 或 query 参数 `tenant_id` 传递。
  12 +
  13 +#### 路径参数
  14 +
  15 +| 参数 | 类型 | 描述 |
  16 +|------|------|------|
  17 +| `doc_id` | string | 商品或文档 ID |
  18 +
  19 +#### 响应示例
  20 +
  21 +```json
  22 +{
  23 + "id": "12345",
  24 + "source": {
  25 + "title": {
  26 + "zh": "芭比时尚娃娃"
  27 + },
  28 + "min_price": 89.99,
  29 + "category1_name": "玩具"
  30 + }
  31 +}
  32 +```
  33 +
  34 +#### 请求示例
  35 +
  36 +```bash
  37 +curl "http://localhost:6002/search/12345" -H "X-Tenant-ID: 162"
  38 +# 或使用 query 参数：curl "http://localhost:6002/search/12345?tenant_id=162"
  39 +```
  40 +
... ...
docs/搜索API对接指南-05-索引接口（Indexer）.md 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,767 @@
  1 +# 搜索API对接指南-05-索引接口（Indexer）
  2 +
  3 +本篇覆盖数据同步/索引构建相关的所有接口（原文第 5 章），用于 `external indexer` 和 `Indexer 服务` 的对接。
  4 +
  5 +## 索引接口
  6 +
  7 +本节内容与 `api/routes/indexer.py` 中的索引相关服务一致，包含以下接口：
  8 +
  9 +| 接口 | 方法 | 路径 | 说明 |
  10 +|------|------|------|------|
  11 +| 全量重建索引 | POST | `/indexer/reindex` | 将指定租户所有 SPU 导入 ES（不删现有索引） |
  12 +| 增量索引 | POST | `/indexer/index` | 按 SPU ID 列表索引/删除，支持自动检测删除与显式删除 |
  13 +| 查询文档 | POST | `/indexer/documents` | 按 SPU ID 列表查询 ES 文档，不写入 ES |
  14 +| 构建 ES 文档（正式） | POST | `/indexer/build-docs` | 由上游提供 MySQL 行数据，返回 ES-ready 文档，不写 ES |
  15 +| 构建 ES 文档（测试） | POST | `/indexer/build-docs-from-db` | 由本服务查库并构建文档，仅测试/调试用 |
  16 +| 内容理解字段生成 | POST | `/indexer/enrich-content` | 根据商品标题批量生成 qanchors、semantic_attributes、tags（供微服务组合方式使用） |
  17 +| 索引健康检查 | GET | `/indexer/health` | 检查索引服务与数据库连接状态 |
  18 +
  19 +#### 5.0 支撑外部 indexer 的三种方式
  20 +
  21 +本服务对**外部 indexer 程序**（如 Java 索引系统）提供三种对接方式，可按需选择：
  22 +
  23 +| 方式 | 说明 | 适用场景 |
  24 +|------|------|----------|
  25 +| **1）doc 填充接口** | 调用 `POST /indexer/build-docs` 或 `POST /indexer/build-docs-from-db`，由本服务基于 MySQL 行数据构建完整 ES 文档（含多语言、向量、规格等），**不写入 ES**，由调用方自行写入。 | 希望一站式拿到 ES-ready doc，由己方控制写 ES 的时机与索引名。 |
  26 +| **2）微服务组合** | 单独调用**翻译**、**向量化**、**内容理解字段生成**等接口，由 indexer 程序自己组装 doc 并写入 ES。翻译与向量化为独立微服务（见第 7 节）；内容理解为 Indexer 服务内接口 `POST /indexer/enrich-content`。 | 需要灵活编排、或希望将 LLM/向量等耗时步骤与主链路解耦（如异步补齐 qanchors/tags）。 |
  27 +| **3）本服务直接写 ES** | 调用全量索引 `POST /indexer/reindex`、增量索引 `POST /indexer/index`（指定 SPU ID 列表），由本服务从 MySQL 拉数并直接写入 ES。 | 自建运维、联调或不需要由 Java 写 ES 的场景。 |
  28 +
  29 +- **方式 1** 与 **方式 2** 下，ES 的写入方均为外部 indexer（或 Java），职责清晰。
  30 +- **方式 3** 下，本服务同时负责读库、构建 doc 与写 ES。
  31 +
  32 +### 5.1 为租户创建索引
  33 +
  34 +为租户创建索引需要两个步骤：
  35 +
  36 +1. **创建索引结构**（可选，仅在需要更新 mapping 或在新环境首次创建时执行）
  37 + - 使用脚本创建 ES 索引结构（基于 `mappings/search_products.json`）
  38 + - 如果索引已存在，会提示用户确认（会删除现有数据）
  39 +
  40 +2. **导入数据**（必需）
  41 + - 使用全量索引接口 `/indexer/reindex` 导入数据
  42 +
  43 +**创建索引结构（支持多环境 namespace）**：
  44 +
  45 +```bash
  46 +# 以 UAT 环境为例：
  47 +# 1. 准备 UAT 环境的 .env（包含 UAT 的 ES_HOST/DB_HOST 等）
  48 +# 2. 设置环境前缀（也可以直接在 .env 中配置）：
  49 +export RUNTIME_ENV=uat
  50 +export ES_INDEX_NAMESPACE=uat_
  51 +
  52 +# 3. 为 tenant_id=170 创建索引结构
  53 +./scripts/create_tenant_index.sh 170
  54 +```
  55 +
  56 +脚本会自动从项目根目录的 `.env` 文件加载 ES 配置，并根据 `ES_INDEX_NAMESPACE` 创建：
  57 +
  58 +- prod 环境（ES_INDEX_NAMESPACE 为空）：`search_products_tenant_170`
  59 +- UAT 环境（ES_INDEX_NAMESPACE=uat_）：`uat_search_products_tenant_170`
  60 +
  61 +**注意事项**：
  62 +- ⚠️ 如果索引已存在，脚本会提示确认，确认后会删除现有数据
  63 +- 创建索引后，**必须**调用 `/indexer/reindex` 导入数据
  64 +- 如果只是更新数据而不需要修改索引结构，直接使用 `/indexer/reindex` 即可
  65 +
  66 +---
  67 +
  68 +### 5.2 全量索引接口
  69 +
  70 +- **端点**: `POST /indexer/reindex`
  71 +- **描述**: 全量索引，将指定租户的所有SPU数据导入到ES索引（不会删除现有索引）。**推荐仅用于自测/运维场景**；生产环境下更推荐由 Java 等上游控制调度与写 ES。
  72 +
  73 +#### 请求参数
  74 +
  75 +```json
  76 +{
  77 + "tenant_id": "162",
  78 + "batch_size": 500
  79 +}
  80 +```
  81 +
  82 +| 参数 | 类型 | 必填 | 默认值 | 说明 |
  83 +|------|------|------|--------|------|
  84 +| `tenant_id` | string | Y | - | 租户ID |
  85 +| `batch_size` | integer | N | 500 | 批量导入大小 |
  86 +
  87 +#### 响应格式
  88 +
  89 +**成功响应（200 OK）**（示例，实际 `index_name` 会带上 tenant 和环境前缀）:
  90 +
  91 +```json
  92 +{
  93 + "success": true,
  94 + "total": 1000,
  95 + "indexed": 1000,
  96 + "failed": 0,
  97 + "elapsed_time": 12.34,
  98 + "index_name": "search_products_tenant_162",
  99 + "tenant_id": "162"
  100 +}
  101 +```
  102 +
  103 +**错误响应**:
  104 +- `400 Bad Request`: 参数错误
  105 +- `503 Service Unavailable`: 服务未初始化
  106 +
  107 +#### 请求示例
  108 +
  109 +**全量索引（不会删除现有索引）**:
  110 +
  111 +```bash
  112 +curl -X POST "http://localhost:6004/indexer/reindex" \
  113 + -H "Content-Type: application/json" \
  114 + -d '{
  115 + "tenant_id": "162",
  116 + "batch_size": 500
  117 + }'
  118 +```
  119 +
  120 +**查看日志**:
  121 +
  122 +```bash
  123 +# 查看API日志（包含索引操作日志）
  124 +tail -f logs/api.log
  125 +
  126 +# 或者查看所有日志文件
  127 +tail -f logs/*.log
  128 +```
  129 +
  130 +> ⚠️ **重要提示**：如需 **创建索引结构**，请参考 [5.1 为租户创建索引](#51-为租户创建索引) 章节，使用 `./scripts/create_tenant_index.sh <tenant_id>`。创建后需要调用 `/indexer/reindex` 导入数据。
  131 +
  132 +**查看索引日志**:
  133 +
  134 +索引操作的所有关键信息都会记录到 `logs/indexer.log` 文件中（JSON 格式），包括：
  135 +- 请求开始和结束时间
  136 +- 租户ID、SPU ID、操作类型
  137 +- 每个SPU的处理状态
  138 +- ES批量写入结果
  139 +- 成功/失败统计和详细错误信息
  140 +
  141 +```bash
  142 +# 实时查看索引日志（包含全量和增量索引的所有操作）
  143 +tail -f logs/indexer.log
  144 +
  145 +# 使用 grep 查询（简单方式）
  146 +# 查看全量索引日志
  147 +grep "\"index_type\":\"bulk\"" logs/indexer.log | tail -100
  148 +
  149 +# 查看增量索引日志
  150 +grep "\"index_type\":\"incremental\"" logs/indexer.log | tail -100
  151 +
  152 +# 查看特定租户的索引日志
  153 +grep "\"tenant_id\":\"162\"" logs/indexer.log | tail -100
  154 +
  155 +# 使用 jq 查询（推荐，更精确的 JSON 查询）
  156 +# 安装 jq: sudo apt-get install jq 或 brew install jq
  157 +
  158 +# 查看全量索引日志
  159 +cat logs/indexer.log | jq 'select(.index_type == "bulk")' | tail -100
  160 +
  161 +# 查看增量索引日志
  162 +cat logs/indexer.log | jq 'select(.index_type == "incremental")' | tail -100
  163 +
  164 +# 查看特定租户的索引日志
  165 +cat logs/indexer.log | jq 'select(.tenant_id == "162")' | tail -100
  166 +
  167 +# 查看失败的索引操作
  168 +cat logs/indexer.log | jq 'select(.operation == "request_complete" and .failed_count > 0)'
  169 +
  170 +# 查看特定SPU的处理日志
  171 +cat logs/indexer.log | jq 'select(.spu_id == "123")'
  172 +
  173 +# 查看最近的索引请求统计
  174 +cat logs/indexer.log | jq 'select(.operation == "request_complete") | {timestamp, index_type, tenant_id, total_count, success_count, failed_count, elapsed_time}'
  175 +```
  176 +
  177 +### 5.3 增量索引接口
  178 +
  179 +- **端点**: `POST /indexer/index`
  180 +- **描述**: 增量索引接口，根据指定的SPU ID列表进行索引，直接将数据写入ES。用于增量更新指定商品。**推荐仅作为内部/调试入口**；正式对接建议改用 `/indexer/build-docs`，由上游写 ES。
  181 +
  182 +**删除说明**：
  183 +- `spu_ids`中的SPU：如果数据库`deleted=1`，自动从ES删除，响应状态为`deleted`
  184 +- `delete_spu_ids`中的SPU：直接删除，响应状态为`deleted`、`not_found`或`failed`
  185 +
  186 +#### 请求参数
  187 +
  188 +```json
  189 +{
  190 + "tenant_id": "162",
  191 + "spu_ids": ["123", "456", "789"],
  192 + "delete_spu_ids": ["100", "101"]
  193 +}
  194 +```
  195 +
  196 +| 参数 | 类型 | 必填 | 说明 |
  197 +|------|------|------|------|
  198 +| `tenant_id` | string | Y | 租户ID |
  199 +| `spu_ids` | array[string] | N | SPU ID列表（1-100个），要索引的SPU。如果为空，则只执行删除操作 |
  200 +| `delete_spu_ids` | array[string] | N | 显式指定要删除的SPU ID列表（1-100个），可选。无论数据库状态如何，都会从ES中删除这些SPU |
  201 +
  202 +**注意**：
  203 +- `spu_ids` 和 `delete_spu_ids` 不能同时为空
  204 +- 每个列表最多支持100个SPU ID
  205 +- 如果SPU在`spu_ids`中且数据库`deleted=1`，会自动从ES删除（自动检测删除）
  206 +
  207 +#### 响应格式
  208 +
  209 +```json
  210 +{
  211 + "spu_ids": [
  212 + {
  213 + "spu_id": "123",
  214 + "status": "indexed"
  215 + },
  216 + {
  217 + "spu_id": "456",
  218 + "status": "deleted"
  219 + },
  220 + {
  221 + "spu_id": "789",
  222 + "status": "failed",
  223 + "msg": "SPU not found (unexpected)"
  224 + }
  225 + ],
  226 + "delete_spu_ids": [
  227 + {
  228 + "spu_id": "100",
  229 + "status": "deleted"
  230 + },
  231 + {
  232 + "spu_id": "101",
  233 + "status": "not_found"
  234 + },
  235 + {
  236 + "spu_id": "102",
  237 + "status": "failed",
  238 + "msg": "Failed to delete from ES: Connection timeout"
  239 + }
  240 + ],
  241 + "total": 6,
  242 + "success_count": 4,
  243 + "failed_count": 2,
  244 + "elapsed_time": 1.23,
  245 + "index_name": "search_products",
  246 + "tenant_id": "162"
  247 +}
  248 +```
  249 +
  250 +| 字段 | 类型 | 说明 |
  251 +|------|------|------|
  252 +| `spu_ids` | array | spu_ids对应的响应列表，每个元素包含 `spu_id` 和 `status` |
  253 +| `spu_ids[].status` | string | 状态：`indexed`（已索引）、`deleted`（已删除，自动检测）、`failed`（失败） |
  254 +| `spu_ids[].msg` | string | 当status为`failed`时，包含失败原因（可选） |
  255 +| `delete_spu_ids` | array | delete_spu_ids对应的响应列表，每个元素包含 `spu_id` 和 `status` |
  256 +| `delete_spu_ids[].status` | string | 状态：`deleted`（已删除）、`not_found`（ES中不存在）、`failed`（失败） |
  257 +| `delete_spu_ids[].msg` | string | 当status为`failed`时，包含失败原因（可选） |
  258 +| `total` | integer | 总处理数量（spu_ids数量 + delete_spu_ids数量） |
  259 +| `success_count` | integer | 成功数量（indexed + deleted + not_found） |
  260 +| `failed_count` | integer | 失败数量 |
  261 +| `elapsed_time` | float | 耗时（秒） |
  262 +| `index_name` | string | 索引名称 |
  263 +| `tenant_id` | string | 租户ID |
  264 +
  265 +**状态说明**：
  266 +- `spu_ids` 的状态：
  267 + - `indexed`: SPU已成功索引到ES
  268 + - `deleted`: SPU在数据库中被标记为deleted=1，已从ES删除（自动检测）
  269 + - `failed`: 处理失败，会包含`msg`字段说明失败原因
  270 +- `delete_spu_ids` 的状态：
  271 + - `deleted`: SPU已从ES成功删除
  272 + - `not_found`: SPU在ES中不存在（也算成功，可能已经被删除过）
  273 + - `failed`: 删除失败，会包含`msg`字段说明失败原因
  274 +
  275 +#### 请求示例
  276 +
  277 +**示例1：普通增量索引（自动检测删除）**:
  278 +
  279 +```bash
  280 +curl -X POST "http://localhost:6004/indexer/index" \
  281 + -H "Content-Type: application/json" \
  282 + -d '{
  283 + "tenant_id": "162",
  284 + "spu_ids": ["123", "456", "789"]
  285 + }'
  286 +```
  287 +
  288 +说明：如果SPU 456在数据库中`deleted=1`，会自动从ES删除，在响应中`spu_ids`列表里456的状态为`deleted`。
  289 +
  290 +**示例2：显式删除（批量删除）**:
  291 +
  292 +```bash
  293 +curl -X POST "http://localhost:6004/indexer/index" \
  294 + -H "Content-Type: application/json" \
  295 + -d '{
  296 + "tenant_id": "162",
  297 + "spu_ids": ["123", "456"],
  298 + "delete_spu_ids": ["100", "101", "102"]
  299 + }'
  300 +```
  301 +
  302 +说明：SPU 100、101、102会被显式删除，无论数据库状态如何。
  303 +
  304 +**示例3：仅删除（不索引）**:
  305 +
  306 +```bash
  307 +curl -X POST "http://localhost:6004/indexer/index" \
  308 + -H "Content-Type: application/json" \
  309 + -d '{
  310 + "tenant_id": "162",
  311 + "spu_ids": [],
  312 + "delete_spu_ids": ["100", "101"]
  313 + }'
  314 +```
  315 +
  316 +说明：只执行删除操作，不进行索引。
  317 +
  318 +**示例4：混合操作（索引+删除）**:
  319 +
  320 +```bash
  321 +curl -X POST "http://localhost:6004/indexer/index" \
  322 + -H "Content-Type: application/json" \
  323 + -d '{
  324 + "tenant_id": "162",
  325 + "spu_ids": ["123", "456", "789"],
  326 + "delete_spu_ids": ["100", "101"]
  327 + }'
  328 +```
  329 +
  330 +说明：同时执行索引和删除操作。
  331 +
  332 +#### 日志说明
  333 +
  334 +增量索引操作的所有关键信息都会记录到 `logs/indexer.log` 文件中（JSON格式），包括：
  335 +- 请求开始和结束时间
  336 +- 每个SPU的处理状态（获取、转换、索引、删除）
  337 +- ES批量写入结果
  338 +- 成功/失败统计
  339 +- 详细的错误信息
  340 +
  341 +日志查询方式请参考[5.1节查看索引日志](#51-全量重建索引接口)部分。
  342 +
  343 +### 5.4 查询文档接口
  344 +
  345 +- **端点**: `POST /indexer/documents`
  346 +- **描述**: 查询文档接口，根据SPU ID列表获取ES文档数据（**不写入ES**）。用于查看、调试或验证SPU数据。
  347 +
  348 +#### 请求参数
  349 +
  350 +```json
  351 +{
  352 + "tenant_id": "162",
  353 + "spu_ids": ["123", "456", "789"]
  354 +}
  355 +```
  356 +
  357 +| 参数 | 类型 | 必填 | 说明 |
  358 +|------|------|------|------|
  359 +| `tenant_id` | string | Y | 租户ID |
  360 +| `spu_ids` | array[string] | Y | SPU ID列表（1-100个） |
  361 +
  362 +#### 响应格式
  363 +
  364 +```json
  365 +{
  366 + "success": [
  367 + {
  368 + "spu_id": "123",
  369 + "document": {
  370 + "tenant_id": "162",
  371 + "spu_id": "123",
  372 + "title": {
  373 + "zh": "商品标题"
  374 + },
  375 + ...
  376 + }
  377 + },
  378 + {
  379 + "spu_id": "456",
  380 + "document": {...}
  381 + }
  382 + ],
  383 + "failed": [
  384 + {
  385 + "spu_id": "789",
  386 + "error": "SPU not found or deleted"
  387 + }
  388 + ],
  389 + "total": 3,
  390 + "success_count": 2,
  391 + "failed_count": 1
  392 +}
  393 +```
  394 +
  395 +| 字段 | 类型 | 说明 |
  396 +|------|------|------|
  397 +| `success` | array | 成功获取的SPU列表，每个元素包含 `spu_id` 和 `document`（完整的ES文档数据） |
  398 +| `failed` | array | 失败的SPU列表，每个元素包含 `spu_id` 和 `error`（失败原因） |
  399 +| `total` | integer | 总SPU数量 |
  400 +| `success_count` | integer | 成功数量 |
  401 +| `failed_count` | integer | 失败数量 |
  402 +
  403 +#### 请求示例
  404 +
  405 +**单个SPU查询**:
  406 +
  407 +```bash
  408 +curl -X POST "http://localhost:6004/indexer/documents" \
  409 + -H "Content-Type: application/json" \
  410 + -d '{
  411 + "tenant_id": "162",
  412 + "spu_ids": ["123"]
  413 + }'
  414 +```
  415 +
  416 +**批量SPU查询**:
  417 +
  418 +```bash
  419 +curl -X POST "http://localhost:6004/indexer/documents" \
  420 + -H "Content-Type: application/json" \
  421 + -d '{
  422 + "tenant_id": "162",
  423 + "spu_ids": ["123", "456", "789"]
  424 + }'
  425 +```
  426 +
  427 +#### 与 `/indexer/index` 的区别
  428 +
  429 +| 接口 | 功能 | 是否写入ES | 返回内容 |
  430 +|------|------|-----------|----------|
  431 +| `/indexer/documents` | 查询SPU文档数据 | 否 | 返回完整的ES文档数据 |
  432 +| `/indexer/index` | 增量索引 | 是 | 返回成功/失败列表和统计信息 |
  433 +
  434 +**使用场景**：
  435 +- `/indexer/documents`：用于查看、调试或验证SPU数据，不修改ES索引
  436 +- `/indexer/index`：用于实际的增量索引操作，将更新的SPU数据同步到ES
  437 +
  438 +### 5.5 索引健康检查接口
  439 +
  440 +- **端点**: `GET /indexer/health`
  441 +- **描述**: 检查索引服务健康状态（与 `api/routes/indexer.py` 中 `indexer_health_check` 一致）
  442 +
  443 +#### 响应格式
  444 +
  445 +```json
  446 +{
  447 + "status": "available",
  448 + "database": "connected",
  449 + "preloaded_data": {
  450 + "category_mappings": 150
  451 + }
  452 +}
  453 +```
  454 +
  455 +| 字段 | 类型 | 说明 |
  456 +|------|------|------|
  457 +| `status` | string | `available`（服务可用）、`unavailable`（未初始化）、`error`（异常） |
  458 +| `database` | string | 数据库连接状态，如 `connected` 或 `disconnected: ...` |
  459 +| `preloaded_data.category_mappings` | integer | 已加载的分类映射数量 |
  460 +
  461 +#### 请求示例
  462 +
  463 +```bash
  464 +curl -X GET "http://localhost:6004/indexer/health"
  465 +```
  466 +
  467 +### 5.6 文档构建接口（正式对接推荐）
  468 +
  469 +#### 5.6.1 `POST /indexer/build-docs`
  470 +
  471 +- **描述**:
  472 + 基于调用方（通常是 Java 索引程序）提供的 **MySQL 行数据** 构建 ES 文档（doc），**不写入 ES**。
  473 + 由本服务负责“如何构建 doc”（多语言、翻译、向量、规格聚合等），由调用方负责“何时调度 + 如何写 ES”。
  474 +
  475 +#### 请求参数
  476 +
  477 +```json
  478 +{
  479 + "tenant_id": "170",
  480 + "items": [
  481 + {
  482 + "spu": { "id": 223167, "tenant_id": 170, "title": "..." },
  483 + "skus": [
  484 + { "id": 3988393, "spu_id": 223167, "price": 25.99, "compare_at_price": 25.99 }
  485 + ],
  486 + "options": []
  487 + }
  488 + ]
  489 +}
  490 +```
  491 +
  492 +| 参数 | 类型 | 必填 | 说明 |
  493 +|------|------|------|------|
  494 +| `tenant_id` | string | Y | 租户 ID |
  495 +| `items` | array | Y | 需构建 doc 的 SPU 列表（每项含 `spu`、`skus`、`options`），**单次最多 200 条** |
  496 +
  497 +> `spu` / `skus` / `options` 字段应当直接使用从 `shoplazza_product_spu` / `shoplazza_product_sku` / `shoplazza_product_option` 查询出的行字段。
  498 +
  499 +#### 请求示例（完整 curl）
  500 +
  501 +> 完整请求体参考 `scripts/test_build_docs_api.py` 中的 `build_sample_request()`。
  502 +
  503 +```bash
  504 +# 单条 SPU 示例（含 spu、skus、options）
  505 +curl -X POST "http://localhost:6004/indexer/build-docs" \
  506 + -H "Content-Type: application/json" \
  507 + -d '{
  508 + "tenant_id": "162",
  509 + "items": [
  510 + {
  511 + "spu": {
  512 + "id": 10001,
  513 + "tenant_id": "162",
  514 + "title": "测试T恤 纯棉短袖",
  515 + "brief": "舒适纯棉，多色可选",
  516 + "description": "这是一款适合日常穿着的纯棉T恤，透气吸汗。",
  517 + "vendor": "测试品牌",
  518 + "category": "服装/上衣/T恤",
  519 + "category_id": 100,
  520 + "category_level": 2,
  521 + "category_path": "服装/上衣/T恤",
  522 + "fake_sales": 1280,
  523 + "image_src": "https://oss.essa.cn/98532128-cf8e-456c-9e30-6f2a5ea0c19f.jpg",
  524 + "tags": "T恤,纯棉,短袖,夏季",
  525 + "create_time": "2024-01-01T00:00:00Z",
  526 + "update_time": "2024-01-01T00:00:00Z"
  527 + },
  528 + "skus": [
  529 + {
  530 + "id": 20001,
  531 + "spu_id": 10001,
  532 + "price": 99.0,
  533 + "compare_at_price": 129.0,
  534 + "sku": "SKU-TSHIRT-001",
  535 + "inventory_quantity": 50,
  536 + "option1": "黑色",
  537 + "option2": "M",
  538 + "option3": null
  539 + },
  540 + {
  541 + "id": 20002,
  542 + "spu_id": 10001,
  543 + "price": 99.0,
  544 + "compare_at_price": 129.0,
  545 + "sku": "SKU-TSHIRT-002",
  546 + "inventory_quantity": 30,
  547 + "option1": "白色",
  548 + "option2": "L",
  549 + "option3": null
  550 + }
  551 + ],
  552 + "options": [
  553 + {"id": 1, "position": 1, "name": "颜色"},
  554 + {"id": 2, "position": 2, "name": "尺码"}
  555 + ]
  556 + }
  557 + ]
  558 +}'
  559 +```
  560 +
  561 +生产环境替换 `localhost:6004` 为实际 Indexer 地址，如 `http://43.166.252.75:6004`。
  562 +
  563 +#### 响应示例（节选）
  564 +
  565 +```json
  566 +{
  567 + "tenant_id": "170",
  568 + "docs": [
  569 + {
  570 + "tenant_id": "170",
  571 + "spu_id": "223167",
  572 + "title": { "en": "...", "zh": "..." },
  573 + "tags": ["Floerns", "Clothing", "Shoes & Jewelry"],
  574 + "skus": [
  575 + {
  576 + "sku_id": "3988393",
  577 + "price": 25.99,
  578 + "compare_at_price": 25.99,
  579 + "stock": 100
  580 + }
  581 + ],
  582 + "min_price": 25.99,
  583 + "max_price": 25.99,
  584 + "compare_at_price": 25.99,
  585 + "total_inventory": 100,
  586 + "title_embedding": [/* 1024 维向量 */]
  587 + // 其余字段与 mappings/search_products.json 一致
  588 + }
  589 + ],
  590 + "total": 1,
  591 + "success_count": 1,
  592 + "failed_count": 0,
  593 + "failed": []
  594 +}
  595 +```
  596 +
  597 +| 字段 | 类型 | 说明 |
  598 +|------|------|------|
  599 +| `tenant_id` | string | 租户 ID |
  600 +| `docs` | array | 构建成功的 ES 文档列表，与 `mappings/search_products.json` 一致 |
  601 +| `total` | integer | 请求的 items 总数 |
  602 +| `success_count` | integer | 成功构建数量 |
  603 +| `failed_count` | integer | 失败数量 |
  604 +| `failed` | array | 失败项列表，每项含 `spu_id`、`error` |
  605 +
  606 +#### 使用建议
  607 +
  608 +- **生产环境推荐流程**：
  609 + 1. Java 根据业务逻辑决定哪些 SPU 需要（全量/增量）处理；
  610 + 2. Java 从 MySQL 查询 SPU/SKU/Option 行，拼成 `items`；
  611 + 3. 调用 `/indexer/build-docs` 获取 ES-ready `docs`；
  612 + 4. Java 使用自己的 ES 客户端写入 `search_products_tenant_{tenant_id}`。
  613 +
  614 +### 5.7 文档构建接口（测试 / 自测）
  615 +
  616 +#### 5.7.1 `POST /indexer/build-docs-from-db`
  617 +
  618 +- **描述**:
  619 + 仅用于测试/调试：调用方只提供 `tenant_id` 和 `spu_ids`，由 indexer 服务内部从 MySQL 查询 SPU/SKU/Option，然后调用与 `/indexer/build-docs` 相同的文档构建逻辑，返回 ES-ready doc。**生产环境请使用 `/indexer/build-docs`，由上游查库并写 ES。**
  620 +
  621 +#### 请求参数
  622 +
  623 +```json
  624 +{
  625 + "tenant_id": "170",
  626 + "spu_ids": ["223167", "223168"]
  627 +}
  628 +```
  629 +
  630 +| 参数 | 类型 | 必填 | 说明 |
  631 +|------|------|------|------|
  632 +| `tenant_id` | string | Y | 租户 ID |
  633 +| `spu_ids` | array[string] | Y | SPU ID 列表，**单次最多 200 个** |
  634 +
  635 +#### 响应格式
  636 +
  637 +与 `/indexer/build-docs` 相同：`tenant_id`、`docs`、`total`、`success_count`、`failed_count`、`failed`。
  638 +
  639 +#### 请求示例
  640 +
  641 +```bash
  642 +curl -X POST "http://127.0.0.1:6004/indexer/build-docs-from-db" \
  643 + -H "Content-Type: application/json" \
  644 + -d '{"tenant_id": "170", "spu_ids": ["223167"]}'
  645 +```
  646 +
  647 +返回结构与 `/indexer/build-docs` 相同，可直接用于对比 ES 实际文档或调试字段映射问题。
  648 +
  649 +### 5.8 内容理解字段生成接口
  650 +
  651 +- **端点**: `POST /indexer/enrich-content`
  652 +- **描述**: 根据商品内容信息批量生成 **qanchors**（锚文本）、**semantic_attributes**（语义属性）、**tags**（细分标签），供外部 indexer 在「微服务组合」方式下自行拼装 doc 时使用。请求以 `items[]` 传入商品内容字段（必填/可选见下表）。内部逻辑与 `indexer.product_enrich` 一致，支持多语言与 Redis 缓存；单次请求在线程池中执行，避免阻塞其他接口。
  653 +
  654 +#### 请求参数
  655 +
  656 +```json
  657 +{
  658 + "tenant_id": "170",
  659 + "items": [
  660 + {
  661 + "spu_id": "223167",
  662 + "title": "纯棉短袖T恤 夏季男装",
  663 + "brief": "夏季透气纯棉短袖，舒适亲肤",
  664 + "description": "100%棉，圆领版型，适合日常通勤与休闲穿搭。",
  665 + "image_url": "https://example.com/images/223167.jpg"
  666 + },
  667 + {
  668 + "spu_id": "223168",
  669 + "title": "12PCS Dolls with Bottles",
  670 + "image_url": "https://example.com/images/223168.jpg"
  671 + }
  672 + ],
  673 + "languages": ["zh", "en"]
  674 +}
  675 +```
  676 +
  677 +| 参数 | 类型 | 必填 | 默认值 | 说明 |
  678 +|------|------|------|--------|------|
  679 +| `tenant_id` | string | Y | - | 租户 ID。目前仅用于记录日志，不产生实际作用|
  680 +| `items` | array | Y | - | 待分析列表；**单次最多 50 条** |
  681 +| `languages` | array[string] | N | `["zh", "en"]` | 目标语言，需在支持范围内：`zh`、`en`、`de`、`ru`、`fr` |
  682 +
  683 +`items[]` 字段说明：
  684 +
  685 +| 字段 | 类型 | 必填 | 说明 |
  686 +|------|------|------|------|
  687 +| `spu_id` | string | Y | SPU ID，用于回填结果；目前仅用于记录日志，不产生实际作用|
  688 +| `title` | string | Y | 商品标题 |
  689 +| `image_url` | string | N | 商品主图 URL；当前会参与内容缓存键，后续可用于图像/多模态内容理解 |
  690 +| `brief` | string | N | 商品简介/短描述；当前会参与内容缓存键 |
  691 +| `description` | string | N | 商品详情/长描述；当前会参与内容缓存键 |
  692 +
  693 +缓存说明：
  694 +
  695 +- 内容缓存键仅由 `target_lang + items[]` 中会影响内容理解结果的输入文本构成，目前包括：`title`、`brief`、`description`、`image_url` 的规范化内容 hash。
  696 +- `tenant_id`、`spu_id` 只用于请求归属与结果回填，不参与缓存键。
  697 +- 因此，输入内容不变时可跨请求直接命中缓存；任一输入字段变化时，会自然落到新的缓存 key。
  698 +
  699 +批量请求建议：
  700 +- **全量**：强烈建议 尽可能 **20 个 SPU/doc** 攒成一个批次后再请求一次。
  701 +- **增量**：可按时效要求设置时间窗口（例如 **5 分钟**），在窗口内尽可能攒到 **20 个**；达到 20 或窗口到期就发送一次请求。
  702 +- 允许超过20，服务内部会拆分成小批次逐个处理。也允许小于20，但是将造成费用和耗时的成本上升，特别是每次请求一个doc的情况。
  703 +
  704 +#### 响应格式
  705 +
  706 +```json
  707 +{
  708 + "tenant_id": "170",
  709 + "total": 2,
  710 + "results": [
  711 + {
  712 + "spu_id": "223167",
  713 + "qanchors": {
  714 + "zh": "短袖T恤,纯棉,男装,夏季",
  715 + "en": "cotton t-shirt, short sleeve, men, summer"
  716 + },
  717 + "semantic_attributes": [
  718 + { "lang": "zh", "name": "tags", "value": "纯棉" },
  719 + { "lang": "zh", "name": "usage_scene", "value": "日常" },
  720 + { "lang": "en", "name": "tags", "value": "cotton" }
  721 + ],
  722 + "tags": ["纯棉", "短袖", "男装", "cotton", "short sleeve"]
  723 + },
  724 + {
  725 + "spu_id": "223168",
  726 + "qanchors": { "en": "dolls, toys, 12pcs" },
  727 + "semantic_attributes": [],
  728 + "tags": ["dolls", "toys"]
  729 + }
  730 + ]
  731 +}
  732 +```
  733 +
  734 +| 字段 | 类型 | 说明 |
  735 +|------|------|------|
  736 +| `results` | array | 与请求 `items` 一一对应，每项含 `spu_id`、`qanchors`、`semantic_attributes`、`tags` |
  737 +| `results[].qanchors` | object | 按语言键的锚文本（逗号分隔短语），可写入 ES 文档的 `qanchors.{lang}` |
  738 +| `results[].semantic_attributes` | array | 语义属性列表，每项为 `{ "lang", "name", "value" }`，可写入 ES 的 `semantic_attributes` nested 字段 |
  739 +| `results[].tags` | array | 从语义属性中抽取的 `name=tags` 的 value 集合，可与业务原有 `tags` 合并后写入 ES 的 `tags` 字段 |
  740 +| `results[].error` | string | 若该条处理失败（如 LLM 异常），会在此字段返回错误信息 |
  741 +
  742 +**错误响应**:
  743 +- `400`: `items` 为空或超过 50 条
  744 +- `503`: 未配置 `DASHSCOPE_API_KEY`，内容理解服务不可用
  745 +
  746 +#### 请求示例
  747 +
  748 +```bash
  749 +curl -X POST "http://localhost:6004/indexer/enrich-content" \
  750 + -H "Content-Type: application/json" \
  751 + -d '{
  752 + "tenant_id": "170",
  753 + "items": [
  754 + {
  755 + "spu_id": "223167",
  756 + "title": "纯棉短袖T恤 夏季男装",
  757 + "brief": "夏季透气纯棉短袖，舒适亲肤",
  758 + "description": "100%棉，圆领版型，适合日常通勤与休闲穿搭。",
  759 + "image_url": "https://example.com/images/223167.jpg"
  760 + }
  761 + ],
  762 + "languages": ["zh", "en"]
  763 + }'
  764 +```
  765 +
  766 +---
  767 +
... ...
docs/搜索API对接指南-06-管理接口（Admin）.md 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,53 @@
  1 +# 搜索API对接指南-06-管理接口（Admin）
  2 +
  3 +用于查看服务健康状态、获取租户配置与索引统计信息（原文第 6 章）。
  4 +
  5 +## 管理接口
  6 +
  7 +### 6.1 健康检查
  8 +
  9 +- **端点**: `GET /admin/health`
  10 +- **描述**: 检查服务与依赖（如 Elasticsearch）状态。
  11 +
  12 +```json
  13 +{
  14 + "status": "healthy",
  15 + "elasticsearch": "connected",
  16 + "tenant_id": "tenant1"
  17 +}
  18 +```
  19 +
  20 +### 6.2 获取配置
  21 +
  22 +- **端点**: `GET /admin/config`
  23 +- **描述**: 返回当前租户的脱敏配置，便于核对索引及排序表达式。
  24 +
  25 +```json
  26 +{
  27 + "tenant_id": "tenant1",
  28 + "tenant_name": "Tenant1 Test Instance",
  29 + "es_index_name": "search_tenant1",
  30 + "num_fields": 20,
  31 + "num_indexes": 4,
  32 + "supported_languages": ["zh", "en", "ru"],
  33 + "spu_enabled": false
  34 +}
  35 +```
  36 +
  37 +### 6.3 索引统计
  38 +
  39 +- **端点**: `GET /admin/stats`
  40 +- **描述**: 获取指定租户索引文档数量与磁盘大小，方便监控。
  41 +- **租户标识**：通过请求头 `X-Tenant-ID` 或 query 参数 `tenant_id` 传递（必填）。
  42 +
  43 +```json
  44 +{
  45 + "tenant_id": "162",
  46 + "index_name": "search_products_tenant_162",
  47 + "document_count": 10000,
  48 + "size_mb": 523.45
  49 +}
  50 +```
  51 +
  52 +---
  53 +
... ...
docs/搜索API对接指南-07-微服务接口（Embedding-Reranker-Translation）.md 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,401 @@
  1 +# 搜索API对接指南-07-微服务接口（Embedding-Reranker-Translation）
  2 +
  3 +本篇覆盖向量服务（Embedding）、重排服务（Reranker）、翻译服务（Translation）以及 Indexer 服务内的内容理解字段生成（原文第 7 章）。
  4 +
  5 +## 7. 微服务接口（向量、重排、翻译）
  6 +
  7 +以下三个微服务独立部署，**外部系统可直接调用**。它们被搜索后端（6002）和索引服务（6004）内部使用，也可供其他业务系统直接对接。
  8 +
  9 +| 服务 | 默认端口 | Base URL | 说明 |
  10 +|------|----------|----------|------|
  11 +| 向量服务（文本） | 6005 | `http://localhost:6005` | 文本向量化，用于 query/doc 语义检索 |
  12 +| 向量服务（图片） | 6008 | `http://localhost:6008` | 图片向量化，用于以图搜图 |
  13 +| 翻译服务 | 6006 | `http://localhost:6006` | 多语言翻译（云端与本地模型统一入口） |
  14 +| 重排服务 | 6007 | `http://localhost:6007` | 对检索结果进行二次排序 |
  15 +
  16 +生产环境请将 `localhost` 替换为实际服务地址。
  17 +服务管理入口与完整启停规则见：`docs/Usage-Guide.md` -> `服务管理总览`。
  18 +
  19 +### 7.1 向量服务（Embedding）
  20 +
  21 +- **Base URL**:
  22 + - 文本：`http://localhost:6005`（可通过 `EMBEDDING_TEXT_SERVICE_URL` 覆盖）
  23 + - 图片：`http://localhost:6008`（可通过 `EMBEDDING_IMAGE_SERVICE_URL` 覆盖）
  24 +- **启动**:
  25 + - 文本：`./scripts/start_embedding_text_service.sh`
  26 + - 图片：`./scripts/start_embedding_image_service.sh`
  27 +- **依赖**:
  28 + - 文本向量后端默认走 TEI（`http://127.0.0.1:8080`）
  29 + - 图片向量依赖 `cnclip`（`grpc://127.0.0.1:51000`）
  30 + - TEI 默认使用 GPU（`TEI_DEVICE=cuda`）；当配置为 GPU 且不可用时会启动失败（不会自动降级到 CPU）
  31 + - cnclip 默认使用 `cuda`；若配置为 `cuda` 但 GPU 不可用会启动失败（不会自动降级到 `cpu`）
  32 + - 当前单机部署建议保持单实例，通过**文本/图片拆分 + 独立限流**隔离压力
  33 +
  34 +补充说明：
  35 +
  36 +- 文本和图片现在已经拆成**不同进程 / 不同端口**，避免图片下载与编码波动影响文本向量化。
  37 +- 服务端对 text / image 有**独立 admission control**：
  38 + - `TEXT_MAX_INFLIGHT`
  39 + - `IMAGE_MAX_INFLIGHT`
  40 +- 当超过处理能力时，服务会直接返回过载错误，而不是无限排队。
  41 +- `GET /health` 会返回各自的 `limits`、`stats`、`cache_enabled` 等状态；`GET /ready` 用于就绪探针。
  42 +
  43 +#### 7.1.1 `POST /embed/text` — 文本向量化
  44 +
  45 +将文本列表转为 1024 维向量，用于语义搜索、文档索引等。
  46 +
  47 +**请求体**（JSON 数组）:
  48 +
  49 +```json
  50 +["文本1", "文本2", "文本3"]
  51 +```
  52 +
  53 +**响应**（JSON 数组，与输入一一对应）:
  54 +
  55 +```json
  56 +[[0.01, -0.02, ...], [0.03, 0.01, ...], ...]
  57 +```
  58 +
  59 +**完整 curl 示例**:
  60 +
  61 +```bash
  62 +curl -X POST "http://localhost:6005/embed/text?normalize=true" \
  63 + -H "Content-Type: application/json" \
  64 + -d '["芭比娃娃 儿童玩具", "纯棉T恤 短袖"]'
  65 +```
  66 +
  67 +#### 7.1.2 `POST /embed/image` — 图片向量化
  68 +
  69 +将图片 URL 或路径转为向量，用于以图搜图。
  70 +
  71 +前置条件：`cnclip` 服务已启动（默认端口 `51000`）。若未启动，图片 embedding 服务启动会失败或请求返回错误。
  72 +
  73 +**请求体**（JSON 数组）:
  74 +
  75 +```json
  76 +["https://example.com/image1.jpg", "https://example.com/image2.jpg"]
  77 +```
  78 +
  79 +**响应**（JSON 数组，与输入一一对应）:
  80 +
  81 +```json
  82 +[[0.01, -0.02, ...], [0.03, 0.01, ...], ...]
  83 +```
  84 +
  85 +**完整 curl 示例**:
  86 +
  87 +```bash
  88 +curl -X POST "http://localhost:6008/embed/image?normalize=true" \
  89 + -H "Content-Type: application/json" \
  90 + -d '["https://oss.essa.cn/98532128-cf8e-456c-9e30-6f2a5ea0c19f.jpg"]'
  91 +```
  92 +
  93 +#### 7.1.3 `GET /health` — 健康检查
  94 +
  95 +```bash
  96 +curl "http://localhost:6005/health"
  97 +curl "http://localhost:6008/health"
  98 +```
  99 +
  100 +返回中会包含：
  101 +
  102 +- `service_kind`：`text` / `image` / `all`
  103 +- `cache_enabled`：text/image Redis 缓存是否可用
  104 +- `limits`：当前 inflight limit、active、rejected_total 等
  105 +- `stats`：request_total、cache_hits、cache_misses、avg_latency_ms 等
  106 +
  107 +#### 7.1.4 `GET /ready` — 就绪检查
  108 +
  109 +```bash
  110 +curl "http://localhost:6005/ready"
  111 +curl "http://localhost:6008/ready"
  112 +```
  113 +
  114 +#### 7.1.5 缓存与限流说明
  115 +
  116 +- 文本与图片都会先查 Redis 向量缓存。
  117 +- Redis 中 value 仍是 **BF16 bytes**，读取后恢复成 `float32` 返回。
  118 +- cache key 已区分 `normalize=true/false`，避免不同归一化策略命中同一条缓存。
  119 +- 当服务端发现请求是 **full-cache-hit** 时，会直接返回，不占用模型并发槽位。
  120 +- 当服务端发现超过 `TEXT_MAX_INFLIGHT` / `IMAGE_MAX_INFLIGHT` 时，会直接拒绝，而不是无限排队。
  121 +
  122 +#### 7.1.6 TEI 统一调优建议（主服务）
  123 +
  124 +使用单套主服务即可同时兼顾：
  125 +- 在线 query 向量化（低延迟，常见 `batch=1~4`）
  126 +- 索引构建向量化（高吞吐，常见 `batch=15~20`）
  127 +
  128 +统一启动（主链路）：
  129 +
  130 +```bash
  131 +./scripts/start_tei_service.sh
  132 +./scripts/service_ctl.sh restart embedding
  133 +```
  134 +
  135 +默认端口：
  136 +- TEI: `http://127.0.0.1:8080`
  137 +- 文本向量服务（`/embed/text`）: `http://127.0.0.1:6005`
  138 +- 图片向量服务（`/embed/image`）: `http://127.0.0.1:6008`
  139 +
  140 +当前主 TEI 启动默认值（已按 T4/短文本场景调优）：
  141 +- `TEI_MAX_BATCH_TOKENS=4096`
  142 +- `TEI_MAX_CLIENT_BATCH_SIZE=24`
  143 +- `TEI_DTYPE=float16`
  144 +
  145 +### 7.2 重排服务（Reranker）
  146 +
  147 +- **Base URL**: `http://localhost:6007`（可通过 `RERANKER_SERVICE_URL` 覆盖）
  148 +- **启动**: `./scripts/start_reranker.sh`
  149 +
  150 +说明：默认后端为 `qwen3_vllm`（`Qwen/Qwen3-Reranker-0.6B`），需要可用 GPU 显存。
  151 +
  152 +补充：`docs` 的请求大小与模型推理 `batch size` 解耦。即使一次传入 1000 条文档，服务端也会按 `services.rerank.backends.qwen3_vllm.infer_batch_size` 自动拆分；若 `sort_by_doc_length=true`，会先按文档长度排序后分批，减少 padding，再按原输入顺序返回分数。`length_sort_mode` 可选 `char`（更快）或 `token`（更精确）。
  153 +
  154 +#### 7.2.1 `POST /rerank` — 结果重排
  155 +
  156 +根据 query 与 doc 的相关性对文档列表重新打分排序。
  157 +
  158 +**请求体**:
  159 +```json
  160 +{
  161 + "query": "玩具 芭比",
  162 + "docs": [
  163 + "12PCS 6 Types of Dolls with Bottles",
  164 + "纯棉T恤 短袖 夏季"
  165 + ],
  166 + "normalize": true
  167 +}
  168 +```
  169 +
  170 +| 参数 | 类型 | 必填 | 说明 |
  171 +|------|------|------|------|
  172 +| `query` | string | Y | 搜索查询 |
  173 +| `docs` | array[string] | Y | 待重排的文档列表（单次最多由服务端配置限制） |
  174 +| `normalize` | boolean | N | 是否对分数做 sigmoid 归一化，默认 true |
  175 +
  176 +**响应**:
  177 +```json
  178 +{
  179 + "scores": [0.92, 0.15],
  180 + "meta": {
  181 + "service_elapsed_ms": 45.2,
  182 + "input_docs": 2,
  183 + "unique_docs": 2
  184 + }
  185 +}
  186 +```
  187 +
  188 +**完整 curl 示例**:
  189 +```bash
  190 +curl -X POST "http://localhost:6007/rerank" \
  191 + -H "Content-Type: application/json" \
  192 + -d '{
  193 + "query": "玩具 芭比",
  194 + "docs": ["12PCS 6 Types of Dolls with Bottles", "纯棉T恤 短袖"],
  195 + "top_n":386,
  196 + "normalize": true
  197 + }'
  198 +```
  199 +
  200 +#### 7.2.2 `GET /health` — 健康检查
  201 +
  202 +```bash
  203 +curl "http://localhost:6007/health"
  204 +```
  205 +
  206 +### 7.3 翻译服务（Translation）
  207 +
  208 +- **Base URL**: `http://localhost:6006`（以 `config/config.yaml -> services.translation.service_url` 为准）
  209 +- **启动**: `./scripts/start_translator.sh`
  210 +
  211 +#### 7.3.1 `POST /translate` — 文本翻译
  212 +
  213 +支持 translator service 内所有已启用 capability，适用于商品名称、描述、query 等电商场景。当前可配置能力包括 `qwen-mt`、`llm`、`deepl` 以及本地模型 `nllb-200-distilled-600m`、`opus-mt-zh-en`、`opus-mt-en-zh`。
  214 +
  215 +**请求体**（支持单条字符串或字符串列表）:
  216 +```json
  217 +{
  218 + "text": "商品名称",
  219 + "target_lang": "en",
  220 + "source_lang": "zh",
  221 + "model": "qwen-mt",
  222 + "scene": "sku_name"
  223 +}
  224 +```
  225 +
  226 +也支持批量列表形式:
  227 +```json
  228 +{
  229 + "text": ["商品名称1", "商品名称2"],
  230 + "target_lang": "en",
  231 + "source_lang": "zh",
  232 + "model": "qwen-mt",
  233 + "scene": "sku_name"
  234 +}
  235 +```
  236 +
  237 +| 参数 | 类型 | 必填 | 说明 |
  238 +|------|------|------|------|
  239 +| `text` | string \| string[] | Y | 待翻译文本，既支持单条字符串，也支持字符串列表（批量翻译） |
  240 +| `target_lang` | string | Y | 目标语言：`zh`、`en`、`ru` 等 |
  241 +| `source_lang` | string | N | 源语言。云端模型可不传；`nllb-200-distilled-600m` 建议显式传入 |
  242 +| `model` | string | N | 已启用 capability 名称，如 `qwen-mt`、`llm`、`deepl`、`nllb-200-distilled-600m`、`opus-mt-zh-en`、`opus-mt-en-zh` |
  243 +| `scene` | string | N | 翻译场景参数，与 `model` 配套使用；当前标准值为 `sku_name`、`ecommerce_search_query`、`general` |
  244 +
  245 +说明：
  246 +- 外部接口不接受 `prompt`；LLM prompt 由服务端按 `scene` 自动生成。
  247 +- 传入未定义的 `scene` 或未启用的 `model` 会返回 `400`。
  248 +
  249 +**SKU 名称场景选型建议**:
  250 +- 批量 SKU 名称翻译，优先考虑本地大吞吐方案时，可使用 `"model": "nllb-200-distilled-600m"`（该模型"scene":参数无效）。
  251 +- 如果目标是更高质量，且可以接受更慢速度与额外 LLM API 费用，可使用 `"model": "llm"` + `"scene": "sku_name"`。
  252 +- 如果是en-zh互译、期待更高的速度，可以考虑`opus-mt-zh-en` / `opus-mt-en-zh`。（质量未详细评测，一些文章说比blib-200-600m更好，但是我看了些case感觉要差不少）
  253 +
  254 +**实时翻译选型建议**:
  255 +- 在线 query 翻译如果只是 `en/zh` 互译，优先使用 `opus-mt-zh-en` 或 `opus-mt-en-zh`，它们是当前已测本地模型里延迟最低的一档。
  256 +- 如果涉及其他语言，或对质量要求高于本地轻量模型，优先考虑 `deepl`。
  257 +- `nllb-200-distilled-600m` 不建议作为在线 query 翻译默认方案；我们在 `Tesla T4` 上测到 `batch_size=1` 时，`zh -> en` p50 约 `292.54 ms`、p95 约 `624.12 ms`，`en -> zh` p50 约 `481.61 ms`、p95 约 `1171.71 ms`。
  258 +
  259 +**Batch Size / 调用方式建议**:
  260 +- 本接口支持 `text: string[]`；离线或批量索引翻译时，应尽量合并请求，让底层 backend 发挥批处理能力。
  261 +- `nllb-200-distilled-600m` 在当前 `Tesla T4` 压测中，推荐配置是 `batch_size=16`、`max_new_tokens=64`、`attn_implementation=sdpa`；继续升到 `batch_size=32` 虽可能提高吞吐，但 tail latency 会明显变差。
  262 +- 在线 query 场景可直接把“单条请求”理解为 `batch_size=1`；更关注 request latency，而不是离线吞吐。
  263 +- `opus-mt-zh-en` / `opus-mt-en-zh` 当前生产配置也是 `batch_size=16`，适合作为中英互译的低延迟本地默认值；若走在线单条调用，同样按 `batch_size=1` 理解即可。
  264 +- `llm` 按单条请求即可。
  265 +
  266 +**响应**:
  267 +```json
  268 +{
  269 + "text": "商品名称",
  270 + "target_lang": "en",
  271 + "source_lang": "zh",
  272 + "translated_text": "Product name",
  273 + "status": "success",
  274 + "model": "qwen-mt",
  275 + "scene": "sku_name"
  276 +}
  277 +```
  278 +
  279 +当请求为列表形式时，`text` 与 `translated_text` 均为等长数组:
  280 +```json
  281 +{
  282 + "text": ["商品名称1", "商品名称2"],
  283 + "target_lang": "en",
  284 + "source_lang": "zh",
  285 + "translated_text": ["Product name 1", "Product name 2"],
  286 + "status": "success",
  287 + "model": "qwen-mt",
  288 + "scene": "sku_name"
  289 +}
  290 +```
  291 +
  292 +> **失败语义（批量）**：当 `text` 为列表时，如果其中某条翻译失败，对应位置返回 `null`（即 `translated_text[i] = null`），并保持数组长度与顺序不变；接口整体仍返回 `status="success"`，用于避免“部分失败”导致整批请求失败。
  293 +
  294 +> **实现提示（可忽略）**：服务端会尽可能使用底层 backend 的批量能力（若支持），否则自动拆分逐条翻译；无论采用哪种方式，上述批量契约保持一致。
  295 +
  296 +**完整 curl 示例**:
  297 +
  298 +中文 → 英文:
  299 +```bash
  300 +curl -X POST "http://localhost:6006/translate" \
  301 + -H "Content-Type: application/json" \
  302 + -d '{
  303 + "text": "商品名称",
  304 + "target_lang": "en",
  305 + "source_lang": "zh"
  306 + }'
  307 +```
  308 +
  309 +俄文 → 英文:
  310 +```bash
  311 +curl -X POST "http://localhost:6006/translate" \
  312 + -H "Content-Type: application/json" \
  313 + -d '{
  314 + "text": "Название товара",
  315 + "target_lang": "en",
  316 + "source_lang": "ru"
  317 + }'
  318 +```
  319 +
  320 +使用 DeepL 模型:
  321 +```bash
  322 +curl -X POST "http://localhost:6006/translate" \
  323 + -H "Content-Type: application/json" \
  324 + -d '{
  325 + "text": "商品名称",
  326 + "target_lang": "en",
  327 + "source_lang": "zh",
  328 + "model": "deepl"
  329 + }'
  330 +```
  331 +
  332 +使用本地 OPUS 模型（中文 → 英文）:
  333 +```bash
  334 +curl -X POST "http://localhost:6006/translate" \
  335 + -H "Content-Type: application/json" \
  336 + -d '{
  337 + "text": "蓝牙耳机",
  338 + "target_lang": "en",
  339 + "source_lang": "zh",
  340 + "model": "opus-mt-zh-en",
  341 + "scene": "sku_name"
  342 + }'
  343 +```
  344 +
  345 +使用本地 NLLB 做 SKU 名称批量翻译:
  346 +```bash
  347 +curl -X POST "http://localhost:6006/translate" \
  348 + -H "Content-Type: application/json" \
  349 + -d '{
  350 + "text": ["商品名称1", "商品名称2", "商品名称3"],
  351 + "target_lang": "en",
  352 + "source_lang": "zh",
  353 + "model": "nllb-200-distilled-600m",
  354 + "scene": "sku_name"
  355 + }'
  356 +```
  357 +
  358 +使用 LLM 做高质量 SKU 名称翻译:
  359 +```bash
  360 +curl -X POST "http://localhost:6006/translate" \
  361 + -H "Content-Type: application/json" \
  362 + -d '{
  363 + "text": "男士偏光飞行员太阳镜",
  364 + "target_lang": "en",
  365 + "source_lang": "zh",
  366 + "model": "llm",
  367 + "scene": "sku_name"
  368 + }'
  369 +```
  370 +
  371 +#### 7.3.2 `GET /health` — 健康检查
  372 +
  373 +```bash
  374 +curl "http://localhost:6006/health"
  375 +```
  376 +
  377 +典型响应：
  378 +```json
  379 +{
  380 + "status": "healthy",
  381 + "service": "translation",
  382 + "default_model": "llm",
  383 + "default_scene": "general",
  384 + "available_models": ["qwen-mt", "llm", "opus-mt-zh-en"],
  385 + "enabled_capabilities": ["qwen-mt", "llm", "opus-mt-zh-en"],
  386 + "loaded_models": ["llm"]
  387 +}
  388 +```
  389 +
  390 +### 7.4 内容理解字段生成（Indexer 服务内）
  391 +
  392 +内容理解字段生成接口部署在 **Indexer 服务**（默认端口 6004）内，与「翻译、向量化」等独立端口微服务并列，供采用**微服务组合**方式的 indexer 调用。
  393 +
  394 +- **Base URL**: Indexer 服务地址，如 `http://localhost:6004`
  395 +- **路径**: `POST /indexer/enrich-content`
  396 +- **说明**: 根据商品标题批量生成 `qanchors`、`semantic_attributes`、`tags`，用于拼装 ES 文档。内部使用大模型（需配置 `DASHSCOPE_API_KEY`），支持多语言与 Redis 缓存；单次最多 50 条，建议批量调用以提升效率。
  397 +
  398 +请求/响应格式、示例及错误码见 [-05-索引接口（Indexer）](./搜索API对接指南-05-索引接口（Indexer）.md#58-内容理解字段生成接口)。
  399 +
  400 +---
  401 +
... ...
docs/搜索API对接指南-08-数据模型与字段速查.md 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,97 @@
  1 +# 搜索API对接指南-08-数据模型与字段速查
  2 +
  3 +本篇覆盖原文第 9 章：商品字段定义、字段类型速查、常用字段列表、支持的分析器。
  4 +
  5 +## 9. 数据模型
  6 +
  7 +### 9.1 商品字段定义
  8 +
  9 +| 字段名 | 类型 | 描述 |
  10 +|--------|------|------|
  11 +| `tenant_id` | keyword | 租户ID（多租户隔离） |
  12 +| `spu_id` | keyword | SPU ID |
  13 +| `title.<lang>` | object/text | 商品标题（多语言对象，如 `title.zh`, `title.en`） |
  14 +| `brief.<lang>` | object/text | 商品短描述（多语言对象，如 `brief.zh`, `brief.en`） |
  15 +| `description.<lang>` | object/text | 商品详细描述（多语言对象，如 `description.zh`, `description.en`） |
  16 +| `vendor.<lang>` | object/text | 供应商/品牌（多语言对象，且带 keyword 子字段，如 `vendor.zh.keyword`） |
  17 +| `category_path.<lang>` | object/text | 类目路径（多语言对象，用于搜索，如 `category_path.zh`） |
  18 +| `category_name_text.<lang>` | object/text | 类目名称（多语言对象，用于搜索，如 `category_name_text.zh`） |
  19 +| `category_id` | keyword | 类目ID |
  20 +| `category_name` | keyword | 类目名称（用于过滤） |
  21 +| `category_level` | integer | 类目层级 |
  22 +| `category1_name`, `category2_name`, `category3_name` | keyword | 多级类目名称（用于过滤和分面） |
  23 +| `tags` | keyword | 标签（数组） |
  24 +| `specifications` | nested | 规格（嵌套对象数组） |
  25 +| `option1_name`, `option2_name`, `option3_name` | keyword | 选项名称 |
  26 +| `min_price`, `max_price` | float | 最低/最高价格 |
  27 +| `compare_at_price` | float | 原价 |
  28 +| `sku_prices` | float | SKU价格列表（数组） |
  29 +| `sku_weights` | long | SKU重量列表（数组） |
  30 +| `sku_weight_units` | keyword | SKU重量单位列表（数组） |
  31 +| `total_inventory` | long | 总库存 |
  32 +| `sales` | long | 销量（展示销量） |
  33 +| `skus` | nested | SKU详细信息（嵌套对象数组） |
  34 +| `create_time`, `update_time` | date | 创建/更新时间 |
  35 +| `title_embedding` | dense_vector | 标题向量（1024维，仅用于搜索） |
  36 +| `image_embedding` | nested | 图片向量（嵌套，仅用于搜索） |
  37 +
  38 +> 所有租户共享统一的索引结构。文本字段支持中英文双语，后端根据 `language` 参数自动选择对应字段返回。
  39 +
  40 +### 9.2 字段类型速查
  41 +
  42 +| 类型 | ES Mapping | 用途 |
  43 +|------|------------|------|
  44 +| `text` | `text` | 全文检索（支持中英文分析器） |
  45 +| `keyword` | `keyword` | 精确匹配、聚合、排序 |
  46 +| `integer` | `integer` | 整数 |
  47 +| `long` | `long` | 长整数 |
  48 +| `float` | `float` | 浮点数 |
  49 +| `date` | `date` | 日期时间 |
  50 +| `nested` | `nested` | 嵌套对象（specifications, skus, image_embedding） |
  51 +| `dense_vector` | `dense_vector` | 向量字段（title_embedding，仅用于搜索） |
  52 +
  53 +### 9.3 常用字段列表
  54 +
  55 +#### 过滤字段
  56 +
  57 +- `category_name`: 类目名称
  58 +- `category1_name`, `category2_name`, `category3_name`: 多级类目
  59 +- `category_id`: 类目ID
  60 +- `vendor.zh.keyword`, `vendor.en.keyword`: 供应商/品牌（使用keyword子字段）
  61 +- `tags`: 标签（keyword类型）
  62 +- `option1_name`, `option2_name`, `option3_name`: 选项名称
  63 +- `specifications`: 规格过滤（嵌套字段，格式见[过滤器详解](./搜索API对接指南-01-搜索接口.md#33-过滤器详解)）
  64 +
  65 +#### 范围字段
  66 +
  67 +- `min_price`: 最低价格
  68 +- `max_price`: 最高价格
  69 +- `compare_at_price`: 原价
  70 +- `create_time`: 创建时间
  71 +- `update_time`: 更新时间
  72 +
  73 +#### 排序字段
  74 +
  75 +- `price`: 价格（后端自动根据sort_order映射：asc→min_price，desc→max_price）
  76 +- `sales`: 销量
  77 +- `create_time`: 创建时间
  78 +- `update_time`: 更新时间
  79 +- `relevance_score`: 相关性分数（默认，不指定sort_by时使用）
  80 +
  81 +**注意**: 前端只需传 `price`，后端会自动处理：
  82 +- `sort_by: "price"` + `sort_order: "asc"` → 按 `min_price` 升序（价格从低到高）
  83 +- `sort_by: "price"` + `sort_order: "desc"` → 按 `max_price` 降序（价格从高到低）
  84 +
  85 +### 9.4 支持的分析器
  86 +
  87 +| 分析器 | 语言 | 描述 |
  88 +|--------|------|------|
  89 +| `index_ik` | 中文 | 中文索引分析器（用于中文字段） |
  90 +| `query_ik` | 中文 | 中文查询分析器（用于中文字段） |
  91 +| `hanlp_index` ⚠️ TODO（暂不支持） | 中文 | 中文索引分析器（用于中文字段） |
  92 +| `hanlp_standard` ⚠️ TODO（暂不支持） | 中文 | 中文查询分析器（用于中文字段） |
  93 +| `english` | 英文 | 标准英文分析器（用于英文字段） |
  94 +| `lowercase` | - | 小写标准化器（用于keyword子字段） |
  95 +
  96 +---
  97 +
... ...
docs/搜索API对接指南-10-接口级压测脚本.md 0 → 100644
  Show/Hide comments   View file @0342d89
... ... @@ -0,0 +1,61 @@
  1 +# 搜索API对接指南-10-接口级压测脚本
  2 +
  3 +原文第 10 章：压测脚本与用例。
  4 +
  5 +## 10. 接口级压测脚本
  6 +
  7 +仓库提供统一压测脚本：`scripts/perf_api_benchmark.py`，用于对以下接口做并发压测：
  8 +
  9 +- 后端搜索：`POST /search/`
  10 +- 搜索建议：`GET /search/suggestions`
  11 +- 向量服务：`POST /embed/text`
  12 +- 翻译服务：`POST /translate`
  13 +- 重排服务：`POST /rerank`
  14 +
  15 +说明：脚本对 `embed_text` 场景会校验返回向量内容有效性（必须是有限数值，不允许 `null/NaN/Inf`），不是只看 HTTP 200。
  16 +
  17 +### 10.1 快速示例
  18 +
  19 +```bash
  20 +# suggest 压测（tenant 162）
  21 +python scripts/perf_api_benchmark.py \
  22 + --scenario backend_suggest \
  23 + --tenant-id 162 \
  24 + --duration 30 \
  25 + --concurrency 50
  26 +
  27 +# search 压测
  28 +python scripts/perf_api_benchmark.py \
  29 + --scenario backend_search \
  30 + --tenant-id 162 \
  31 + --duration 30 \
  32 + --concurrency 20
  33 +
  34 +# 全链路压测（search + suggest + embedding + translate + rerank）
  35 +python scripts/perf_api_benchmark.py \
  36 + --scenario all \
  37 + --tenant-id 162 \
  38 + --duration 60 \
  39 + --concurrency 30 \
  40 + --output perf_reports/all.json
  41 +```
  42 +
  43 +### 10.2 自定义用例
  44 +
  45 +可通过 `--cases-file` 覆盖默认请求模板。示例文件：
  46 +
  47 +```bash
  48 +scripts/perf_cases.json.example
  49 +```
  50 +
  51 +执行示例：
  52 +
  53 +```bash
  54 +python scripts/perf_api_benchmark.py \
  55 + --scenario all \
  56 + --tenant-id 162 \
  57 + --cases-file scripts/perf_cases.json.example \
  58 + --duration 60 \
  59 + --concurrency 40
  60 +```
  61 +
... ...
docs/搜索API对接指南.md renamed to docs/搜索API对接指南—拆分前版本存档.md
  Show/Hide comments   View file @0342d89
embeddings/config.py
  Show/Hide comments   View file @0342d89
1   -"""
2   -Embedding module configuration.
  1 +"""Embedding service compatibility config derived from unified app config."""
3 2  
4   -This module is intentionally a plain Python file (no env var parsing, no extra deps).
5   -Edit values here to configure:
6   -- server host/port
7   -- local model settings (paths/devices/batch sizes)
8   -"""
  3 +from __future__ import annotations
9 4  
10 5 from typing import Optional
11   -import os
  6 +
  7 +from config.loader import get_app_config
12 8  
13 9  
14 10 class EmbeddingConfig(object):
15   - # Server
16   - HOST = os.getenv("EMBEDDING_HOST", "0.0.0.0")
17   - PORT = int(os.getenv("EMBEDDING_PORT", 6005))
18   -
19   - # Text backend defaults
20   - TEXT_MODEL_ID = os.getenv("TEXT_MODEL_ID", "Qwen/Qwen3-Embedding-0.6B")
21   - # Keep TEXT_MODEL_DIR as an alias so code can refer to one canonical text model value.
22   - TEXT_MODEL_DIR = TEXT_MODEL_ID
23   - TEXT_DEVICE = os.getenv("TEXT_DEVICE", "cuda") # "cuda" or "cpu"
24   - TEXT_BATCH_SIZE = int(os.getenv("TEXT_BATCH_SIZE", "32"))
25   - TEXT_NORMALIZE_EMBEDDINGS = os.getenv("TEXT_NORMALIZE_EMBEDDINGS", "true").lower() in ("1", "true", "yes")
26   - TEI_BASE_URL = os.getenv("TEI_BASE_URL", "http://127.0.0.1:8080")
27   - TEI_TIMEOUT_SEC = int(os.getenv("TEI_TIMEOUT_SEC", "60"))
28   -
29   - # Image embeddings
30   - # Option A: clip-as-service (Jina CLIP server, recommended)
31   - USE_CLIP_AS_SERVICE = os.getenv("USE_CLIP_AS_SERVICE", "true").lower() in ("1", "true", "yes")
32   - CLIP_AS_SERVICE_SERVER = os.getenv("CLIP_AS_SERVICE_SERVER", "grpc://127.0.0.1:51000")
33   - CLIP_AS_SERVICE_MODEL_NAME = os.getenv("CLIP_AS_SERVICE_MODEL_NAME", "CN-CLIP/ViT-L-14")
34   -
35   - # Option B: local CN-CLIP (when USE_CLIP_AS_SERVICE=false)
36   - IMAGE_MODEL_NAME = os.getenv("IMAGE_MODEL_NAME", "ViT-L-14")
37   - IMAGE_DEVICE = None # type: Optional[str] # "cuda" / "cpu" / None(auto)
38   -
39   - # Service behavior
40   - IMAGE_BATCH_SIZE = 8
41   - IMAGE_NORMALIZE_EMBEDDINGS = os.getenv("IMAGE_NORMALIZE_EMBEDDINGS", "true").lower() in ("1", "true", "yes")
  11 + def __init__(self) -> None:
  12 + app_config = get_app_config()
  13 + runtime = app_config.runtime
  14 + services = app_config.services.embedding
  15 + text_backend = services.get_backend_config()
  16 + image_backend = services.get_image_backend_config()
  17 +
  18 + self.HOST = runtime.embedding_host
  19 + self.PORT = runtime.embedding_port
  20 +
  21 + self.TEXT_MODEL_ID = str(text_backend.get("model_id") or "Qwen/Qwen3-Embedding-0.6B")
  22 + self.TEXT_MODEL_DIR = self.TEXT_MODEL_ID
  23 + self.TEXT_DEVICE = str(text_backend.get("device") or "cuda")
  24 + self.TEXT_BATCH_SIZE = int(text_backend.get("batch_size", 32))
  25 + self.TEXT_NORMALIZE_EMBEDDINGS = bool(text_backend.get("normalize_embeddings", True))
  26 + self.TEI_BASE_URL = str(text_backend.get("base_url") or "http://127.0.0.1:8080")
  27 + self.TEI_TIMEOUT_SEC = int(text_backend.get("timeout_sec", 60))
  28 +
  29 + self.USE_CLIP_AS_SERVICE = services.image_backend == "clip_as_service"
  30 + self.CLIP_AS_SERVICE_SERVER = str(image_backend.get("server") or "grpc://127.0.0.1:51000")
  31 + self.CLIP_AS_SERVICE_MODEL_NAME = str(image_backend.get("model_name") or "CN-CLIP/ViT-L-14")
  32 +
  33 + self.IMAGE_MODEL_NAME = str(image_backend.get("model_name") or "ViT-L-14")
  34 + self.IMAGE_DEVICE = image_backend.get("device") # type: Optional[str]
  35 + self.IMAGE_BATCH_SIZE = int(image_backend.get("batch_size", 8))
  36 + self.IMAGE_NORMALIZE_EMBEDDINGS = bool(image_backend.get("normalize_embeddings", True))
42 37  
43 38  
44 39 CONFIG = EmbeddingConfig()
... ...
embeddings/image_encoder.py
  Show/Hide comments   View file @0342d89
... ... @@ -9,8 +9,8 @@ from PIL import Image
9 9  
10 10 logger = logging.getLogger(__name__)
11 11  
  12 +from config.loader import get_app_config
12 13 from config.services_config import get_embedding_image_base_url
13   -from config.env_config import REDIS_CONFIG
14 14 from embeddings.cache_keys import build_image_cache_key
15 15 from embeddings.redis_embedding_cache import RedisEmbeddingCache
16 16  
... ... @@ -24,10 +24,11 @@ class CLIPImageEncoder:
24 24  
25 25 def __init__(self, service_url: Optional[str] = None):
26 26 resolved_url = service_url or get_embedding_image_base_url()
  27 + redis_config = get_app_config().infrastructure.redis
27 28 self.service_url = str(resolved_url).rstrip("/")
28 29 self.endpoint = f"{self.service_url}/embed/image"
29 30 # Reuse embedding cache prefix, but separate namespace for images to avoid collisions.
30   - self.cache_prefix = str(REDIS_CONFIG.get("embedding_cache_prefix", "embedding")).strip() or "embedding"
  31 + self.cache_prefix = str(redis_config.embedding_cache_prefix).strip() or "embedding"
31 32 logger.info("Creating CLIPImageEncoder instance with service URL: %s", self.service_url)
32 33 self.cache = RedisEmbeddingCache(
33 34 key_prefix=self.cache_prefix,
... ...
embeddings/redis_embedding_cache.py
  Show/Hide comments   View file @0342d89
... ... @@ -20,7 +20,7 @@ try:
20 20 except ImportError: # pragma: no cover - runtime fallback for minimal envs
21 21 redis = None # type: ignore[assignment]
22 22  
23   -from config.env_config import REDIS_CONFIG
  23 +from config.loader import get_app_config
24 24 from embeddings.bf16 import decode_embedding_from_redis, encode_embedding_for_redis
25 25  
26 26 logger = logging.getLogger(__name__)
... ... @@ -37,7 +37,8 @@ class RedisEmbeddingCache:
37 37 ):
38 38 self.key_prefix = (key_prefix or "").strip() or "embedding"
39 39 self.namespace = (namespace or "").strip()
40   - self.expire_time = expire_time or timedelta(days=REDIS_CONFIG.get("cache_expire_days", 180))
  40 + redis_config = get_app_config().infrastructure.redis
  41 + self.expire_time = expire_time or timedelta(days=redis_config.cache_expire_days)
41 42  
42 43 if redis_client is not None:
43 44 self.redis_client = redis_client
... ... @@ -50,13 +51,13 @@ class RedisEmbeddingCache:
50 51  
51 52 try:
52 53 client = redis.Redis(
53   - host=REDIS_CONFIG.get("host", "localhost"),
54   - port=REDIS_CONFIG.get("port", 6479),
55   - password=REDIS_CONFIG.get("password"),
  54 + host=redis_config.host,
  55 + port=redis_config.port,
  56 + password=redis_config.password,
56 57 decode_responses=False,
57   - socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),
58   - socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),
59   - retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False),
  58 + socket_timeout=redis_config.socket_timeout,
  59 + socket_connect_timeout=redis_config.socket_connect_timeout,
  60 + retry_on_timeout=redis_config.retry_on_timeout,
60 61 health_check_interval=10,
61 62 )
62 63 client.ping()
... ...
embeddings/server.py
  Show/Hide comments   View file @0342d89
... ... @@ -470,16 +470,8 @@ def load_models():
470 470 if backend_name == "tei":
471 471 from embeddings.text_embedding_tei import TEITextModel
472 472  
473   - base_url = (
474   - os.getenv("TEI_BASE_URL")
475   - or backend_cfg.get("base_url")
476   - or CONFIG.TEI_BASE_URL
477   - )
478   - timeout_sec = int(
479   - os.getenv("TEI_TIMEOUT_SEC")
480   - or backend_cfg.get("timeout_sec")
481   - or CONFIG.TEI_TIMEOUT_SEC
482   - )
  473 + base_url = backend_cfg.get("base_url") or CONFIG.TEI_BASE_URL
  474 + timeout_sec = int(backend_cfg.get("timeout_sec") or CONFIG.TEI_TIMEOUT_SEC)
483 475 logger.info("Loading text backend: tei (base_url=%s)", base_url)
484 476 _text_model = TEITextModel(
485 477 base_url=str(base_url),
... ... @@ -488,11 +480,7 @@ def load_models():
488 480 elif backend_name == "local_st":
489 481 from embeddings.text_embedding_sentence_transformers import Qwen3TextModel
490 482  
491   - model_id = (
492   - os.getenv("TEXT_MODEL_ID")
493   - or backend_cfg.get("model_id")
494   - or CONFIG.TEXT_MODEL_ID
495   - )
  483 + model_id = backend_cfg.get("model_id") or CONFIG.TEXT_MODEL_ID
496 484 logger.info("Loading text backend: local_st (model=%s)", model_id)
497 485 _text_model = Qwen3TextModel(model_id=str(model_id))
498 486 _start_text_batch_worker()
... ...
embeddings/text_encoder.py
  Show/Hide comments   View file @0342d89
... ... @@ -9,13 +9,11 @@ import requests
9 9  
10 10 logger = logging.getLogger(__name__)
11 11  
  12 +from config.loader import get_app_config
12 13 from config.services_config import get_embedding_text_base_url
13 14 from embeddings.cache_keys import build_text_cache_key
14 15 from embeddings.redis_embedding_cache import RedisEmbeddingCache
15 16  
16   -# Try to import REDIS_CONFIG, but allow import to fail
17   -from config.env_config import REDIS_CONFIG
18   -
19 17  
20 18 class TextEmbeddingEncoder:
21 19 """
... ... @@ -24,10 +22,11 @@ class TextEmbeddingEncoder:
24 22  
25 23 def __init__(self, service_url: Optional[str] = None):
26 24 resolved_url = service_url or get_embedding_text_base_url()
  25 + redis_config = get_app_config().infrastructure.redis
27 26 self.service_url = str(resolved_url).rstrip("/")
28 27 self.endpoint = f"{self.service_url}/embed/text"
29   - self.expire_time = timedelta(days=REDIS_CONFIG.get("cache_expire_days", 180))
30   - self.cache_prefix = str(REDIS_CONFIG.get("embedding_cache_prefix", "embedding")).strip() or "embedding"
  28 + self.expire_time = timedelta(days=redis_config.cache_expire_days)
  29 + self.cache_prefix = str(redis_config.embedding_cache_prefix).strip() or "embedding"
31 30 logger.info("Creating TextEmbeddingEncoder instance with service URL: %s", self.service_url)
32 31  
33 32 self.cache = RedisEmbeddingCache(
... ...
indexer/document_transformer.py
  Show/Hide comments   View file @0342d89
... ... @@ -13,7 +13,6 @@ import numpy as np
13 13 import logging
14 14 import re
15 15 from typing import Dict, Any, Optional, List
16   -from config import ConfigLoader
17 16 from indexer.product_enrich import analyze_products
18 17  
19 18 logger = logging.getLogger(__name__)
... ...
indexer/incremental_service.py
  Show/Hide comments   View file @0342d89
... ... @@ -13,7 +13,7 @@ from indexer.mapping_generator import get_tenant_index_name
13 13 from indexer.indexer_logger import (
14 14 get_indexer_logger, log_index_request, log_index_result, log_spu_processing
15 15 )
16   -from config import ConfigLoader
  16 +from config import get_app_config
17 17 from translation import create_translation_client
18 18  
19 19 # Configure logger
... ... @@ -51,7 +51,7 @@ class IncrementalIndexerService:
51 51  
52 52 def _eager_init(self) -> None:
53 53 """Strict eager initialization. Any dependency failure should fail fast."""
54   - self._config = ConfigLoader("config/config.yaml").load_config()
  54 + self._config = get_app_config().search
55 55 self._searchable_option_dimensions = (
56 56 getattr(self._config.spu_config, "searchable_option_dimensions", None)
57 57 or ["option1", "option2", "option3"]
... ...
indexer/indexing_utils.py
  Show/Hide comments   View file @0342d89
... ... @@ -7,7 +7,7 @@
7 7 import logging
8 8 from typing import Dict, Any, Optional
9 9 from sqlalchemy import Engine, text
10   -from config import ConfigLoader
  10 +from config import get_app_config
11 11 from config.tenant_config_loader import get_tenant_config_loader
12 12 from indexer.document_transformer import SPUDocumentTransformer
13 13 from translation import create_translation_client
... ... @@ -92,8 +92,7 @@ def create_document_transformer(
92 92 or config is None
93 93 ):
94 94 if config is None:
95   - config_loader = ConfigLoader()
96   - config = config_loader.load_config()
  95 + config = get_app_config().search
97 96  
98 97 if searchable_option_dimensions is None:
99 98 searchable_option_dimensions = config.spu_config.searchable_option_dimensions
... ...
indexer/mapping_generator.py
  Show/Hide comments   View file @0342d89
... ... @@ -9,7 +9,7 @@ import json
9 9 import logging
10 10 from pathlib import Path
11 11  
12   -from config.env_config import ES_INDEX_NAMESPACE
  12 +from config.loader import get_app_config
13 13  
14 14 logger = logging.getLogger(__name__)
15 15  
... ... @@ -30,7 +30,7 @@ def get_tenant_index_name(tenant_id: str) -&gt; str:
30 30 其中 ES_INDEX_NAMESPACE 由 config.env_config.ES_INDEX_NAMESPACE 控制，
31 31 用于区分 prod/uat/test 等不同运行环境。
32 32 """
33   - prefix = ES_INDEX_NAMESPACE or ""
  33 + prefix = get_app_config().runtime.index_namespace or ""
34 34 return f"{prefix}search_products_tenant_{tenant_id}"
35 35  
36 36  
... ...
indexer/product_enrich.py
  Show/Hide comments   View file @0342d89
... ... @@ -12,15 +12,18 @@ import logging
12 12 import re
13 13 import time
14 14 import hashlib
  15 +import uuid
  16 +import threading
15 17 from collections import OrderedDict
16 18 from datetime import datetime
  19 +from concurrent.futures import ThreadPoolExecutor
17 20 from typing import List, Dict, Tuple, Any, Optional
18 21  
19 22 import redis
20 23 import requests
21 24 from pathlib import Path
22 25  
23   -from config.env_config import REDIS_CONFIG
  26 +from config.loader import get_app_config
24 27 from config.tenant_config_loader import SOURCE_LANG_CODE_MAP
25 28 from indexer.product_enrich_prompts import (
26 29 SYSTEM_MESSAGE,
... ... @@ -31,6 +34,9 @@ from indexer.product_enrich_prompts import (
31 34  
32 35 # 配置
33 36 BATCH_SIZE = 20
  37 +# enrich-content LLM 批次并发 worker 上限（线程池；仅对 uncached batch 并发）
  38 +_APP_CONFIG = get_app_config()
  39 +CONTENT_UNDERSTANDING_MAX_WORKERS = int(_APP_CONFIG.product_enrich.max_workers)
34 40 # 华北2（北京）：https://dashscope.aliyuncs.com/compatible-mode/v1
35 41 # 新加坡：https://dashscope-intl.aliyuncs.com/compatible-mode/v1
36 42 # 美国（弗吉尼亚）：https://dashscope-us.aliyuncs.com/compatible-mode/v1
... ... @@ -56,6 +62,24 @@ timestamp = datetime.now().strftime(&quot;%Y%m%d_%H%M%S&quot;)
56 62 log_file = LOG_DIR / f"product_enrich_{timestamp}.log"
57 63 verbose_log_file = LOG_DIR / "product_enrich_verbose.log"
58 64 _logged_shared_context_keys: "OrderedDict[str, None]" = OrderedDict()
  65 +_logged_shared_context_lock = threading.Lock()
  66 +
  67 +_content_understanding_executor: Optional[ThreadPoolExecutor] = None
  68 +_content_understanding_executor_lock = threading.Lock()
  69 +
  70 +
  71 +def _get_content_understanding_executor() -> ThreadPoolExecutor:
  72 + """
  73 + 使用模块级单例线程池，避免同一进程内多次请求叠加创建线程池导致并发失控。
  74 + """
  75 + global _content_understanding_executor
  76 + with _content_understanding_executor_lock:
  77 + if _content_understanding_executor is None:
  78 + _content_understanding_executor = ThreadPoolExecutor(
  79 + max_workers=CONTENT_UNDERSTANDING_MAX_WORKERS,
  80 + thread_name_prefix="product-enrich-llm",
  81 + )
  82 + return _content_understanding_executor
59 83  
60 84 # 主日志 logger：执行流程、批次信息等
61 85 logger = logging.getLogger("product_enrich")
... ... @@ -91,19 +115,20 @@ logger.info(&quot;Verbose LLM logs are written to: %s&quot;, verbose_log_file)
91 115  
92 116  
93 117 # Redis 缓存（用于 anchors / 语义属性）
94   -ANCHOR_CACHE_PREFIX = REDIS_CONFIG.get("anchor_cache_prefix", "product_anchors")
95   -ANCHOR_CACHE_EXPIRE_DAYS = int(REDIS_CONFIG.get("anchor_cache_expire_days", 30))
  118 +_REDIS_CONFIG = _APP_CONFIG.infrastructure.redis
  119 +ANCHOR_CACHE_PREFIX = _REDIS_CONFIG.anchor_cache_prefix
  120 +ANCHOR_CACHE_EXPIRE_DAYS = int(_REDIS_CONFIG.anchor_cache_expire_days)
96 121 _anchor_redis: Optional[redis.Redis] = None
97 122  
98 123 try:
99 124 _anchor_redis = redis.Redis(
100   - host=REDIS_CONFIG.get("host", "localhost"),
101   - port=REDIS_CONFIG.get("port", 6479),
102   - password=REDIS_CONFIG.get("password"),
  125 + host=_REDIS_CONFIG.host,
  126 + port=_REDIS_CONFIG.port,
  127 + password=_REDIS_CONFIG.password,
103 128 decode_responses=True,
104   - socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),
105   - socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),
106   - retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False),
  129 + socket_timeout=_REDIS_CONFIG.socket_timeout,
  130 + socket_connect_timeout=_REDIS_CONFIG.socket_connect_timeout,
  131 + retry_on_timeout=_REDIS_CONFIG.retry_on_timeout,
107 132 health_check_interval=10,
108 133 )
109 134 _anchor_redis.ping()
... ... @@ -242,19 +267,21 @@ def _hash_text(text: str) -&gt; str:
242 267  
243 268  
244 269 def _mark_shared_context_logged_once(shared_context_key: str) -> bool:
245   - if shared_context_key in _logged_shared_context_keys:
246   - _logged_shared_context_keys.move_to_end(shared_context_key)
247   - return False
  270 + with _logged_shared_context_lock:
  271 + if shared_context_key in _logged_shared_context_keys:
  272 + _logged_shared_context_keys.move_to_end(shared_context_key)
  273 + return False
248 274  
249   - _logged_shared_context_keys[shared_context_key] = None
250   - if len(_logged_shared_context_keys) > LOGGED_SHARED_CONTEXT_CACHE_SIZE:
251   - _logged_shared_context_keys.popitem(last=False)
252   - return True
  275 + _logged_shared_context_keys[shared_context_key] = None
  276 + if len(_logged_shared_context_keys) > LOGGED_SHARED_CONTEXT_CACHE_SIZE:
  277 + _logged_shared_context_keys.popitem(last=False)
  278 + return True
253 279  
254 280  
255 281 def reset_logged_shared_context_keys() -> None:
256 282 """测试辅助：清理已记录的共享 prompt key。"""
257   - _logged_shared_context_keys.clear()
  283 + with _logged_shared_context_lock:
  284 + _logged_shared_context_keys.clear()
258 285  
259 286  
260 287 def create_prompt(
... ... @@ -625,7 +652,9 @@ def process_batch(
625 652 "final_results": results_with_ids,
626 653 }
627 654  
628   - batch_log_file = LOG_DIR / f"batch_{batch_num:04d}_{timestamp}.json"
  655 + # 并发写 batch json 日志时，保证文件名唯一避免覆盖
  656 + batch_call_id = uuid.uuid4().hex[:12]
  657 + batch_log_file = LOG_DIR / f"batch_{batch_num:04d}_{timestamp}_{batch_call_id}.json"
629 658 with open(batch_log_file, "w", encoding="utf-8") as f:
630 659 json.dump(batch_log, f, ensure_ascii=False, indent=2)
631 660  
... ... @@ -707,28 +736,70 @@ def analyze_products(
707 736 bs = max(1, min(req_bs, BATCH_SIZE))
708 737 total_batches = (len(uncached_items) + bs - 1) // bs
709 738  
  739 + batch_jobs: List[Tuple[int, List[Tuple[int, Dict[str, str]]], List[Dict[str, str]]]] = []
710 740 for i in range(0, len(uncached_items), bs):
711 741 batch_num = i // bs + 1
712 742 batch_slice = uncached_items[i : i + bs]
713 743 batch = [item for _, item in batch_slice]
  744 + batch_jobs.append((batch_num, batch_slice, batch))
  745 +
  746 + # 只有一个批次时走串行，减少线程池创建开销与日志/日志文件的不可控交织
  747 + if total_batches <= 1 or CONTENT_UNDERSTANDING_MAX_WORKERS <= 1:
  748 + for batch_num, batch_slice, batch in batch_jobs:
  749 + logger.info(
  750 + f"[analyze_products] Processing batch {batch_num}/{total_batches}, "
  751 + f"size={len(batch)}, target_lang={target_lang}"
  752 + )
  753 + batch_results = process_batch(batch, batch_num=batch_num, target_lang=target_lang)
  754 +
  755 + for (original_idx, product), item in zip(batch_slice, batch_results):
  756 + results_by_index[original_idx] = item
  757 + title_input = str(item.get("title_input") or "").strip()
  758 + if not title_input:
  759 + continue
  760 + if item.get("error"):
  761 + # 不缓存错误结果，避免放大临时故障
  762 + continue
  763 + try:
  764 + _set_cached_anchor_result(product, target_lang, item)
  765 + except Exception:
  766 + # 已在内部记录 warning
  767 + pass
  768 + else:
  769 + max_workers = min(CONTENT_UNDERSTANDING_MAX_WORKERS, len(batch_jobs))
714 770 logger.info(
715   - f"[analyze_products] Processing batch {batch_num}/{total_batches}, "
716   - f"size={len(batch)}, target_lang={target_lang}"
  771 + "[analyze_products] Using ThreadPoolExecutor for uncached batches: "
  772 + "max_workers=%s, total_batches=%s, bs=%s, target_lang=%s",
  773 + max_workers,
  774 + total_batches,
  775 + bs,
  776 + target_lang,
717 777 )
718   - batch_results = process_batch(batch, batch_num=batch_num, target_lang=target_lang)
719 778  
720   - for (original_idx, product), item in zip(batch_slice, batch_results):
721   - results_by_index[original_idx] = item
722   - title_input = str(item.get("title_input") or "").strip()
723   - if not title_input:
724   - continue
725   - if item.get("error"):
726   - # 不缓存错误结果，避免放大临时故障
727   - continue
728   - try:
729   - _set_cached_anchor_result(product, target_lang, item)
730   - except Exception:
731   - # 已在内部记录 warning
732   - pass
  779 + # 只把“LLM 调用 + markdown 解析”放到线程里；Redis get/set 保持在主线程，避免并发写入带来额外风险。
  780 + # 注意：线程池是模块级单例，因此这里的 max_workers 主要用于日志语义（实际并发受单例池上限约束）。
  781 + executor = _get_content_understanding_executor()
  782 + future_by_batch_num: Dict[int, Any] = {}
  783 + for batch_num, _batch_slice, batch in batch_jobs:
  784 + future_by_batch_num[batch_num] = executor.submit(
  785 + process_batch, batch, batch_num=batch_num, target_lang=target_lang
  786 + )
  787 +
  788 + # 按 batch_num 回填，确保输出稳定（results_by_index 是按原始 input index 映射的）
  789 + for batch_num, batch_slice, _batch in batch_jobs:
  790 + batch_results = future_by_batch_num[batch_num].result()
  791 + for (original_idx, product), item in zip(batch_slice, batch_results):
  792 + results_by_index[original_idx] = item
  793 + title_input = str(item.get("title_input") or "").strip()
  794 + if not title_input:
  795 + continue
  796 + if item.get("error"):
  797 + # 不缓存错误结果，避免放大临时故障
  798 + continue
  799 + try:
  800 + _set_cached_anchor_result(product, target_lang, item)
  801 + except Exception:
  802 + # 已在内部记录 warning
  803 + pass
733 804  
734 805 return [item for item in results_by_index if item is not None]
... ...
... ... @@ -16,8 +16,7 @@ import json
16 16 # Add parent directory to path
17 17 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
18 18  
19   -from config import ConfigLoader
20   -from config.env_config import ES_CONFIG
  19 +from config import get_app_config
21 20 from utils import ESClient
22 21 from search import Searcher
23 22 from suggestion import SuggestionIndexBuilder
... ... @@ -61,8 +60,7 @@ def cmd_serve_indexer(args):
61 60 def cmd_search(args):
62 61 """Test search from command line."""
63 62 # Load config
64   - config_loader = ConfigLoader("config/config.yaml")
65   - config = config_loader.load_config()
  63 + config = get_app_config().search
66 64  
67 65 # Initialize ES and searcher
68 66 es_client = ESClient(hosts=[args.es_host])
... ... @@ -106,8 +104,9 @@ def cmd_search(args):
106 104 def cmd_build_suggestions(args):
107 105 """Build/update suggestion index for a tenant."""
108 106 # Initialize ES client with optional authentication
109   - es_username = os.getenv("ES_USERNAME") or ES_CONFIG.get("username")
110   - es_password = os.getenv("ES_PASSWORD") or ES_CONFIG.get("password")
  107 + es_cfg = get_app_config().infrastructure.elasticsearch
  108 + es_username = es_cfg.username
  109 + es_password = es_cfg.password
111 110 if es_username and es_password:
112 111 es_client = ESClient(hosts=[args.es_host], username=es_username, password=es_password)
113 112 else:
... ... @@ -117,11 +116,12 @@ def cmd_build_suggestions(args):
117 116 return 1
118 117  
119 118 # Build DB config directly from environment to avoid dotenv dependency
120   - db_host = os.getenv("DB_HOST")
121   - db_port = int(os.getenv("DB_PORT", "3306"))
122   - db_name = os.getenv("DB_DATABASE")
123   - db_user = os.getenv("DB_USERNAME")
124   - db_pass = os.getenv("DB_PASSWORD")
  119 + db_cfg = get_app_config().infrastructure.database
  120 + db_host = db_cfg.host
  121 + db_port = db_cfg.port
  122 + db_name = db_cfg.database
  123 + db_user = db_cfg.username
  124 + db_pass = db_cfg.password
125 125 if not all([db_host, db_name, db_user, db_pass]):
126 126 print("ERROR: DB_HOST/DB_PORT/DB_DATABASE/DB_USERNAME/DB_PASSWORD must be set in environment")
127 127 return 1
... ... @@ -170,7 +170,7 @@ def main():
170 170 serve_parser = subparsers.add_parser('serve', help='Start API service (multi-tenant)')
171 171 serve_parser.add_argument('--host', default='0.0.0.0', help='Host to bind to')
172 172 serve_parser.add_argument('--port', type=int, default=6002, help='Port to bind to')
173   - serve_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host')
  173 + serve_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
174 174 serve_parser.add_argument('--reload', action='store_true', help='Enable auto-reload')
175 175  
176 176 # Serve-indexer command
... ... @@ -180,14 +180,14 @@ def main():
180 180 )
181 181 serve_indexer_parser.add_argument('--host', default='0.0.0.0', help='Host to bind to')
182 182 serve_indexer_parser.add_argument('--port', type=int, default=6004, help='Port to bind to')
183   - serve_indexer_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host')
  183 + serve_indexer_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
184 184 serve_indexer_parser.add_argument('--reload', action='store_true', help='Enable auto-reload')
185 185  
186 186 # Search command
187 187 search_parser = subparsers.add_parser('search', help='Test search from command line')
188 188 search_parser.add_argument('query', help='Search query')
189 189 search_parser.add_argument('--tenant-id', required=True, help='Tenant ID (required)')
190   - search_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host')
  190 + search_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
191 191 search_parser.add_argument('--size', type=int, default=10, help='Number of results')
192 192 search_parser.add_argument('--no-translation', action='store_true', help='Disable translation')
193 193 search_parser.add_argument('--no-embedding', action='store_true', help='Disable embeddings')
... ... @@ -199,7 +199,7 @@ def main():
199 199 help='Build tenant suggestion index (full/incremental)'
200 200 )
201 201 suggest_build_parser.add_argument('--tenant-id', required=True, help='Tenant ID')
202   - suggest_build_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host')
  202 + suggest_build_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
203 203 suggest_build_parser.add_argument(
204 204 '--mode',
205 205 choices=['full', 'incremental'],
... ...
query/query_parser.py
  Show/Hide comments   View file @0342d89
... ... @@ -336,13 +336,13 @@ class QueryParser:
336 336 translations = {}
337 337 translation_futures = {}
338 338 translation_executor = None
339   - index_langs = ["en", "zh"]
  339 + index_langs: List[str] = []
340 340 try:
341 341 # 根据租户配置的 index_languages 决定翻译目标语言
342 342 from config.tenant_config_loader import get_tenant_config_loader
343 343 tenant_loader = get_tenant_config_loader()
344 344 tenant_cfg = tenant_loader.get_tenant_config(tenant_id or "default")
345   - raw_index_langs = tenant_cfg.get("index_languages") or ["en", "zh"]
  345 + raw_index_langs = tenant_cfg.get("index_languages") or []
346 346 index_langs = []
347 347 seen_langs = set()
348 348 for lang in raw_index_langs:
... ... @@ -618,17 +618,3 @@ class QueryParser:
618 618 queries.append(translation)
619 619  
620 620 return queries
621   -
622   - def update_rewrite_rules(self, rules: Dict[str, str]) -> None:
623   - """
624   - Update query rewrite rules.
625   -
626   - Args:
627   - rules: Dictionary of pattern -> replacement mappings
628   - """
629   - for pattern, replacement in rules.items():
630   - self.rewriter.add_rule(pattern, replacement)
631   -
632   - def get_rewrite_rules(self) -> Dict[str, str]:
633   - """Get current rewrite rules."""
634   - return self.rewriter.get_rules()
... ...
reranker/backends/dashscope_rerank.py
  Show/Hide comments   View file @0342d89
... ... @@ -63,43 +63,19 @@ class DashScopeRerankBackend:
63 63 - max_retries: int, default 1
64 64 - retry_backoff_sec: float, default 0.2
65 65  
66   - Env overrides:
67   - - RERANK_DASHSCOPE_ENDPOINT
68   - - RERANK_DASHSCOPE_MODEL
69   - - RERANK_DASHSCOPE_TIMEOUT_SEC
70   - - RERANK_DASHSCOPE_TOP_N_CAP
71   - - RERANK_DASHSCOPE_BATCHSIZE
72 66 """
73 67  
74 68 def __init__(self, config: Dict[str, Any]) -> None:
75 69 self._config = config or {}
76   - self._model_name = str(
77   - os.getenv("RERANK_DASHSCOPE_MODEL")
78   - or self._config.get("model_name")
79   - or "qwen3-rerank"
80   - )
  70 + self._model_name = str(self._config.get("model_name") or "qwen3-rerank")
81 71 self._endpoint = str(
82   - os.getenv("RERANK_DASHSCOPE_ENDPOINT")
83   - or self._config.get("endpoint")
84   - or "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
  72 + self._config.get("endpoint") or "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
85 73 ).strip()
86 74 self._api_key_env = str(self._config.get("api_key_env") or "").strip()
87 75 self._api_key = str(os.getenv(self._api_key_env) or "").strip().strip('"').strip("'")
88   - self._timeout_sec = float(
89   - os.getenv("RERANK_DASHSCOPE_TIMEOUT_SEC")
90   - or self._config.get("timeout_sec")
91   - or 15.0
92   - )
93   - self._top_n_cap = int(
94   - os.getenv("RERANK_DASHSCOPE_TOP_N_CAP")
95   - or self._config.get("top_n_cap")
96   - or 0
97   - )
98   - self._batchsize = int(
99   - os.getenv("RERANK_DASHSCOPE_BATCHSIZE")
100   - or self._config.get("batchsize")
101   - or 0
102   - )
  76 + self._timeout_sec = float(self._config.get("timeout_sec") or 15.0)
  77 + self._top_n_cap = int(self._config.get("top_n_cap") or 0)
  78 + self._batchsize = int(self._config.get("batchsize") or 0)
103 79 self._instruct = str(self._config.get("instruct") or "").strip()
104 80 self._max_retries = int(self._config.get("max_retries", 1))
105 81 self._retry_backoff_sec = float(self._config.get("retry_backoff_sec", 0.2))
... ...
reranker/config.py
  Show/Hide comments   View file @0342d89
1   -"""Reranker service configuration (simple Python config)."""
  1 +"""Reranker service compatibility config derived from unified app config."""
2 2  
3   -import os
  3 +from __future__ import annotations
  4 +
  5 +from config.loader import get_app_config
4 6  
5 7  
6 8 class RerankerConfig(object):
7   - # Server
8   - HOST = os.getenv("RERANKER_HOST", "0.0.0.0")
9   - PORT = int(os.getenv("RERANKER_PORT", 6007))
10   -
11   - # Model
12   - MODEL_NAME = "Qwen/Qwen3-Reranker-0.6B"
13   - DEVICE = None # None -> auto (cuda if available)
14   - USE_FP16 = True
15   - BATCH_SIZE = 64
16   - MAX_LENGTH = 512
17   - CACHE_DIR = "./model_cache"
18   - ENABLE_WARMUP = True
19   -
20   - # Request limits
21   - MAX_DOCS = 1000
22   -
23   - # Output
24   - NORMALIZE = True
  9 + def __init__(self) -> None:
  10 + app_config = get_app_config()
  11 + runtime = app_config.runtime
  12 + service = app_config.services.rerank
  13 + backend = service.get_backend_config()
  14 + request = service.request
  15 +
  16 + self.HOST = runtime.reranker_host
  17 + self.PORT = runtime.reranker_port
  18 +
  19 + self.MODEL_NAME = str(backend.get("model_name") or "Qwen/Qwen3-Reranker-0.6B")
  20 + self.DEVICE = backend.get("device")
  21 + self.USE_FP16 = bool(backend.get("use_fp16", True))
  22 + self.BATCH_SIZE = int(backend.get("batch_size", backend.get("infer_batch_size", 64)))
  23 + self.MAX_LENGTH = int(backend.get("max_length", 512))
  24 + self.CACHE_DIR = str(backend.get("cache_dir") or "./model_cache")
  25 + self.ENABLE_WARMUP = bool(backend.get("enable_warmup", True))
  26 +
  27 + self.MAX_DOCS = int(request.get("max_docs", 1000))
  28 + self.NORMALIZE = bool(request.get("normalize", True))
25 29  
26 30  
27 31 CONFIG = RerankerConfig()
... ...
suggestion/builder.py
  Show/Hide comments   View file @0342d89
... ... @@ -18,7 +18,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple
18 18  
19 19 from sqlalchemy import text
20 20  
21   -from config.env_config import ES_INDEX_NAMESPACE
  21 +from config.loader import get_app_config
22 22 from config.tenant_config_loader import get_tenant_config_loader
23 23 from suggestion.mapping import build_suggestion_mapping
24 24 from utils.es_client import ESClient
... ... @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
27 27  
28 28  
29 29 def _index_prefix() -> str:
30   - return ES_INDEX_NAMESPACE or ""
  30 + return get_app_config().runtime.index_namespace or ""
31 31  
32 32  
33 33 def get_suggestion_alias_name(tenant_id: str) -> str:
... ...
tests/test_process_products_batching.py
  Show/Hide comments   View file @0342d89
... ... @@ -45,7 +45,8 @@ def test_analyze_products_caps_batch_size_to_20(monkeypatch):
45 45 )
46 46  
47 47 assert len(out) == 45
48   - assert seen_batch_sizes == [20, 20, 5]
  48 + # 并发执行时 batch 调用顺序可能变化，因此校验“批大小集合”而不是严格顺序
  49 + assert sorted(seen_batch_sizes) == [5, 20, 20]
49 50  
50 51  
51 52 def test_analyze_products_uses_min_batch_size_1(monkeypatch):
... ...
translation/backends/deepl.py
  Show/Hide comments   View file @0342d89
... ... @@ -3,7 +3,6 @@
3 3 from __future__ import annotations
4 4  
5 5 import logging
6   -import os
7 6 import re
8 7 from typing import List, Optional, Sequence, Tuple, Union
9 8  
... ... @@ -24,7 +23,7 @@ class DeepLTranslationBackend:
24 23 timeout: float,
25 24 glossary_id: Optional[str] = None,
26 25 ) -> None:
27   - self.api_key = api_key or os.getenv("DEEPL_AUTH_KEY")
  26 + self.api_key = api_key
28 27 self.api_url = api_url
29 28 self.timeout = float(timeout)
30 29 self.glossary_id = glossary_id
... ...
translation/backends/llm.py
  Show/Hide comments   View file @0342d89
... ... @@ -3,13 +3,11 @@
3 3 from __future__ import annotations
4 4  
5 5 import logging
6   -import os
7 6 import time
8 7 from typing import List, Optional, Sequence, Union
9 8  
10 9 from openai import OpenAI
11 10  
12   -from config.env_config import DASHSCOPE_API_KEY
13 11 from translation.languages import LANGUAGE_LABELS
14 12 from translation.prompts import TRANSLATION_PROMPTS
15 13 from translation.scenes import normalize_scene_name
... ... @@ -52,11 +50,13 @@ class LLMTranslationBackend:
52 50 model: str,
53 51 timeout_sec: float,
54 52 base_url: str,
  53 + api_key: Optional[str],
55 54 ) -> None:
56 55 self.capability_name = capability_name
57 56 self.model = model
58 57 self.timeout_sec = float(timeout_sec)
59 58 self.base_url = base_url
  59 + self.api_key = api_key
60 60 self.client = self._create_client()
61 61  
62 62 @property
... ... @@ -64,12 +64,11 @@ class LLMTranslationBackend:
64 64 return True
65 65  
66 66 def _create_client(self) -> Optional[OpenAI]:
67   - api_key = DASHSCOPE_API_KEY or os.getenv("DASHSCOPE_API_KEY")
68   - if not api_key:
  67 + if not self.api_key:
69 68 logger.warning("DASHSCOPE_API_KEY not set; llm translation unavailable")
70 69 return None
71 70 try:
72   - return OpenAI(api_key=api_key, base_url=self.base_url)
  71 + return OpenAI(api_key=self.api_key, base_url=self.base_url)
73 72 except Exception as exc:
74 73 logger.error("Failed to initialize llm translation client: %s", exc, exc_info=True)
75 74 return None
... ...
translation/backends/qwen_mt.py
  Show/Hide comments   View file @0342d89
... ... @@ -3,14 +3,12 @@
3 3 from __future__ import annotations
4 4  
5 5 import logging
6   -import os
7 6 import re
8 7 import time
9 8 from typing import List, Optional, Sequence, Union
10 9  
11 10 from openai import OpenAI
12 11  
13   -from config.env_config import DASHSCOPE_API_KEY
14 12 from translation.languages import QWEN_LANGUAGE_CODES
15 13  
16 14 logger = logging.getLogger(__name__)
... ... @@ -64,7 +62,7 @@ class QwenMTTranslationBackend:
64 62 @staticmethod
65 63 def _default_api_key(model: str) -> Optional[str]:
66 64 del model
67   - return DASHSCOPE_API_KEY or os.getenv("DASHSCOPE_API_KEY")
  65 + return None
68 66  
69 67 def translate(
70 68 self,
... ...
translation/cache.py
  Show/Hide comments   View file @0342d89
... ... @@ -6,9 +6,12 @@ import hashlib
6 6 import logging
7 7 from typing import Mapping, Optional
8 8  
9   -import redis
  9 +try:
  10 + import redis
  11 +except ImportError: # pragma: no cover - runtime fallback for minimal envs
  12 + redis = None # type: ignore[assignment]
10 13  
11   -from config.env_config import REDIS_CONFIG
  14 +from config.loader import get_app_config
12 15  
13 16 logger = logging.getLogger(__name__)
14 17  
... ... @@ -70,15 +73,19 @@ class TranslationCache:
70 73  
71 74 @staticmethod
72 75 def _init_redis_client() -> Optional[redis.Redis]:
  76 + if redis is None:
  77 + logger.warning("redis package is not installed; translation cache disabled")
  78 + return None
  79 + redis_config = get_app_config().infrastructure.redis
73 80 try:
74 81 client = redis.Redis(
75   - host=REDIS_CONFIG.get("host", "localhost"),
76   - port=REDIS_CONFIG.get("port", 6479),
77   - password=REDIS_CONFIG.get("password"),
  82 + host=redis_config.host,
  83 + port=redis_config.port,
  84 + password=redis_config.password,
78 85 decode_responses=True,
79   - socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),
80   - socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),
81   - retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False),
  86 + socket_timeout=redis_config.socket_timeout,
  87 + socket_connect_timeout=redis_config.socket_connect_timeout,
  88 + retry_on_timeout=redis_config.retry_on_timeout,
82 89 health_check_interval=10,
83 90 )
84 91 client.ping()
... ...
translation/client.py
  Show/Hide comments   View file @0342d89
... ... @@ -7,7 +7,7 @@ from typing import List, Optional, Sequence, Union
7 7  
8 8 import requests
9 9  
10   -from config.services_config import get_translation_config
  10 +from config.loader import get_app_config
11 11 from translation.settings import normalize_translation_model, normalize_translation_scene
12 12  
13 13 logger = logging.getLogger(__name__)
... ... @@ -24,7 +24,7 @@ class TranslationServiceClient:
24 24 default_scene: Optional[str] = None,
25 25 timeout_sec: Optional[float] = None,
26 26 ) -> None:
27   - cfg = get_translation_config()
  27 + cfg = get_app_config().services.translation.as_dict()
28 28 self.base_url = str(base_url or cfg["service_url"]).rstrip("/")
29 29 self.default_model = normalize_translation_model(cfg, default_model or cfg["default_model"])
30 30 self.default_scene = normalize_translation_scene(cfg, default_scene or cfg["default_scene"])
... ...
translation/service.py
  Show/Hide comments   View file @0342d89
... ... @@ -5,7 +5,8 @@ from __future__ import annotations
5 5 import logging
6 6 from typing import Dict, List, Optional
7 7  
8   -from config.services_config import get_translation_config
  8 +from config.loader import get_app_config
  9 +from config.schema import AppConfig
9 10 from translation.cache import TranslationCache
10 11 from translation.protocols import TranslateInput, TranslateOutput, TranslationBackendProtocol
11 12 from translation.settings import (
... ... @@ -22,8 +23,9 @@ logger = logging.getLogger(__name__)
22 23 class TranslationService:
23 24 """Owns translation backends and routes calls by model and scene."""
24 25  
25   - def __init__(self, config: Optional[TranslationConfig] = None) -> None:
26   - self.config = config or get_translation_config()
  26 + def __init__(self, config: Optional[TranslationConfig] = None, app_config: Optional[AppConfig] = None) -> None:
  27 + self._app_config = app_config or get_app_config()
  28 + self.config = config or self._app_config.services.translation.as_dict()
27 29 self._enabled_capabilities = self._collect_enabled_capabilities()
28 30 if not self._enabled_capabilities:
29 31 raise ValueError("No enabled translation backends found in services.translation.capabilities")
... ... @@ -85,7 +87,7 @@ class TranslationService:
85 87 capability_name=name,
86 88 model=str(cfg["model"]).strip(),
87 89 base_url=str(cfg["base_url"]).strip(),
88   - api_key=cfg.get("api_key"),
  90 + api_key=self._app_config.infrastructure.secrets.dashscope_api_key,
89 91 timeout=int(cfg["timeout_sec"]),
90 92 glossary_id=cfg.get("glossary_id"),
91 93 )
... ... @@ -94,7 +96,7 @@ class TranslationService:
94 96 from translation.backends.deepl import DeepLTranslationBackend
95 97  
96 98 return DeepLTranslationBackend(
97   - api_key=cfg.get("api_key"),
  99 + api_key=self._app_config.infrastructure.secrets.deepl_auth_key,
98 100 api_url=str(cfg["api_url"]).strip(),
99 101 timeout=float(cfg["timeout_sec"]),
100 102 glossary_id=cfg.get("glossary_id"),
... ... @@ -108,6 +110,7 @@ class TranslationService:
108 110 model=str(cfg["model"]).strip(),
109 111 timeout_sec=float(cfg["timeout_sec"]),
110 112 base_url=str(cfg["base_url"]).strip(),
  113 + api_key=self._app_config.infrastructure.secrets.dashscope_api_key,
111 114 )
112 115  
113 116 def _create_local_nllb_backend(self, *, name: str, cfg: Dict[str, object]) -> TranslationBackendProtocol:
... ...
utils/es_client.py
  Show/Hide comments   View file @0342d89
... ... @@ -5,10 +5,9 @@ Elasticsearch client wrapper.
5 5 from elasticsearch import Elasticsearch
6 6 from elasticsearch.helpers import bulk
7 7 from typing import Dict, Any, List, Optional
8   -import os
9 8 import logging
10 9  
11   -from config.env_config import ES_CONFIG
  10 +from config.loader import get_app_config
12 11  
13 12 logger = logging.getLogger(__name__)
14 13  
... ... @@ -33,7 +32,7 @@ class ESClient:
33 32 **kwargs: Additional ES client parameters
34 33 """
35 34 if hosts is None:
36   - hosts = [os.getenv('ES_HOST', 'http://localhost:9200')]
  35 + hosts = [get_app_config().infrastructure.elasticsearch.host]
37 36  
38 37 # Build client config
39 38 client_config = {
... ... @@ -325,16 +324,9 @@ def get_es_client_from_env() -&gt; ESClient:
325 324 Returns:
326 325 ESClient instance
327 326 """
328   - if ES_CONFIG:
329   - return ESClient(
330   - hosts=[ES_CONFIG['host']],
331   - username=ES_CONFIG.get('username'),
332   - password=ES_CONFIG.get('password')
333   - )
334   - else:
335   - # Fallback to env variables
336   - return ESClient(
337   - hosts=[os.getenv('ES_HOST', 'http://localhost:9200')],
338   - username=os.getenv('ES_USERNAME'),
339   - password=os.getenv('ES_PASSWORD')
340   - )
  327 + cfg = get_app_config().infrastructure.elasticsearch
  328 + return ESClient(
  329 + hosts=[cfg.host],
  330 + username=cfg.username,
  331 + password=cfg.password,
  332 + )
... ...