ai-saas / saas-search

Commit 86d8358b25faf30015b8035300bad7fadb4d308f

Authored by tangwang
1 parent 77bfa7e3

config optimize

Showing 37 changed files with 2087 additions and 1123 deletions   Show diff stats
Inline Side-by-side
api/app.py
  Show/Hide comments   View file @86d8358
... ... @@ -86,8 +86,7 @@ limiter = Limiter(key_func=get_remote_address)
86 86 # Add parent directory to path
87 87 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
88 88  
89   -from config.env_config import ES_CONFIG, DB_CONFIG
90   -from config import ConfigLoader
  89 +from config import get_app_config
91 90 from utils import ESClient
92 91 from search import Searcher
93 92 from query import QueryParser
... ... @@ -99,7 +98,7 @@ _es_client: Optional[ESClient] = None
99 98 _searcher: Optional[Searcher] = None
100 99 _query_parser: Optional[QueryParser] = None
101 100 _suggestion_service: Optional[SuggestionService] = None
102   -_config = None
  101 +_app_config = None
103 102  
104 103  
105 104 def init_service(es_host: str = "http://localhost:9200"):
... ... @@ -109,20 +108,20 @@ def init_service(es_host: str = "http://localhost:9200"):
109 108 Args:
110 109 es_host: Elasticsearch host URL
111 110 """
112   - global _es_client, _searcher, _query_parser, _suggestion_service, _config
  111 + global _es_client, _searcher, _query_parser, _suggestion_service, _app_config
113 112  
114 113 start_time = time.time()
115 114 logger.info("Initializing search service (multi-tenant)")
116 115  
117 116 # Load configuration
118 117 logger.info("Loading configuration...")
119   - config_loader = ConfigLoader("config/config.yaml")
120   - _config = config_loader.load_config()
  118 + _app_config = get_app_config()
  119 + search_config = _app_config.search
121 120 logger.info("Configuration loaded")
122 121  
123 122 # Get ES credentials
124   - es_username = os.getenv('ES_USERNAME') or ES_CONFIG.get('username')
125   - es_password = os.getenv('ES_PASSWORD') or ES_CONFIG.get('password')
  123 + es_username = _app_config.infrastructure.elasticsearch.username
  124 + es_password = _app_config.infrastructure.elasticsearch.password
126 125  
127 126 # Connect to Elasticsearch
128 127 logger.info(f"Connecting to Elasticsearch at {es_host}...")
... ... @@ -139,15 +138,15 @@ def init_service(es_host: str = "http://localhost:9200"):
139 138  
140 139 # Initialize components
141 140 logger.info("Initializing query parser...")
142   - _query_parser = QueryParser(_config)
  141 + _query_parser = QueryParser(search_config)
143 142  
144 143 logger.info("Initializing searcher...")
145   - _searcher = Searcher(_es_client, _config, _query_parser)
  144 + _searcher = Searcher(_es_client, search_config, _query_parser)
146 145 logger.info("Initializing suggestion service...")
147 146 _suggestion_service = SuggestionService(_es_client)
148 147  
149 148 elapsed = time.time() - start_time
150   - logger.info(f"Search service ready! (took {elapsed:.2f}s) | Index: {_config.es_index_name}")
  149 + logger.info(f"Search service ready! (took {elapsed:.2f}s) | Index: {search_config.es_index_name}")
151 150  
152 151  
153 152  
... ... @@ -182,9 +181,9 @@ def get_suggestion_service() -> SuggestionService:
182 181  
183 182 def get_config():
184 183 """Get global config instance."""
185   - if _config is None:
  184 + if _app_config is None:
186 185 raise RuntimeError("Service not initialized")
187   - return _config
  186 + return _app_config
188 187  
189 188  
190 189 # Create FastAPI app with enhanced configuration
... ... @@ -240,7 +239,7 @@ async def startup_event():
240 239 except Exception as e:
241 240 logger.warning(f"Failed to set thread pool size: {e}, using default")
242 241  
243   - es_host = os.getenv("ES_HOST", "http://localhost:9200")
  242 + es_host = get_app_config().infrastructure.elasticsearch.host
244 243 logger.info("Starting E-Commerce Search API (Multi-Tenant)")
245 244 logger.info(f"Elasticsearch Host: {es_host}")
246 245  
... ...
api/indexer_app.py
  Show/Hide comments   View file @86d8358
... ... @@ -38,8 +38,7 @@ logger = logging.getLogger(__name__)
38 38 # Add parent directory to path
39 39 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
40 40  
41   -from config.env_config import ES_CONFIG # noqa: E402
42   -from config import ConfigLoader # noqa: E402
  41 +from config import get_app_config # noqa: E402
43 42 from utils import ESClient # noqa: E402
44 43 from utils.db_connector import create_db_connection # noqa: E402
45 44 from indexer.incremental_service import IncrementalIndexerService # noqa: E402
... ... @@ -55,7 +54,7 @@ from .service_registry import (
55 54  
56 55  
57 56 _es_client: Optional[ESClient] = None
58   -_config = None
  57 +_app_config = None
59 58 _incremental_service: Optional[IncrementalIndexerService] = None
60 59 _bulk_indexing_service: Optional[BulkIndexingService] = None
61 60 _suggestion_builder: Optional[SuggestionIndexBuilder] = None
... ... @@ -68,20 +67,19 @@ def init_indexer_service(es_host: str = "http://localhost:9200"):
68 67 This mirrors the indexing-related initialization logic in api.app.init_service
69 68 but without search-related components.
70 69 """
71   - global _es_client, _config, _incremental_service, _bulk_indexing_service, _suggestion_builder
  70 + global _es_client, _app_config, _incremental_service, _bulk_indexing_service, _suggestion_builder
72 71  
73 72 start_time = time.time()
74 73 logger.info("Initializing Indexer service")
75 74  
76 75 # Load configuration (kept for parity/logging; indexer routes don't depend on it)
77 76 logger.info("Loading configuration...")
78   - config_loader = ConfigLoader("config/config.yaml")
79   - _config = config_loader.load_config()
  77 + _app_config = get_app_config()
80 78 logger.info("Configuration loaded")
81 79  
82 80 # Get ES credentials
83   - es_username = os.getenv("ES_USERNAME") or ES_CONFIG.get("username")
84   - es_password = os.getenv("ES_PASSWORD") or ES_CONFIG.get("password")
  81 + es_username = _app_config.infrastructure.elasticsearch.username
  82 + es_password = _app_config.infrastructure.elasticsearch.password
85 83  
86 84 # Connect to Elasticsearch
87 85 logger.info(f"Connecting to Elasticsearch at {es_host} for indexer...")
... ... @@ -97,11 +95,12 @@ def init_indexer_service(es_host: str = "http://localhost:9200"):
97 95 set_es_client(_es_client)
98 96  
99 97 # Initialize indexing services (DB is required here)
100   - db_host = os.getenv("DB_HOST")
101   - db_port = int(os.getenv("DB_PORT", 3306))
102   - db_database = os.getenv("DB_DATABASE")
103   - db_username = os.getenv("DB_USERNAME")
104   - db_password = os.getenv("DB_PASSWORD")
  98 + db_config = _app_config.infrastructure.database
  99 + db_host = db_config.host
  100 + db_port = db_config.port
  101 + db_database = db_config.database
  102 + db_username = db_config.username
  103 + db_password = db_config.password
105 104  
106 105 if all([db_host, db_database, db_username, db_password]):
107 106 logger.info("Initializing database connection for indexing services...")
... ... @@ -166,7 +165,7 @@ async def startup_event():
166 165 except Exception as e:
167 166 logger.warning(f"Failed to set thread pool size: {e}, using default")
168 167  
169   - es_host = os.getenv("ES_HOST", "http://localhost:9200")
  168 + es_host = get_app_config().infrastructure.elasticsearch.host
170 169 logger.info("Starting Indexer API service")
171 170 logger.info(f"Elasticsearch Host: {es_host}")
172 171 try:
... ... @@ -176,14 +175,11 @@ async def startup_event():
176 175 # Eager warmup: build per-tenant transformer bundles at startup to avoid
177 176 # first-request latency (config/provider/encoder + transformer wiring).
178 177 try:
179   - if _incremental_service is not None and _config is not None:
  178 + if _incremental_service is not None and _app_config is not None:
180 179 tenants = []
181   - # config.tenant_config shape: {"default": {...}, "tenants": {"1": {...}, ...}}
182   - tc = getattr(_config, "tenant_config", None) or {}
183   - if isinstance(tc, dict):
184   - tmap = tc.get("tenants")
185   - if isinstance(tmap, dict):
186   - tenants = [str(k) for k in tmap.keys()]
  180 + tmap = _app_config.tenants.tenants
  181 + if isinstance(tmap, dict):
  182 + tenants = [str(k) for k in tmap.keys()]
187 183 # If no explicit tenants configured, skip warmup.
188 184 if tenants:
189 185 warm = _incremental_service.warmup_transformers(tenants)
... ...
api/routes/admin.py
  Show/Hide comments   View file @86d8358
... ... @@ -42,23 +42,32 @@ async def health_check():
42 42 @router.get("/config")
43 43 async def get_configuration():
44 44 """
45   - Get current search configuration (sanitized).
  45 + Get the effective application configuration (sanitized).
46 46 """
47 47 try:
48 48 from ..app import get_config
49 49  
50   - config = get_config()
  50 + return get_config().sanitized_dict()
  51 +
  52 + except HTTPException:
  53 + raise
  54 + except Exception as e:
  55 + raise HTTPException(status_code=500, detail=str(e))
51 56  
  57 +
  58 +@router.get("/config/meta")
  59 +async def get_configuration_meta():
  60 + """Get configuration metadata for observability."""
  61 + try:
  62 + from ..app import get_config
  63 +
  64 + config = get_config()
52 65 return {
53   - "es_index_name": config.es_index_name,
54   - "num_field_boosts": len(config.field_boosts),
55   - "multilingual_fields": config.query_config.multilingual_fields,
56   - "shared_fields": config.query_config.shared_fields,
57   - "core_multilingual_fields": config.query_config.core_multilingual_fields,
58   - "supported_languages": config.query_config.supported_languages,
59   - "spu_enabled": config.spu_config.enabled
  66 + "environment": config.runtime.environment,
  67 + "config_hash": config.metadata.config_hash,
  68 + "loaded_files": list(config.metadata.loaded_files),
  69 + "deprecated_keys": list(config.metadata.deprecated_keys),
60 70 }
61   -
62 71 except HTTPException:
63 72 raise
64 73 except Exception as e:
... ...
config/__init__.py
  Show/Hide comments   View file @86d8358
1   -"""
2   -Configuration package for search engine.
  1 +"""Unified configuration package exports."""
3 2  
4   -Provides configuration loading, validation, and utility functions.
5   -"""
6   -
7   -from .config_loader import (
8   - SearchConfig,
9   - QueryConfig,
10   - IndexConfig,
11   - SPUConfig,
  3 +from config.config_loader import ConfigLoader, ConfigurationError
  4 +from config.loader import AppConfigLoader, get_app_config, reload_app_config
  5 +from config.schema import (
  6 + AppConfig,
12 7 FunctionScoreConfig,
  8 + IndexConfig,
  9 + QueryConfig,
13 10 RerankConfig,
14   - ConfigLoader,
15   - ConfigurationError
16   -)
17   -
18   -from .utils import (
19   - get_match_fields_for_index,
20   - get_domain_fields
  11 + SPUConfig,
  12 + SearchConfig,
  13 + ServicesConfig,
21 14 )
22   -from .services_config import (
23   - get_translation_config,
24   - get_embedding_config,
25   - get_rerank_config,
  15 +from config.services_config import (
26 16 get_embedding_backend_config,
27   - get_rerank_backend_config,
28   - get_translation_base_url,
29   - get_embedding_text_base_url,
  17 + get_embedding_config,
  18 + get_embedding_image_backend_config,
30 19 get_embedding_image_base_url,
  20 + get_embedding_text_base_url,
  21 + get_rerank_backend_config,
  22 + get_rerank_config,
31 23 get_rerank_service_url,
  24 + get_translation_base_url,
32 25 get_translation_cache_config,
33   - ServiceConfig,
  26 + get_translation_config,
34 27 )
  28 +from config.utils import get_domain_fields, get_match_fields_for_index
35 29  
36 30 __all__ = [
37   - # Main config classes
38   - 'SearchConfig',
39   - 'QueryConfig',
40   - 'IndexConfig',
41   - 'SPUConfig',
42   - 'FunctionScoreConfig',
43   - 'RerankConfig',
44   -
45   - # Loader and utilities
46   - 'ConfigLoader',
47   - 'ConfigurationError',
48   - 'get_match_fields_for_index',
49   - 'get_domain_fields',
50   - 'get_translation_config',
51   - 'get_embedding_config',
52   - 'get_rerank_config',
53   - 'get_embedding_backend_config',
54   - 'get_rerank_backend_config',
55   - 'get_translation_base_url',
56   - 'get_embedding_text_base_url',
57   - 'get_embedding_image_base_url',
58   - 'get_rerank_service_url',
59   - 'get_translation_cache_config',
60   - 'ServiceConfig',
  31 + "AppConfig",
  32 + "AppConfigLoader",
  33 + "ConfigLoader",
  34 + "ConfigurationError",
  35 + "FunctionScoreConfig",
  36 + "IndexConfig",
  37 + "QueryConfig",
  38 + "RerankConfig",
  39 + "SPUConfig",
  40 + "SearchConfig",
  41 + "ServicesConfig",
  42 + "get_app_config",
  43 + "reload_app_config",
  44 + "get_domain_fields",
  45 + "get_match_fields_for_index",
  46 + "get_translation_config",
  47 + "get_embedding_config",
  48 + "get_rerank_config",
  49 + "get_embedding_backend_config",
  50 + "get_embedding_image_backend_config",
  51 + "get_rerank_backend_config",
  52 + "get_translation_base_url",
  53 + "get_embedding_text_base_url",
  54 + "get_embedding_image_base_url",
  55 + "get_rerank_service_url",
  56 + "get_translation_cache_config",
61 57 ]
... ...
config/config.yaml
  Show/Hide comments   View file @86d8358
... ... @@ -5,6 +5,10 @@
5 5 # Elasticsearch Index
6 6 es_index_name: "search_products"
7 7  
  8 +# Config assets
  9 +assets:
  10 + query_rewrite_dictionary_path: "config/dictionaries/query_rewrite.dict"
  11 +
8 12 # ES Index Settings (ๅŸบ็ก€่ฎพ็ฝฎ)
9 13 es_settings:
10 14 number_of_shards: 1
... ... @@ -211,6 +215,19 @@ services:
211 215 device: "cuda"
212 216 batch_size: 32
213 217 normalize_embeddings: true
  218 + # ๆœๅŠกๅ†…ๅ›พ็‰‡ๅŽ็ซฏ๏ผˆembedding ่ฟ›็จ‹ๅฏๅŠจๆ—ถ่ฏปๅ–๏ผ‰
  219 + image_backend: "clip_as_service" # clip_as_service | local_cnclip
  220 + image_backends:
  221 + clip_as_service:
  222 + server: "grpc://127.0.0.1:51000"
  223 + model_name: "CN-CLIP/ViT-L-14"
  224 + batch_size: 8
  225 + normalize_embeddings: true
  226 + local_cnclip:
  227 + model_name: "ViT-L-14"
  228 + device: null
  229 + batch_size: 8
  230 + normalize_embeddings: true
214 231 rerank:
215 232 provider: "http"
216 233 base_url: "http://127.0.0.1:6007"
... ... @@ -218,6 +235,9 @@ services:
218 235 http:
219 236 base_url: "http://127.0.0.1:6007"
220 237 service_url: "http://127.0.0.1:6007/rerank"
  238 + request:
  239 + max_docs: 1000
  240 + normalize: true
221 241 # ๆœๅŠกๅ†…ๅŽ็ซฏ๏ผˆreranker ่ฟ›็จ‹ๅฏๅŠจๆ—ถ่ฏปๅ–๏ผ‰
222 242 backend: "qwen3_vllm" # bge | qwen3_vllm | qwen3_transformers | dashscope_rerank
223 243 backends:
... ...
config/config_loader.py
  Show/Hide comments   View file @86d8358
1 1 """
2   -Configuration loader and validator for search engine configurations.
  2 +Compatibility wrapper for search-behavior config access.
3 3  
4   -This module handles loading, parsing, and validating YAML configuration files
5   -that define how search should be executed (NOT how data should be indexed).
6   -
7   -็ดขๅผ•็ป“ๆž„็”ฑ mappings/search_products.json ๅฎšไน‰ใ€‚
8   -ๆญค้…็ฝฎๅชๅฎšไน‰ๆœ็ดข่กŒไธบ๏ผšๅญ—ๆฎตๆƒ้‡ใ€ๆœ็ดขๅŸŸใ€ๆŸฅ่ฏข็ญ–็•ฅ็ญ‰ใ€‚
  4 +The unified loader lives in :mod:`config.loader`. This module now exposes the
  5 +search subtree only, so existing search/indexer code can consume a single
  6 +source-of-truth search config without reparsing YAML separately.
9 7 """
10 8  
11   -import yaml
12   -from typing import Dict, Any, List, Optional
13   -from dataclasses import dataclass, field
14   -from pathlib import Path
15   -
16   -
17   -@dataclass
18   -class IndexConfig:
19   - """Configuration for an index domain (e.g., default, title, brand)."""
20   - name: str
21   - label: str
22   - fields: List[str] # List of field names to include in this search domain
23   - boost: float = 1.0
24   - example: Optional[str] = None
25   -
26   -
27   -@dataclass
28   -class QueryConfig:
29   - """Configuration for query processing."""
30   - supported_languages: List[str] = field(default_factory=lambda: ["zh", "en"])
31   - default_language: str = "en"
32   -
33   - # Feature flags
34   - enable_text_embedding: bool = True
35   - enable_query_rewrite: bool = True
36   -
37   - # Query rewrite dictionary (loaded from external file)
38   - rewrite_dictionary: Dict[str, str] = field(default_factory=dict)
39   -
40   - # Embedding field names
41   - text_embedding_field: Optional[str] = "title_embedding"
42   - image_embedding_field: Optional[str] = None
43   -
44   - # Source fields configuration
45   - source_fields: Optional[List[str]] = None
46   -
47   - # KNN boost configuration
48   - knn_boost: float = 0.25 # Boost value for KNN (embedding recall)
49   -
50   - # Dynamic text fields for multi-language retrieval
51   - multilingual_fields: List[str] = field(
52   - default_factory=lambda: ["title", "brief", "description", "vendor", "category_path", "category_name_text"]
53   - )
54   - shared_fields: List[str] = field(
55   - default_factory=lambda: ["tags", "option1_values", "option2_values", "option3_values"]
56   - )
57   - core_multilingual_fields: List[str] = field(
58   - default_factory=lambda: ["title", "brief", "vendor", "category_name_text"]
59   - )
60   -
61   - # Unified text strategy tuning
62   - base_minimum_should_match: str = "75%"
63   - translation_minimum_should_match: str = "75%"
64   - translation_boost: float = 0.4
65   - translation_boost_when_source_missing: float = 1.0
66   - source_boost_when_missing: float = 0.6
67   - original_query_fallback_boost_when_translation_missing: float = 0.2
68   - tie_breaker_base_query: float = 0.9
69   -
70   - # Query-time translation model selection (configurable)
71   - # - zh_to_en_model: model for zh -> en
72   - # - en_to_zh_model: model for en -> zh
73   - # - default_translation_model: fallback model for all other language pairs
74   - zh_to_en_model: str = "opus-mt-zh-en"
75   - en_to_zh_model: str = "opus-mt-en-zh"
76   - default_translation_model: str = "nllb-200-distilled-600m"
77   -
78   -
79   -@dataclass
80   -class SPUConfig:
81   - """Configuration for SPU aggregation."""
82   - enabled: bool = False
83   - spu_field: Optional[str] = None
84   - inner_hits_size: int = 3
85   - # ้…็ฝฎๅ“ชไบ›option็ปดๅบฆๅ‚ไธŽๆฃ€็ดข๏ผˆ่ฟ›็ดขๅผ•ใ€ไปฅๅŠๅœจ็บฟๆœ็ดข๏ผ‰
86   - searchable_option_dimensions: List[str] = field(default_factory=lambda: ['option1', 'option2', 'option3'])
87   -
88   -
89   -@dataclass
90   -class FunctionScoreConfig:
91   - """Function Score้…็ฝฎ๏ผˆESๅฑ‚ๆ‰“ๅˆ†่ง„ๅˆ™๏ผ‰"""
92   - score_mode: str = "sum"
93   - boost_mode: str = "multiply"
94   - functions: List[Dict[str, Any]] = field(default_factory=list)
95   -
96   -
97   -@dataclass
98   -class RerankConfig:
99   - """้‡ๆŽ’้…็ฝฎ๏ผˆprovider/URL ๅœจ services.rerank๏ผ‰"""
100   - enabled: bool = True
101   - rerank_window: int = 384
102   - timeout_sec: float = 15.0
103   - weight_es: float = 0.4
104   - weight_ai: float = 0.6
105   - rerank_query_template: str = "{query}"
106   - rerank_doc_template: str = "{title}"
107   -
108   -
109   -@dataclass
110   -class SearchConfig:
111   - """Complete configuration for search engine (multi-tenant)."""
112   -
113   - # ๅญ—ๆฎตๆƒ้‡้…็ฝฎ๏ผˆ็”จไบŽๆœ็ดข๏ผ‰
114   - field_boosts: Dict[str, float]
115   -
116   - # Legacy index domains (deprecated; kept for compatibility)
117   - indexes: List[IndexConfig]
118   -
119   - # Query processing
120   - query_config: QueryConfig
121   -
122   - # Function Score configuration (ESๅฑ‚ๆ‰“ๅˆ†)
123   - function_score: FunctionScoreConfig
124   -
125   - # Rerank configuration (ๆœฌๅœฐ้‡ๆŽ’)
126   - rerank: RerankConfig
127   -
128   - # SPU configuration
129   - spu_config: SPUConfig
130   -
131   - # ES index settings
132   - es_index_name: str
133   -
134   - # Tenant configuration
135   - tenant_config: Dict[str, Any] = field(default_factory=dict)
136   -
137   - # ES settings
138   - es_settings: Dict[str, Any] = field(default_factory=dict)
139   - # Extensible service/provider registry (translation/embedding/rerank/...)
140   - services: Dict[str, Any] = field(default_factory=dict)
  9 +from __future__ import annotations
141 10  
  11 +from dataclasses import asdict
  12 +from pathlib import Path
  13 +from typing import Any, Dict, List, Optional
142 14  
143   -class ConfigurationError(Exception):
144   - """Raised when configuration validation fails."""
145   - pass
  15 +from config.loader import AppConfigLoader, ConfigurationError
  16 +from config.schema import (
  17 + FunctionScoreConfig,
  18 + IndexConfig,
  19 + QueryConfig,
  20 + RerankConfig,
  21 + SPUConfig,
  22 + SearchConfig,
  23 +)
146 24  
147 25  
148 26 class ConfigLoader:
149   - """Loads and validates unified search engine configuration from YAML file."""
150   -
151   - def __init__(self, config_file: Optional[Path] = None):
152   - """
153   - Initialize config loader.
154   -
155   - Args:
156   - config_file: Path to config YAML file (defaults to config/config.yaml)
157   - """
158   - if config_file is None:
159   - config_file = Path(__file__).parent / "config.yaml"
160   - self.config_file = Path(config_file)
161   -
162   - def _load_rewrite_dictionary(self) -> Dict[str, str]:
163   - """Load query rewrite dictionary from external file."""
164   - rewrite_file = Path(__file__).parent / "rewrite_dictionary.txt"
165   - rewrite_dict = {}
166   -
167   - if not rewrite_file.exists():
168   - return rewrite_dict
169   -
170   - try:
171   - with open(rewrite_file, 'r', encoding='utf-8') as f:
172   - for line in f:
173   - line = line.strip()
174   - if not line or line.startswith('#'):
175   - continue
176   -
177   - parts = line.split('\t')
178   - if len(parts) >= 2:
179   - original = parts[0].strip()
180   - replacement = parts[1].strip()
181   - if original and replacement:
182   - rewrite_dict[original] = replacement
183   - except Exception as e:
184   - print(f"Warning: Failed to load rewrite dictionary: {e}")
185   -
186   - return rewrite_dict
187   -
188   - def load_config(self, validate: bool = True) -> SearchConfig:
189   - """
190   - Load unified configuration from YAML file.
191   -
192   - Args:
193   - validate: Whether to validate configuration after loading
194   -
195   - Returns:
196   - SearchConfig object
197   -
198   - Raises:
199   - ConfigurationError: If config file not found, invalid, or validation fails
200   - """
201   - if not self.config_file.exists():
202   - raise ConfigurationError(f"Configuration file not found: {self.config_file}")
203   -
204   - try:
205   - with open(self.config_file, 'r', encoding='utf-8') as f:
206   - config_data = yaml.safe_load(f)
207   - except yaml.YAMLError as e:
208   - raise ConfigurationError(f"Invalid YAML in {self.config_file}: {e}")
209   -
210   - config = self._parse_config(config_data)
211   -
212   - # Auto-validate configuration
213   - if validate:
214   - errors = self.validate_config(config)
215   - if errors:
216   - error_msg = "Configuration validation failed:\n" + "\n".join(f" - {err}" for err in errors)
217   - raise ConfigurationError(error_msg)
218   -
219   - return config
220   -
221   - def _parse_config(self, config_data: Dict[str, Any]) -> SearchConfig:
222   - """Parse configuration dictionary into SearchConfig object."""
223   -
224   - # Parse field_boosts
225   - field_boosts = config_data.get("field_boosts", {})
226   - if not isinstance(field_boosts, dict):
227   - raise ConfigurationError("field_boosts must be a dictionary")
228   -
229   - # Parse indexes (deprecated; compatibility only)
230   - indexes = []
231   - for index_data in config_data.get("indexes", []):
232   - indexes.append(self._parse_index_config(index_data))
233   -
234   - # Parse query config
235   - query_config_data = config_data.get("query_config", {})
236   - rewrite_dictionary = self._load_rewrite_dictionary()
237   - search_fields_cfg = query_config_data.get("search_fields", {})
238   - text_strategy_cfg = query_config_data.get("text_query_strategy", {})
  27 + """Load the unified app config and return the search subtree."""
239 28  
240   - query_config = QueryConfig(
241   - supported_languages=query_config_data.get("supported_languages") or ["zh", "en"],
242   - default_language=query_config_data.get("default_language") or "en",
243   - enable_text_embedding=query_config_data.get("enable_text_embedding", True),
244   - enable_query_rewrite=query_config_data.get("enable_query_rewrite", True),
245   - rewrite_dictionary=rewrite_dictionary,
246   - text_embedding_field=query_config_data.get("text_embedding_field"),
247   - image_embedding_field=query_config_data.get("image_embedding_field"),
248   - source_fields=query_config_data.get("source_fields"),
249   - knn_boost=query_config_data.get("knn_boost", 0.25),
250   - multilingual_fields=search_fields_cfg.get(
251   - "multilingual_fields",
252   - ["title", "brief", "description", "vendor", "category_path", "category_name_text"],
253   - ),
254   - shared_fields=search_fields_cfg.get(
255   - "shared_fields",
256   - ["tags", "option1_values", "option2_values", "option3_values"],
257   - ),
258   - core_multilingual_fields=search_fields_cfg.get(
259   - "core_multilingual_fields",
260   - ["title", "brief", "vendor", "category_name_text"],
261   - ),
262   - base_minimum_should_match=str(text_strategy_cfg.get("base_minimum_should_match", "75%")),
263   - translation_minimum_should_match=str(text_strategy_cfg.get("translation_minimum_should_match", "75%")),
264   - translation_boost=float(text_strategy_cfg.get("translation_boost", 0.4)),
265   - translation_boost_when_source_missing=float(
266   - text_strategy_cfg.get("translation_boost_when_source_missing", 1.0)
267   - ),
268   - source_boost_when_missing=float(text_strategy_cfg.get("source_boost_when_missing", 0.6)),
269   - original_query_fallback_boost_when_translation_missing=float(
270   - text_strategy_cfg.get("original_query_fallback_boost_when_translation_missing", 0.2)
271   - ),
272   - tie_breaker_base_query=float(text_strategy_cfg.get("tie_breaker_base_query", 0.9)),
273   - zh_to_en_model=str(query_config_data.get("zh_to_en_model") or "opus-mt-zh-en"),
274   - en_to_zh_model=str(query_config_data.get("en_to_zh_model") or "opus-mt-en-zh"),
275   - default_translation_model=str(
276   - query_config_data.get("default_translation_model") or "nllb-200-distilled-600m"
277   - ),
278   - )
279   -
280   - # Parse Function Score configuration
281   - fs_data = config_data.get("function_score", {})
282   - function_score = FunctionScoreConfig(
283   - score_mode=fs_data.get("score_mode") or "sum",
284   - boost_mode=fs_data.get("boost_mode") or "multiply",
285   - functions=fs_data.get("functions") or []
286   - )
287   -
288   - # Parse Rerank (provider/URL in services.rerank)
289   - rerank_data = config_data.get("rerank", {})
290   - rerank = RerankConfig(
291   - enabled=bool(rerank_data.get("enabled", True)),
292   - rerank_window=int(rerank_data.get("rerank_window", 384)),
293   - timeout_sec=float(rerank_data.get("timeout_sec", 15.0)),
294   - weight_es=float(rerank_data.get("weight_es", 0.4)),
295   - weight_ai=float(rerank_data.get("weight_ai", 0.6)),
296   - rerank_query_template=str(rerank_data.get("rerank_query_template") or "{query}"),
297   - rerank_doc_template=str(rerank_data.get("rerank_doc_template") or "{title}"),
298   - )
299   -
300   - # Parse SPU config
301   - spu_data = config_data.get("spu_config", {})
302   - spu_config = SPUConfig(
303   - enabled=spu_data.get("enabled", False),
304   - spu_field=spu_data.get("spu_field"),
305   - inner_hits_size=spu_data.get("inner_hits_size", 3),
306   - searchable_option_dimensions=spu_data.get("searchable_option_dimensions", ['option1', 'option2', 'option3'])
307   - )
308   -
309   - # Parse tenant config
310   - tenant_config_data = config_data.get("tenant_config", {})
  29 + def __init__(self, config_file: Optional[Path] = None) -> None:
  30 + self._loader = AppConfigLoader(config_file=Path(config_file) if config_file is not None else None)
311 31  
312   - # Parse extensible services/provider registry
313   - services_data = config_data.get("services", {}) or {}
314   - if not isinstance(services_data, dict):
315   - raise ConfigurationError("services must be a dictionary if provided")
  32 + def load_config(self, validate: bool = True) -> SearchConfig:
  33 + return self._loader.load(validate=validate).search
316 34  
317   - return SearchConfig(
318   - field_boosts=field_boosts,
319   - indexes=indexes,
320   - query_config=query_config,
321   - function_score=function_score,
322   - rerank=rerank,
323   - spu_config=spu_config,
324   - tenant_config=tenant_config_data,
325   - es_index_name=config_data.get("es_index_name", "search_products"),
326   - es_settings=config_data.get("es_settings", {}),
327   - services=services_data
328   - )
329   -
330   - def _parse_index_config(self, index_data: Dict[str, Any]) -> IndexConfig:
331   - """Parse index configuration from dictionary."""
332   - return IndexConfig(
333   - name=index_data["name"],
334   - label=index_data.get("label", index_data["name"]),
335   - fields=index_data.get("fields", []),
336   - boost=index_data.get("boost", 1.0),
337   - example=index_data.get("example")
338   - )
339   -
340 35 def validate_config(self, config: SearchConfig) -> List[str]:
341   - """
342   - Validate configuration for common errors.
343   -
344   - Args:
345   - config: SearchConfig to validate
346   -
347   - Returns:
348   - List of error messages (empty if valid)
349   - """
350   - errors = []
351   -
352   - # Validate es_index_name
  36 + errors: List[str] = []
353 37 if not config.es_index_name:
354 38 errors.append("es_index_name is required")
355   -
356   - # Validate field_boosts
357 39 if not config.field_boosts:
358 40 errors.append("field_boosts is empty")
359   -
360   - for field_name, boost in config.field_boosts.items():
361   - if not isinstance(boost, (int, float)):
362   - errors.append(f"field_boosts['{field_name}']: boost must be a number, got {type(boost).__name__}")
363   - elif boost < 0:
364   - errors.append(f"field_boosts['{field_name}']: boost must be non-negative")
365   -
366   - # Validate indexes (deprecated, optional)
367   - index_names = set()
368   - for index in config.indexes:
369   - # Check for duplicate index names
370   - if index.name in index_names:
371   - errors.append(f"Duplicate index name: {index.name}")
372   - index_names.add(index.name)
373   -
374   - # Validate fields in index
375   - if not index.fields:
376   - errors.append(f"Index '{index.name}': fields list is empty")
377   -
378   - # Validate SPU config
379   - if config.spu_config.enabled:
380   - if not config.spu_config.spu_field:
381   - errors.append("SPU aggregation enabled but no spu_field specified")
382   -
383   - # Validate query config
384   - if not config.query_config.supported_languages:
385   - errors.append("At least one supported language must be specified")
386   -
387 41 if config.query_config.default_language not in config.query_config.supported_languages:
388   - errors.append(
389   - f"Default language '{config.query_config.default_language}' "
390   - f"not in supported languages: {config.query_config.supported_languages}"
391   - )
392   -
393   - # Validate dynamic search fields
394   - def _validate_str_list(name: str, values: List[str]) -> None:
395   - if not isinstance(values, list) or not values:
396   - errors.append(f"query_config.{name} must be a non-empty list[str]")
397   - return
398   - for i, val in enumerate(values):
399   - if not isinstance(val, str) or not val.strip():
400   - errors.append(f"query_config.{name}[{i}] must be a non-empty string")
401   -
402   - _validate_str_list("multilingual_fields", config.query_config.multilingual_fields)
403   - _validate_str_list("shared_fields", config.query_config.shared_fields)
404   - _validate_str_list("core_multilingual_fields", config.query_config.core_multilingual_fields)
405   -
406   - core_set = set(config.query_config.core_multilingual_fields)
407   - multi_set = set(config.query_config.multilingual_fields)
408   - if not core_set.issubset(multi_set):
409   - errors.append("query_config.core_multilingual_fields must be subset of multilingual_fields")
410   -
411   - # Validate text query strategy numbers
412   - for name in (
413   - "translation_boost",
414   - "translation_boost_when_source_missing",
415   - "source_boost_when_missing",
416   - "original_query_fallback_boost_when_translation_missing",
417   - "tie_breaker_base_query",
418   - ):
419   - value = getattr(config.query_config, name, None)
420   - if not isinstance(value, (int, float)):
421   - errors.append(f"query_config.{name} must be a number")
422   - elif value < 0:
423   - errors.append(f"query_config.{name} must be non-negative")
424   -
425   - # Validate source_fields tri-state semantics
426   - source_fields = config.query_config.source_fields
427   - if source_fields is not None:
428   - if not isinstance(source_fields, list):
429   - errors.append("query_config.source_fields must be null or list[str]")
430   - else:
431   - for idx, field_name in enumerate(source_fields):
432   - if not isinstance(field_name, str) or not field_name.strip():
433   - errors.append(
434   - f"query_config.source_fields[{idx}] must be a non-empty string"
435   - )
436   -
437   - # Validate tenant config shape (default must exist in config)
438   - tenant_cfg = config.tenant_config
439   - if not isinstance(tenant_cfg, dict):
440   - errors.append("tenant_config must be an object")
441   - else:
442   - default_cfg = tenant_cfg.get("default")
443   - if not isinstance(default_cfg, dict):
444   - errors.append("tenant_config.default must be configured")
445   - else:
446   - index_languages = default_cfg.get("index_languages")
447   - if not isinstance(index_languages, list) or len(index_languages) == 0:
448   - errors.append("tenant_config.default.index_languages must be a non-empty list")
449   -
  42 + errors.append("default_language must be included in supported_languages")
  43 + if config.spu_config.enabled and not config.spu_config.spu_field:
  44 + errors.append("spu_field is required when SPU is enabled")
450 45 return errors
451   -
  46 +
452 47 def to_dict(self, config: SearchConfig) -> Dict[str, Any]:
453   - """Convert SearchConfig to dictionary representation."""
454   -
455   - # Build query_config dict
456   - query_config_dict = {
457   - "supported_languages": config.query_config.supported_languages,
458   - "default_language": config.query_config.default_language,
459   - "enable_text_embedding": config.query_config.enable_text_embedding,
460   - "enable_query_rewrite": config.query_config.enable_query_rewrite,
461   - "text_embedding_field": config.query_config.text_embedding_field,
462   - "image_embedding_field": config.query_config.image_embedding_field,
463   - "source_fields": config.query_config.source_fields,
464   - "search_fields": {
465   - "multilingual_fields": config.query_config.multilingual_fields,
466   - "shared_fields": config.query_config.shared_fields,
467   - "core_multilingual_fields": config.query_config.core_multilingual_fields,
468   - },
469   - "text_query_strategy": {
470   - "base_minimum_should_match": config.query_config.base_minimum_should_match,
471   - "translation_minimum_should_match": config.query_config.translation_minimum_should_match,
472   - "translation_boost": config.query_config.translation_boost,
473   - "translation_boost_when_source_missing": config.query_config.translation_boost_when_source_missing,
474   - "source_boost_when_missing": config.query_config.source_boost_when_missing,
475   - "original_query_fallback_boost_when_translation_missing": (
476   - config.query_config.original_query_fallback_boost_when_translation_missing
477   - ),
478   - "tie_breaker_base_query": config.query_config.tie_breaker_base_query,
479   - }
480   - }
481   -
482   - return {
483   - "es_index_name": config.es_index_name,
484   - "es_settings": config.es_settings,
485   - "field_boosts": config.field_boosts,
486   - "indexes": [self._index_to_dict(index) for index in config.indexes],
487   - "query_config": query_config_dict,
488   - "function_score": {
489   - "score_mode": config.function_score.score_mode,
490   - "boost_mode": config.function_score.boost_mode,
491   - "functions": config.function_score.functions
492   - },
493   - "rerank": {
494   - "enabled": config.rerank.enabled,
495   - "rerank_window": config.rerank.rerank_window,
496   - "timeout_sec": config.rerank.timeout_sec,
497   - "weight_es": config.rerank.weight_es,
498   - "weight_ai": config.rerank.weight_ai,
499   - "rerank_query_template": config.rerank.rerank_query_template,
500   - "rerank_doc_template": config.rerank.rerank_doc_template,
501   - },
502   - "spu_config": {
503   - "enabled": config.spu_config.enabled,
504   - "spu_field": config.spu_config.spu_field,
505   - "inner_hits_size": config.spu_config.inner_hits_size,
506   - "searchable_option_dimensions": config.spu_config.searchable_option_dimensions
507   - },
508   - "services": config.services,
509   - }
510   -
511   - def _index_to_dict(self, index: IndexConfig) -> Dict[str, Any]:
512   - """Convert IndexConfig to dictionary."""
513   - result = {
514   - "name": index.name,
515   - "label": index.label,
516   - "fields": index.fields,
517   - "boost": index.boost
518   - }
519   -
520   - if index.example:
521   - result["example"] = index.example
522   -
523   - return result
  48 + return asdict(config)
... ...
config/dictionaries/query_rewrite.dict 0 โ†’ 100644
  Show/Hide comments   View file @86d8358
... ... @@ -0,0 +1,2 @@
  1 +็Žฉๅ…ท category.keyword:็Žฉๅ…ท OR default:็Žฉๅ…ท
  2 +ๆถˆ้˜ฒ category.keyword:ๆถˆ้˜ฒ OR default:ๆถˆ้˜ฒ
... ...
config/env_config.py
  Show/Hide comments   View file @86d8358
1 1 """
2   -Centralized configuration management for saas-search.
  2 +Compatibility accessors for infrastructure/runtime environment settings.
3 3  
4   -Loads configuration from environment variables and .env file.
5   -This module provides a single point for loading .env and setting defaults.
6   -All configuration variables are exported directly - no need for getter functions.
  4 +All values are derived from the unified application config. This module no
  5 +longer owns any independent loading or precedence rules.
7 6 """
8 7  
9   -import os
10   -from pathlib import Path
11   -from dotenv import load_dotenv
12   -
13   -# Load .env file from project root
14   -PROJECT_ROOT = Path(__file__).parent.parent
15   -load_dotenv(PROJECT_ROOT / '.env')
16   -
17   -
18   -# Elasticsearch Configuration
19   -ES_CONFIG = {
20   - 'host': os.getenv('ES_HOST', 'http://localhost:9200'),
21   - 'username': os.getenv('ES_USERNAME'),
22   - 'password': os.getenv('ES_PASSWORD'),
23   -}
24   -
25   -# Runtime environment & index namespace
26   -# RUNTIME_ENV: ๅฝ“ๅ‰่ฟ่กŒ็Žฏๅขƒ๏ผŒๅปบ่ฎฎไฝฟ็”จ prod / uat / test / dev ็ญ‰ๆžšไธพๅ€ผ
27   -RUNTIME_ENV = os.getenv('RUNTIME_ENV', 'prod')
28   -# ES_INDEX_NAMESPACE: ็”จไบŽๆŒ‰็Žฏๅขƒ้š”็ฆป็ดขๅผ•็š„ๅ‘ฝๅ็ฉบ้—ดๅ‰็ผ€๏ผŒไพ‹ๅฆ‚ "uat_" / "test_"
29   -# ไธบ็ฉบๅญ—็ฌฆไธฒๆ—ถ่กจ็คบไธๅŠ ๅ‰็ผ€๏ผˆ้€šๅธธๆ˜ฏ prod ็Žฏๅขƒ๏ผ‰
30   -ES_INDEX_NAMESPACE = os.getenv('ES_INDEX_NAMESPACE')
31   -if ES_INDEX_NAMESPACE is None:
32   - # ๆœชๆ˜พๅผ้…็ฝฎๆ—ถ๏ผŒ้ž prod ็Žฏๅขƒ้ป˜่ฎคๅŠ  "<env>_" ๅ‰็ผ€๏ผŒprod ็Žฏๅขƒ้ป˜่ฎคไธๅŠ ๅ‰็ผ€
33   - ES_INDEX_NAMESPACE = '' if RUNTIME_ENV == 'prod' else f'{RUNTIME_ENV}_'
34   -
35   -# Redis Configuration
36   -REDIS_CONFIG = {
37   - 'host': os.getenv('REDIS_HOST', 'localhost'),
38   - 'port': int(os.getenv('REDIS_PORT', 6479)),
39   - 'snapshot_db': int(os.getenv('REDIS_SNAPSHOT_DB', 0)),
40   - 'password': os.getenv('REDIS_PASSWORD'),
41   - 'socket_timeout': int(os.getenv('REDIS_SOCKET_TIMEOUT', 1)),
42   - 'socket_connect_timeout': int(os.getenv('REDIS_SOCKET_CONNECT_TIMEOUT', 1)),
43   - 'retry_on_timeout': os.getenv('REDIS_RETRY_ON_TIMEOUT', 'False').lower() == 'true',
44   - 'cache_expire_days': int(os.getenv('REDIS_CACHE_EXPIRE_DAYS', 360*2)), # 6 months
45   - # Embedding ็ผ“ๅญ˜ key ๅ‰็ผ€๏ผŒไพ‹ๅฆ‚ "embedding"
46   - 'embedding_cache_prefix': os.getenv('REDIS_EMBEDDING_CACHE_PREFIX', 'embedding'),
47   -}
48   -
49   -# DeepL API Key
50   -DEEPL_AUTH_KEY = os.getenv('DEEPL_AUTH_KEY')
51   -
52   -# DashScope API Key (for Qwen models)
53   -DASHSCOPE_API_KEY = os.getenv('DASHSCOPE_API_KEY')
54   -
55   -# API Service Configuration
56   -API_HOST = os.getenv('API_HOST', '0.0.0.0')
57   -API_PORT = int(os.getenv('API_PORT', 6002))
58   -# Indexer service
59   -INDEXER_HOST = os.getenv('INDEXER_HOST', '0.0.0.0')
60   -INDEXER_PORT = int(os.getenv('INDEXER_PORT', 6004))
61   -# Optional dependent services
62   -# EMBEDDING_HOST / EMBEDDING_PORT are only used by the optional combined embedding mode.
63   -EMBEDDING_HOST = os.getenv('EMBEDDING_HOST', '127.0.0.1')
64   -EMBEDDING_PORT = int(os.getenv('EMBEDDING_PORT', 6005))
65   -EMBEDDING_TEXT_HOST = os.getenv('EMBEDDING_TEXT_HOST', '127.0.0.1')
66   -EMBEDDING_TEXT_PORT = int(os.getenv('EMBEDDING_TEXT_PORT', 6005))
67   -EMBEDDING_IMAGE_HOST = os.getenv('EMBEDDING_IMAGE_HOST', '127.0.0.1')
68   -EMBEDDING_IMAGE_PORT = int(os.getenv('EMBEDDING_IMAGE_PORT', 6008))
69   -TRANSLATION_HOST = os.getenv('TRANSLATION_HOST', '127.0.0.1')
70   -TRANSLATION_PORT = int(os.getenv('TRANSLATION_PORT', 6006))
71   -RERANKER_HOST = os.getenv('RERANKER_HOST', '127.0.0.1')
72   -RERANKER_PORT = int(os.getenv('RERANKER_PORT', 6007))
73   -RERANK_PROVIDER = os.getenv('RERANK_PROVIDER', 'http')
74   -# API_BASE_URL: ๅฆ‚ๆžœๆœช่ฎพ็ฝฎ๏ผŒๆ นๆฎAPI_HOSTๆž„ๅปบ๏ผˆ0.0.0.0ไฝฟ็”จlocalhost๏ผ‰
75   -API_BASE_URL = os.getenv('API_BASE_URL')
76   -if not API_BASE_URL:
77   - API_BASE_URL = f'http://localhost:{API_PORT}' if API_HOST == '0.0.0.0' else f'http://{API_HOST}:{API_PORT}'
78   -INDEXER_BASE_URL = os.getenv('INDEXER_BASE_URL') or (
79   - f'http://localhost:{INDEXER_PORT}' if INDEXER_HOST == '0.0.0.0' else f'http://{INDEXER_HOST}:{INDEXER_PORT}'
80   -)
81   -EMBEDDING_TEXT_SERVICE_URL = os.getenv('EMBEDDING_TEXT_SERVICE_URL') or (
82   - f'http://{EMBEDDING_TEXT_HOST}:{EMBEDDING_TEXT_PORT}'
83   -)
84   -EMBEDDING_IMAGE_SERVICE_URL = os.getenv('EMBEDDING_IMAGE_SERVICE_URL') or (
85   - f'http://{EMBEDDING_IMAGE_HOST}:{EMBEDDING_IMAGE_PORT}'
86   -)
87   -RERANKER_SERVICE_URL = os.getenv('RERANKER_SERVICE_URL') or f'http://{RERANKER_HOST}:{RERANKER_PORT}/rerank'
  8 +from __future__ import annotations
  9 +
  10 +from typing import Any, Dict
  11 +
  12 +from config.loader import get_app_config
  13 +
  14 +
  15 +def _app():
  16 + return get_app_config()
  17 +
  18 +
  19 +def _runtime():
  20 + return _app().runtime
88 21  
89   -# Model IDs / paths
90   -TEXT_MODEL_DIR = os.getenv('TEXT_MODEL_DIR', os.getenv('TEXT_MODEL_ID', 'Qwen/Qwen3-Embedding-0.6B'))
91   -IMAGE_MODEL_DIR = os.getenv('IMAGE_MODEL_DIR', '/data/tw/models/cn-clip')
92 22  
93   -# Cache Directory
94   -CACHE_DIR = os.getenv('CACHE_DIR', '.cache')
  23 +def _infra():
  24 + return _app().infrastructure
95 25  
96   -# MySQL Database Configuration (Shoplazza)
97   -DB_CONFIG = {
98   - 'host': os.getenv('DB_HOST'),
99   - 'port': int(os.getenv('DB_PORT', 3306)) if os.getenv('DB_PORT') else 3306,
100   - 'database': os.getenv('DB_DATABASE'),
101   - 'username': os.getenv('DB_USERNAME'),
102   - 'password': os.getenv('DB_PASSWORD'),
103   -}
104 26  
  27 +def _elasticsearch_dict() -> Dict[str, Any]:
  28 + cfg = _infra().elasticsearch
  29 + return {
  30 + "host": cfg.host,
  31 + "username": cfg.username,
  32 + "password": cfg.password,
  33 + }
105 34  
106   -def print_config():
107   - """Print current configuration (with sensitive data masked)."""
108   - print("=" * 60)
109   - print("saas-search Configuration")
110   - print("=" * 60)
111 35  
112   - print("\nElasticsearch:")
113   - print(f" Host: {ES_CONFIG['host']}")
114   - print(f" Username: {ES_CONFIG['username']}")
115   - print(f" Password: {'*' * 10 if ES_CONFIG['password'] else 'None'}")
  36 +def _redis_dict() -> Dict[str, Any]:
  37 + cfg = _infra().redis
  38 + return {
  39 + "host": cfg.host,
  40 + "port": cfg.port,
  41 + "snapshot_db": cfg.snapshot_db,
  42 + "password": cfg.password,
  43 + "socket_timeout": cfg.socket_timeout,
  44 + "socket_connect_timeout": cfg.socket_connect_timeout,
  45 + "retry_on_timeout": cfg.retry_on_timeout,
  46 + "cache_expire_days": cfg.cache_expire_days,
  47 + "embedding_cache_prefix": cfg.embedding_cache_prefix,
  48 + "anchor_cache_prefix": cfg.anchor_cache_prefix,
  49 + "anchor_cache_expire_days": cfg.anchor_cache_expire_days,
  50 + }
116 51  
117   - print("\nRedis:")
118   - print(f" Host: {REDIS_CONFIG['host']}")
119   - print(f" Port: {REDIS_CONFIG['port']}")
120   - print(f" Password: {'*' * 10 if REDIS_CONFIG['password'] else 'None'}")
121 52  
122   - print("\nDeepL:")
123   - print(f" API Key: {'*' * 10 if DEEPL_AUTH_KEY else 'None (translation disabled)'}")
  53 +def _db_dict() -> Dict[str, Any]:
  54 + cfg = _infra().database
  55 + return {
  56 + "host": cfg.host,
  57 + "port": cfg.port,
  58 + "database": cfg.database,
  59 + "username": cfg.username,
  60 + "password": cfg.password,
  61 + }
  62 +
  63 +
  64 +ES_CONFIG = _elasticsearch_dict()
  65 +REDIS_CONFIG = _redis_dict()
  66 +DB_CONFIG = _db_dict()
  67 +
  68 +RUNTIME_ENV = _runtime().environment
  69 +ES_INDEX_NAMESPACE = _runtime().index_namespace
  70 +
  71 +DEEPL_AUTH_KEY = _infra().secrets.deepl_auth_key
  72 +DASHSCOPE_API_KEY = _infra().secrets.dashscope_api_key
  73 +
  74 +API_HOST = _runtime().api_host
  75 +API_PORT = _runtime().api_port
  76 +INDEXER_HOST = _runtime().indexer_host
  77 +INDEXER_PORT = _runtime().indexer_port
  78 +EMBEDDING_HOST = _runtime().embedding_host
  79 +EMBEDDING_PORT = _runtime().embedding_port
  80 +EMBEDDING_TEXT_HOST = _runtime().embedding_host
  81 +EMBEDDING_TEXT_PORT = _runtime().embedding_text_port
  82 +EMBEDDING_IMAGE_HOST = _runtime().embedding_host
  83 +EMBEDDING_IMAGE_PORT = _runtime().embedding_image_port
  84 +TRANSLATION_HOST = _runtime().translator_host
  85 +TRANSLATION_PORT = _runtime().translator_port
  86 +RERANKER_HOST = _runtime().reranker_host
  87 +RERANKER_PORT = _runtime().reranker_port
  88 +
  89 +API_BASE_URL = f"http://localhost:{API_PORT}" if API_HOST == "0.0.0.0" else f"http://{API_HOST}:{API_PORT}"
  90 +INDEXER_BASE_URL = (
  91 + f"http://localhost:{INDEXER_PORT}" if INDEXER_HOST == "0.0.0.0" else f"http://{INDEXER_HOST}:{INDEXER_PORT}"
  92 +)
  93 +EMBEDDING_TEXT_SERVICE_URL = _app().services.embedding.get_provider_config().get("text_base_url")
  94 +EMBEDDING_IMAGE_SERVICE_URL = _app().services.embedding.get_provider_config().get("image_base_url")
  95 +RERANKER_SERVICE_URL = (
  96 + _app().services.rerank.get_provider_config().get("service_url")
  97 + or _app().services.rerank.get_provider_config().get("base_url")
  98 +)
  99 +
124 100  
125   - print("\nAPI Service:")
126   - print(f" Host: {API_HOST}")
127   - print(f" Port: {API_PORT}")
  101 +def get_es_config() -> Dict[str, Any]:
  102 + return dict(ES_CONFIG)
128 103  
129   - print("\nModels:")
130   - print(f" Text Model: {TEXT_MODEL_DIR}")
131   - print(f" Image Model: {IMAGE_MODEL_DIR}")
132 104  
133   - print("\nCache:")
134   - print(f" Cache Directory: {CACHE_DIR}")
  105 +def get_db_config() -> Dict[str, Any]:
  106 + return dict(DB_CONFIG)
135 107  
136   - print("\nMySQL Database:")
137   - print(f" Host: {DB_CONFIG['host']}")
138   - print(f" Port: {DB_CONFIG['port']}")
139   - print(f" Database: {DB_CONFIG['database']}")
140   - print(f" Username: {DB_CONFIG['username']}")
141   - print(f" Password: {'*' * 10 if DB_CONFIG['password'] else 'None'}")
142 108  
143   - print("=" * 60)
  109 +def get_redis_config() -> Dict[str, Any]:
  110 + return dict(REDIS_CONFIG)
144 111  
145 112  
146   -if __name__ == "__main__":
147   - print_config()
  113 +def print_config() -> None:
  114 + config = _app().sanitized_dict()
  115 + print(config)
... ...
config/loader.py 0 โ†’ 100644
  Show/Hide comments   View file @86d8358
... ... @@ -0,0 +1,592 @@
  1 +"""
  2 +Unified application configuration loader.
  3 +
  4 +This module is the single source of truth for loading, merging, normalizing,
  5 +and validating application configuration.
  6 +"""
  7 +
  8 +from __future__ import annotations
  9 +
  10 +import hashlib
  11 +import json
  12 +import os
  13 +from copy import deepcopy
  14 +from dataclasses import asdict
  15 +from functools import lru_cache
  16 +from pathlib import Path
  17 +from typing import Any, Dict, Iterable, List, Optional, Tuple
  18 +
  19 +import yaml
  20 +
  21 +try:
  22 + from dotenv import load_dotenv as _load_dotenv # type: ignore
  23 +except Exception: # pragma: no cover
  24 + _load_dotenv = None
  25 +
  26 +from config.schema import (
  27 + AppConfig,
  28 + AssetsConfig,
  29 + ConfigMetadata,
  30 + DatabaseSettings,
  31 + ElasticsearchSettings,
  32 + EmbeddingServiceConfig,
  33 + FunctionScoreConfig,
  34 + IndexConfig,
  35 + InfrastructureConfig,
  36 + QueryConfig,
  37 + RedisSettings,
  38 + RerankConfig,
  39 + RerankServiceConfig,
  40 + RuntimeConfig,
  41 + SearchConfig,
  42 + SecretsConfig,
  43 + ServicesConfig,
  44 + SPUConfig,
  45 + TenantCatalogConfig,
  46 + TranslationServiceConfig,
  47 +)
  48 +from translation.settings import build_translation_config
  49 +
  50 +
  51 +class ConfigurationError(Exception):
  52 + """Raised when configuration validation fails."""
  53 +
  54 +
  55 +def _deep_merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
  56 + result = deepcopy(base)
  57 + for key, value in (override or {}).items():
  58 + if (
  59 + key in result
  60 + and isinstance(result[key], dict)
  61 + and isinstance(value, dict)
  62 + ):
  63 + result[key] = _deep_merge(result[key], value)
  64 + else:
  65 + result[key] = deepcopy(value)
  66 + return result
  67 +
  68 +
  69 +def _load_yaml(path: Path) -> Dict[str, Any]:
  70 + with open(path, "r", encoding="utf-8") as handle:
  71 + data = yaml.safe_load(handle) or {}
  72 + if not isinstance(data, dict):
  73 + raise ConfigurationError(f"Configuration file root must be a mapping: {path}")
  74 + return data
  75 +
  76 +
  77 +def _read_rewrite_dictionary(path: Path) -> Dict[str, str]:
  78 + rewrite_dict: Dict[str, str] = {}
  79 + if not path.exists():
  80 + return rewrite_dict
  81 +
  82 + with open(path, "r", encoding="utf-8") as handle:
  83 + for raw_line in handle:
  84 + line = raw_line.strip()
  85 + if not line or line.startswith("#"):
  86 + continue
  87 + parts = line.split("\t")
  88 + if len(parts) < 2:
  89 + continue
  90 + original = parts[0].strip()
  91 + replacement = parts[1].strip()
  92 + if original and replacement:
  93 + rewrite_dict[original] = replacement
  94 + return rewrite_dict
  95 +
  96 +
  97 +class AppConfigLoader:
  98 + """Load the unified application configuration."""
  99 +
  100 + def __init__(
  101 + self,
  102 + *,
  103 + config_dir: Optional[Path] = None,
  104 + config_file: Optional[Path] = None,
  105 + env_file: Optional[Path] = None,
  106 + ) -> None:
  107 + self.config_dir = Path(config_dir or Path(__file__).parent)
  108 + self.config_file = Path(config_file) if config_file is not None else None
  109 + self.project_root = self.config_dir.parent
  110 + self.env_file = Path(env_file) if env_file is not None else self.project_root / ".env"
  111 +
  112 + def load(self, validate: bool = True) -> AppConfig:
  113 + self._load_env()
  114 + raw_config, loaded_files = self._load_raw_config()
  115 + app_config = self._build_app_config(raw_config, loaded_files)
  116 + if validate:
  117 + self._validate(app_config)
  118 + return app_config
  119 +
  120 + def _load_env(self) -> None:
  121 + if _load_dotenv is not None:
  122 + _load_dotenv(self.env_file, override=False)
  123 + return
  124 + _load_env_file_fallback(self.env_file)
  125 +
  126 + def _load_raw_config(self) -> Tuple[Dict[str, Any], List[str]]:
  127 + env_name = (os.getenv("APP_ENV") or os.getenv("RUNTIME_ENV") or "prod").strip().lower() or "prod"
  128 + loaded_files: List[str] = []
  129 + raw: Dict[str, Any] = {}
  130 +
  131 + if self.config_file is not None:
  132 + config_path = self.config_file
  133 + if not config_path.exists():
  134 + raise ConfigurationError(f"Configuration file not found: {config_path}")
  135 + raw = _deep_merge(raw, _load_yaml(config_path))
  136 + loaded_files.append(str(config_path))
  137 + else:
  138 + base_path = self.config_dir / "base.yaml"
  139 + legacy_path = self.config_dir / "config.yaml"
  140 + primary_path = base_path if base_path.exists() else legacy_path
  141 + if not primary_path.exists():
  142 + raise ConfigurationError(f"Configuration file not found: {primary_path}")
  143 + raw = _deep_merge(raw, _load_yaml(primary_path))
  144 + loaded_files.append(str(primary_path))
  145 +
  146 + env_path = self.config_dir / "environments" / f"{env_name}.yaml"
  147 + if env_path.exists():
  148 + raw = _deep_merge(raw, _load_yaml(env_path))
  149 + loaded_files.append(str(env_path))
  150 +
  151 + tenant_dir = self.config_dir / "tenants"
  152 + if tenant_dir.is_dir():
  153 + tenant_files = sorted(tenant_dir.glob("*.yaml"))
  154 + if tenant_files:
  155 + tenant_config = {"default": {}, "tenants": {}}
  156 + default_path = tenant_dir / "_default.yaml"
  157 + if default_path.exists():
  158 + tenant_config["default"] = _load_yaml(default_path)
  159 + loaded_files.append(str(default_path))
  160 + for tenant_path in tenant_files:
  161 + if tenant_path.name == "_default.yaml":
  162 + continue
  163 + tenant_config["tenants"][tenant_path.stem] = _load_yaml(tenant_path)
  164 + loaded_files.append(str(tenant_path))
  165 + raw["tenant_config"] = tenant_config
  166 +
  167 + return raw, loaded_files
  168 +
  169 + def _build_app_config(self, raw: Dict[str, Any], loaded_files: List[str]) -> AppConfig:
  170 + assets_cfg = raw.get("assets") if isinstance(raw.get("assets"), dict) else {}
  171 + rewrite_path = (
  172 + assets_cfg.get("query_rewrite_dictionary_path")
  173 + or assets_cfg.get("rewrite_dictionary_path")
  174 + or self.config_dir / "dictionaries" / "query_rewrite.dict"
  175 + )
  176 + rewrite_path = Path(rewrite_path)
  177 + if not rewrite_path.is_absolute():
  178 + rewrite_path = (self.project_root / rewrite_path).resolve()
  179 + if not rewrite_path.exists():
  180 + legacy_rewrite_path = (self.config_dir / "query_rewrite.dict").resolve()
  181 + if legacy_rewrite_path.exists():
  182 + rewrite_path = legacy_rewrite_path
  183 +
  184 + rewrite_dictionary = _read_rewrite_dictionary(rewrite_path)
  185 + search_config = self._build_search_config(raw, rewrite_dictionary)
  186 + services_config = self._build_services_config(raw.get("services") or {})
  187 + tenants_config = self._build_tenants_config(raw.get("tenant_config") or {})
  188 + runtime_config = self._build_runtime_config()
  189 + infrastructure_config = self._build_infrastructure_config(runtime_config.environment)
  190 +
  191 + metadata = ConfigMetadata(
  192 + loaded_files=tuple(loaded_files),
  193 + config_hash="",
  194 + deprecated_keys=tuple(self._detect_deprecated_keys(raw)),
  195 + )
  196 +
  197 + app_config = AppConfig(
  198 + runtime=runtime_config,
  199 + infrastructure=infrastructure_config,
  200 + search=search_config,
  201 + services=services_config,
  202 + tenants=tenants_config,
  203 + assets=AssetsConfig(query_rewrite_dictionary_path=rewrite_path),
  204 + metadata=metadata,
  205 + )
  206 +
  207 + config_hash = self._compute_hash(app_config)
  208 + return AppConfig(
  209 + runtime=app_config.runtime,
  210 + infrastructure=app_config.infrastructure,
  211 + search=app_config.search,
  212 + services=app_config.services,
  213 + tenants=app_config.tenants,
  214 + assets=app_config.assets,
  215 + metadata=ConfigMetadata(
  216 + loaded_files=app_config.metadata.loaded_files,
  217 + config_hash=config_hash,
  218 + deprecated_keys=app_config.metadata.deprecated_keys,
  219 + ),
  220 + )
  221 +
  222 + def _build_search_config(self, raw: Dict[str, Any], rewrite_dictionary: Dict[str, str]) -> SearchConfig:
  223 + field_boosts = raw.get("field_boosts") or {}
  224 + if not isinstance(field_boosts, dict):
  225 + raise ConfigurationError("field_boosts must be a mapping")
  226 +
  227 + indexes: List[IndexConfig] = []
  228 + for item in raw.get("indexes") or []:
  229 + if not isinstance(item, dict):
  230 + raise ConfigurationError("indexes items must be mappings")
  231 + indexes.append(
  232 + IndexConfig(
  233 + name=str(item["name"]),
  234 + label=str(item.get("label") or item["name"]),
  235 + fields=list(item.get("fields") or []),
  236 + boost=float(item.get("boost", 1.0)),
  237 + example=item.get("example"),
  238 + )
  239 + )
  240 +
  241 + query_cfg = raw.get("query_config") if isinstance(raw.get("query_config"), dict) else {}
  242 + search_fields = query_cfg.get("search_fields") if isinstance(query_cfg.get("search_fields"), dict) else {}
  243 + text_strategy = (
  244 + query_cfg.get("text_query_strategy")
  245 + if isinstance(query_cfg.get("text_query_strategy"), dict)
  246 + else {}
  247 + )
  248 + query_config = QueryConfig(
  249 + supported_languages=list(query_cfg.get("supported_languages") or ["zh", "en"]),
  250 + default_language=str(query_cfg.get("default_language") or "en"),
  251 + enable_text_embedding=bool(query_cfg.get("enable_text_embedding", True)),
  252 + enable_query_rewrite=bool(query_cfg.get("enable_query_rewrite", True)),
  253 + rewrite_dictionary=rewrite_dictionary,
  254 + text_embedding_field=query_cfg.get("text_embedding_field"),
  255 + image_embedding_field=query_cfg.get("image_embedding_field"),
  256 + source_fields=query_cfg.get("source_fields"),
  257 + knn_boost=float(query_cfg.get("knn_boost", 0.25)),
  258 + multilingual_fields=list(
  259 + search_fields.get(
  260 + "multilingual_fields",
  261 + ["title", "brief", "description", "vendor", "category_path", "category_name_text"],
  262 + )
  263 + ),
  264 + shared_fields=list(
  265 + search_fields.get(
  266 + "shared_fields",
  267 + ["tags", "option1_values", "option2_values", "option3_values"],
  268 + )
  269 + ),
  270 + core_multilingual_fields=list(
  271 + search_fields.get(
  272 + "core_multilingual_fields",
  273 + ["title", "brief", "vendor", "category_name_text"],
  274 + )
  275 + ),
  276 + base_minimum_should_match=str(text_strategy.get("base_minimum_should_match", "75%")),
  277 + translation_minimum_should_match=str(text_strategy.get("translation_minimum_should_match", "75%")),
  278 + translation_boost=float(text_strategy.get("translation_boost", 0.4)),
  279 + translation_boost_when_source_missing=float(
  280 + text_strategy.get("translation_boost_when_source_missing", 1.0)
  281 + ),
  282 + source_boost_when_missing=float(text_strategy.get("source_boost_when_missing", 0.6)),
  283 + original_query_fallback_boost_when_translation_missing=float(
  284 + text_strategy.get("original_query_fallback_boost_when_translation_missing", 0.2)
  285 + ),
  286 + tie_breaker_base_query=float(text_strategy.get("tie_breaker_base_query", 0.9)),
  287 + zh_to_en_model=str(query_cfg.get("zh_to_en_model") or "opus-mt-zh-en"),
  288 + en_to_zh_model=str(query_cfg.get("en_to_zh_model") or "opus-mt-en-zh"),
  289 + default_translation_model=str(
  290 + query_cfg.get("default_translation_model") or "nllb-200-distilled-600m"
  291 + ),
  292 + )
  293 +
  294 + function_score_cfg = raw.get("function_score") if isinstance(raw.get("function_score"), dict) else {}
  295 + rerank_cfg = raw.get("rerank") if isinstance(raw.get("rerank"), dict) else {}
  296 + spu_cfg = raw.get("spu_config") if isinstance(raw.get("spu_config"), dict) else {}
  297 +
  298 + return SearchConfig(
  299 + field_boosts={str(key): float(value) for key, value in field_boosts.items()},
  300 + indexes=indexes,
  301 + query_config=query_config,
  302 + function_score=FunctionScoreConfig(
  303 + score_mode=str(function_score_cfg.get("score_mode") or "sum"),
  304 + boost_mode=str(function_score_cfg.get("boost_mode") or "multiply"),
  305 + functions=list(function_score_cfg.get("functions") or []),
  306 + ),
  307 + rerank=RerankConfig(
  308 + enabled=bool(rerank_cfg.get("enabled", True)),
  309 + rerank_window=int(rerank_cfg.get("rerank_window", 384)),
  310 + timeout_sec=float(rerank_cfg.get("timeout_sec", 15.0)),
  311 + weight_es=float(rerank_cfg.get("weight_es", 0.4)),
  312 + weight_ai=float(rerank_cfg.get("weight_ai", 0.6)),
  313 + rerank_query_template=str(rerank_cfg.get("rerank_query_template") or "{query}"),
  314 + rerank_doc_template=str(rerank_cfg.get("rerank_doc_template") or "{title}"),
  315 + ),
  316 + spu_config=SPUConfig(
  317 + enabled=bool(spu_cfg.get("enabled", False)),
  318 + spu_field=spu_cfg.get("spu_field"),
  319 + inner_hits_size=int(spu_cfg.get("inner_hits_size", 3)),
  320 + searchable_option_dimensions=list(
  321 + spu_cfg.get("searchable_option_dimensions") or ["option1", "option2", "option3"]
  322 + ),
  323 + ),
  324 + es_index_name=str(raw.get("es_index_name") or "search_products"),
  325 + es_settings=dict(raw.get("es_settings") or {}),
  326 + )
  327 +
  328 + def _build_services_config(self, raw: Dict[str, Any]) -> ServicesConfig:
  329 + if not isinstance(raw, dict):
  330 + raise ConfigurationError("services must be a mapping")
  331 +
  332 + translation_raw = raw.get("translation") if isinstance(raw.get("translation"), dict) else {}
  333 + normalized_translation = build_translation_config(translation_raw)
  334 + translation_config = TranslationServiceConfig(
  335 + endpoint=str(normalized_translation["service_url"]).rstrip("/"),
  336 + timeout_sec=float(normalized_translation["timeout_sec"]),
  337 + default_model=str(normalized_translation["default_model"]),
  338 + default_scene=str(normalized_translation["default_scene"]),
  339 + cache=dict(normalized_translation["cache"]),
  340 + capabilities={str(key): dict(value) for key, value in normalized_translation["capabilities"].items()},
  341 + )
  342 +
  343 + embedding_raw = raw.get("embedding") if isinstance(raw.get("embedding"), dict) else {}
  344 + embedding_provider = str(embedding_raw.get("provider") or "http").strip().lower()
  345 + embedding_providers = dict(embedding_raw.get("providers") or {})
  346 + if embedding_provider not in embedding_providers:
  347 + raise ConfigurationError(f"services.embedding.providers.{embedding_provider} must be configured")
  348 + embedding_backend = str(embedding_raw.get("backend") or "").strip().lower()
  349 + embedding_backends = {
  350 + str(key).strip().lower(): dict(value)
  351 + for key, value in dict(embedding_raw.get("backends") or {}).items()
  352 + }
  353 + if embedding_backend not in embedding_backends:
  354 + raise ConfigurationError(f"services.embedding.backends.{embedding_backend} must be configured")
  355 + image_backend = str(embedding_raw.get("image_backend") or "clip_as_service").strip().lower()
  356 + image_backends = {
  357 + str(key).strip().lower(): dict(value)
  358 + for key, value in dict(embedding_raw.get("image_backends") or {}).items()
  359 + }
  360 + if not image_backends:
  361 + image_backends = {
  362 + "clip_as_service": {
  363 + "server": "grpc://127.0.0.1:51000",
  364 + "model_name": "CN-CLIP/ViT-L-14",
  365 + "batch_size": 8,
  366 + "normalize_embeddings": True,
  367 + },
  368 + "local_cnclip": {
  369 + "model_name": "ViT-L-14",
  370 + "device": None,
  371 + "batch_size": 8,
  372 + "normalize_embeddings": True,
  373 + },
  374 + }
  375 + if image_backend not in image_backends:
  376 + raise ConfigurationError(f"services.embedding.image_backends.{image_backend} must be configured")
  377 +
  378 + embedding_config = EmbeddingServiceConfig(
  379 + provider=embedding_provider,
  380 + providers=embedding_providers,
  381 + backend=embedding_backend,
  382 + backends=embedding_backends,
  383 + image_backend=image_backend,
  384 + image_backends=image_backends,
  385 + )
  386 +
  387 + rerank_raw = raw.get("rerank") if isinstance(raw.get("rerank"), dict) else {}
  388 + rerank_provider = str(rerank_raw.get("provider") or "http").strip().lower()
  389 + rerank_providers = dict(rerank_raw.get("providers") or {})
  390 + if rerank_provider not in rerank_providers:
  391 + raise ConfigurationError(f"services.rerank.providers.{rerank_provider} must be configured")
  392 + rerank_backend = str(rerank_raw.get("backend") or "").strip().lower()
  393 + rerank_backends = {
  394 + str(key).strip().lower(): dict(value)
  395 + for key, value in dict(rerank_raw.get("backends") or {}).items()
  396 + }
  397 + if rerank_backend not in rerank_backends:
  398 + raise ConfigurationError(f"services.rerank.backends.{rerank_backend} must be configured")
  399 + rerank_request = dict(rerank_raw.get("request") or {})
  400 + rerank_request.setdefault("max_docs", 1000)
  401 + rerank_request.setdefault("normalize", True)
  402 +
  403 + rerank_config = RerankServiceConfig(
  404 + provider=rerank_provider,
  405 + providers=rerank_providers,
  406 + backend=rerank_backend,
  407 + backends=rerank_backends,
  408 + request=rerank_request,
  409 + )
  410 +
  411 + return ServicesConfig(
  412 + translation=translation_config,
  413 + embedding=embedding_config,
  414 + rerank=rerank_config,
  415 + )
  416 +
  417 + def _build_tenants_config(self, raw: Dict[str, Any]) -> TenantCatalogConfig:
  418 + if not isinstance(raw, dict):
  419 + raise ConfigurationError("tenant_config must be a mapping")
  420 + default_cfg = raw.get("default") if isinstance(raw.get("default"), dict) else {}
  421 + tenants_cfg = raw.get("tenants") if isinstance(raw.get("tenants"), dict) else {}
  422 + return TenantCatalogConfig(
  423 + default=dict(default_cfg),
  424 + tenants={str(key): dict(value) for key, value in tenants_cfg.items()},
  425 + )
  426 +
  427 + def _build_runtime_config(self) -> RuntimeConfig:
  428 + environment = (os.getenv("APP_ENV") or os.getenv("RUNTIME_ENV") or "prod").strip().lower() or "prod"
  429 + namespace = os.getenv("ES_INDEX_NAMESPACE")
  430 + if namespace is None:
  431 + namespace = "" if environment == "prod" else f"{environment}_"
  432 +
  433 + return RuntimeConfig(
  434 + environment=environment,
  435 + index_namespace=namespace,
  436 + api_host=os.getenv("API_HOST", "0.0.0.0"),
  437 + api_port=int(os.getenv("API_PORT", 6002)),
  438 + indexer_host=os.getenv("INDEXER_HOST", "0.0.0.0"),
  439 + indexer_port=int(os.getenv("INDEXER_PORT", 6004)),
  440 + embedding_host=os.getenv("EMBEDDING_HOST", "127.0.0.1"),
  441 + embedding_port=int(os.getenv("EMBEDDING_PORT", 6005)),
  442 + embedding_text_port=int(os.getenv("EMBEDDING_TEXT_PORT", 6005)),
  443 + embedding_image_port=int(os.getenv("EMBEDDING_IMAGE_PORT", 6008)),
  444 + translator_host=os.getenv("TRANSLATION_HOST", "127.0.0.1"),
  445 + translator_port=int(os.getenv("TRANSLATION_PORT", 6006)),
  446 + reranker_host=os.getenv("RERANKER_HOST", "127.0.0.1"),
  447 + reranker_port=int(os.getenv("RERANKER_PORT", 6007)),
  448 + )
  449 +
  450 + def _build_infrastructure_config(self, environment: str) -> InfrastructureConfig:
  451 + del environment
  452 + return InfrastructureConfig(
  453 + elasticsearch=ElasticsearchSettings(
  454 + host=os.getenv("ES_HOST", "http://localhost:9200"),
  455 + username=os.getenv("ES_USERNAME"),
  456 + password=os.getenv("ES_PASSWORD"),
  457 + ),
  458 + redis=RedisSettings(
  459 + host=os.getenv("REDIS_HOST", "localhost"),
  460 + port=int(os.getenv("REDIS_PORT", 6479)),
  461 + snapshot_db=int(os.getenv("REDIS_SNAPSHOT_DB", 0)),
  462 + password=os.getenv("REDIS_PASSWORD"),
  463 + socket_timeout=int(os.getenv("REDIS_SOCKET_TIMEOUT", 1)),
  464 + socket_connect_timeout=int(os.getenv("REDIS_SOCKET_CONNECT_TIMEOUT", 1)),
  465 + retry_on_timeout=os.getenv("REDIS_RETRY_ON_TIMEOUT", "false").strip().lower() == "true",
  466 + cache_expire_days=int(os.getenv("REDIS_CACHE_EXPIRE_DAYS", 360 * 2)),
  467 + embedding_cache_prefix=os.getenv("REDIS_EMBEDDING_CACHE_PREFIX", "embedding"),
  468 + anchor_cache_prefix=os.getenv("REDIS_ANCHOR_CACHE_PREFIX", "product_anchors"),
  469 + anchor_cache_expire_days=int(os.getenv("REDIS_ANCHOR_CACHE_EXPIRE_DAYS", 30)),
  470 + ),
  471 + database=DatabaseSettings(
  472 + host=os.getenv("DB_HOST"),
  473 + port=int(os.getenv("DB_PORT", 3306)) if os.getenv("DB_PORT") else 3306,
  474 + database=os.getenv("DB_DATABASE"),
  475 + username=os.getenv("DB_USERNAME"),
  476 + password=os.getenv("DB_PASSWORD"),
  477 + ),
  478 + secrets=SecretsConfig(
  479 + dashscope_api_key=os.getenv("DASHSCOPE_API_KEY"),
  480 + deepl_auth_key=os.getenv("DEEPL_AUTH_KEY"),
  481 + ),
  482 + )
  483 +
  484 + def _validate(self, app_config: AppConfig) -> None:
  485 + errors: List[str] = []
  486 +
  487 + if not app_config.search.es_index_name:
  488 + errors.append("search.es_index_name is required")
  489 +
  490 + if not app_config.search.field_boosts:
  491 + errors.append("search.field_boosts cannot be empty")
  492 + else:
  493 + for field_name, boost in app_config.search.field_boosts.items():
  494 + if boost < 0:
  495 + errors.append(f"field_boosts.{field_name} must be non-negative")
  496 +
  497 + query_config = app_config.search.query_config
  498 + if not query_config.supported_languages:
  499 + errors.append("query_config.supported_languages must not be empty")
  500 + if query_config.default_language not in query_config.supported_languages:
  501 + errors.append("query_config.default_language must be included in supported_languages")
  502 + for name, values in (
  503 + ("multilingual_fields", query_config.multilingual_fields),
  504 + ("shared_fields", query_config.shared_fields),
  505 + ("core_multilingual_fields", query_config.core_multilingual_fields),
  506 + ):
  507 + if not values:
  508 + errors.append(f"query_config.{name} must not be empty")
  509 +
  510 + if not set(query_config.core_multilingual_fields).issubset(set(query_config.multilingual_fields)):
  511 + errors.append("query_config.core_multilingual_fields must be a subset of multilingual_fields")
  512 +
  513 + if app_config.search.spu_config.enabled and not app_config.search.spu_config.spu_field:
  514 + errors.append("spu_config.spu_field is required when spu_config.enabled is true")
  515 +
  516 + if not app_config.tenants.default or not app_config.tenants.default.get("index_languages"):
  517 + errors.append("tenant_config.default.index_languages must be configured")
  518 +
  519 + if app_config.metadata.deprecated_keys:
  520 + errors.append(
  521 + "Deprecated tenant config keys are not supported: "
  522 + + ", ".join(app_config.metadata.deprecated_keys)
  523 + )
  524 +
  525 + embedding_provider_cfg = app_config.services.embedding.get_provider_config()
  526 + if not embedding_provider_cfg.get("text_base_url"):
  527 + errors.append("services.embedding.providers.<provider>.text_base_url is required")
  528 + if not embedding_provider_cfg.get("image_base_url"):
  529 + errors.append("services.embedding.providers.<provider>.image_base_url is required")
  530 +
  531 + rerank_provider_cfg = app_config.services.rerank.get_provider_config()
  532 + if not rerank_provider_cfg.get("service_url") and not rerank_provider_cfg.get("base_url"):
  533 + errors.append("services.rerank.providers.<provider>.service_url or base_url is required")
  534 +
  535 + if errors:
  536 + raise ConfigurationError("Configuration validation failed:\n" + "\n".join(f" - {err}" for err in errors))
  537 +
  538 + def _compute_hash(self, app_config: AppConfig) -> str:
  539 + payload = asdict(app_config)
  540 + payload["metadata"]["config_hash"] = ""
  541 + payload["infrastructure"]["elasticsearch"]["password"] = "***" if payload["infrastructure"]["elasticsearch"].get("password") else None
  542 + payload["infrastructure"]["database"]["password"] = "***" if payload["infrastructure"]["database"].get("password") else None
  543 + payload["infrastructure"]["redis"]["password"] = "***" if payload["infrastructure"]["redis"].get("password") else None
  544 + payload["infrastructure"]["secrets"]["dashscope_api_key"] = "***" if payload["infrastructure"]["secrets"].get("dashscope_api_key") else None
  545 + payload["infrastructure"]["secrets"]["deepl_auth_key"] = "***" if payload["infrastructure"]["secrets"].get("deepl_auth_key") else None
  546 + blob = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str)
  547 + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:16]
  548 +
  549 + def _detect_deprecated_keys(self, raw: Dict[str, Any]) -> Iterable[str]:
  550 + tenant_raw = raw.get("tenant_config") if isinstance(raw.get("tenant_config"), dict) else {}
  551 + for key in ("default",):
  552 + item = tenant_raw.get(key)
  553 + if isinstance(item, dict):
  554 + for deprecated in ("translate_to_en", "translate_to_zh"):
  555 + if deprecated in item:
  556 + yield f"tenant_config.{key}.{deprecated}"
  557 + tenants = tenant_raw.get("tenants") if isinstance(tenant_raw.get("tenants"), dict) else {}
  558 + for tenant_id, cfg in tenants.items():
  559 + if not isinstance(cfg, dict):
  560 + continue
  561 + for deprecated in ("translate_to_en", "translate_to_zh"):
  562 + if deprecated in cfg:
  563 + yield f"tenant_config.tenants.{tenant_id}.{deprecated}"
  564 +
  565 +
  566 +@lru_cache(maxsize=1)
  567 +def get_app_config() -> AppConfig:
  568 + """Return the process-global application configuration."""
  569 +
  570 + return AppConfigLoader().load()
  571 +
  572 +
  573 +def reload_app_config() -> AppConfig:
  574 + """Clear the cached configuration and reload it."""
  575 +
  576 + get_app_config.cache_clear()
  577 + return get_app_config()
  578 +
  579 +
  580 +def _load_env_file_fallback(path: Path) -> None:
  581 + if not path.exists():
  582 + return
  583 + with open(path, "r", encoding="utf-8") as handle:
  584 + for raw_line in handle:
  585 + line = raw_line.strip()
  586 + if not line or line.startswith("#") or "=" not in line:
  587 + continue
  588 + key, value = line.split("=", 1)
  589 + key = key.strip()
  590 + value = value.strip().strip('"').strip("'")
  591 + if key and key not in os.environ:
  592 + os.environ[key] = value
... ...
config/query_rewrite.dict deleted
View file @77bfa7e
... ... @@ -1,3 +0,0 @@
1   -็Žฉๅ…ท category.keyword:็Žฉๅ…ท OR default:็Žฉๅ…ท
2   -ๆถˆ้˜ฒ category.keyword:ๆถˆ้˜ฒ OR default:ๆถˆ้˜ฒ
3   -
config/schema.py 0 โ†’ 100644
  Show/Hide comments   View file @86d8358
... ... @@ -0,0 +1,307 @@
  1 +"""
  2 +Typed configuration schema for the unified application configuration.
  3 +
  4 +This module defines the normalized in-memory structure used by all services.
  5 +"""
  6 +
  7 +from __future__ import annotations
  8 +
  9 +from dataclasses import asdict, dataclass, field
  10 +from pathlib import Path
  11 +from typing import Any, Dict, List, Optional, Tuple
  12 +
  13 +
  14 +@dataclass(frozen=True)
  15 +class IndexConfig:
  16 + """Deprecated compatibility shape for legacy diagnostics/tests."""
  17 +
  18 + name: str
  19 + label: str
  20 + fields: List[str]
  21 + boost: float = 1.0
  22 + example: Optional[str] = None
  23 +
  24 +
  25 +@dataclass(frozen=True)
  26 +class QueryConfig:
  27 + """Configuration for query processing."""
  28 +
  29 + supported_languages: List[str] = field(default_factory=lambda: ["zh", "en"])
  30 + default_language: str = "en"
  31 + enable_text_embedding: bool = True
  32 + enable_query_rewrite: bool = True
  33 + rewrite_dictionary: Dict[str, str] = field(default_factory=dict)
  34 + text_embedding_field: Optional[str] = "title_embedding"
  35 + image_embedding_field: Optional[str] = None
  36 + source_fields: Optional[List[str]] = None
  37 + knn_boost: float = 0.25
  38 + multilingual_fields: List[str] = field(
  39 + default_factory=lambda: [
  40 + "title",
  41 + "brief",
  42 + "description",
  43 + "vendor",
  44 + "category_path",
  45 + "category_name_text",
  46 + ]
  47 + )
  48 + shared_fields: List[str] = field(
  49 + default_factory=lambda: ["tags", "option1_values", "option2_values", "option3_values"]
  50 + )
  51 + core_multilingual_fields: List[str] = field(
  52 + default_factory=lambda: ["title", "brief", "vendor", "category_name_text"]
  53 + )
  54 + base_minimum_should_match: str = "75%"
  55 + translation_minimum_should_match: str = "75%"
  56 + translation_boost: float = 0.4
  57 + translation_boost_when_source_missing: float = 1.0
  58 + source_boost_when_missing: float = 0.6
  59 + original_query_fallback_boost_when_translation_missing: float = 0.2
  60 + tie_breaker_base_query: float = 0.9
  61 + zh_to_en_model: str = "opus-mt-zh-en"
  62 + en_to_zh_model: str = "opus-mt-en-zh"
  63 + default_translation_model: str = "nllb-200-distilled-600m"
  64 +
  65 +
  66 +@dataclass(frozen=True)
  67 +class SPUConfig:
  68 + """SPU aggregation/search configuration."""
  69 +
  70 + enabled: bool = False
  71 + spu_field: Optional[str] = None
  72 + inner_hits_size: int = 3
  73 + searchable_option_dimensions: List[str] = field(
  74 + default_factory=lambda: ["option1", "option2", "option3"]
  75 + )
  76 +
  77 +
  78 +@dataclass(frozen=True)
  79 +class FunctionScoreConfig:
  80 + """Function score configuration."""
  81 +
  82 + score_mode: str = "sum"
  83 + boost_mode: str = "multiply"
  84 + functions: List[Dict[str, Any]] = field(default_factory=list)
  85 +
  86 +
  87 +@dataclass(frozen=True)
  88 +class RerankConfig:
  89 + """Search-time rerank configuration."""
  90 +
  91 + enabled: bool = True
  92 + rerank_window: int = 384
  93 + timeout_sec: float = 15.0
  94 + weight_es: float = 0.4
  95 + weight_ai: float = 0.6
  96 + rerank_query_template: str = "{query}"
  97 + rerank_doc_template: str = "{title}"
  98 +
  99 +
  100 +@dataclass(frozen=True)
  101 +class SearchConfig:
  102 + """Search behavior configuration shared by backend and indexer."""
  103 +
  104 + field_boosts: Dict[str, float]
  105 + indexes: List[IndexConfig] = field(default_factory=list)
  106 + query_config: QueryConfig = field(default_factory=QueryConfig)
  107 + function_score: FunctionScoreConfig = field(default_factory=FunctionScoreConfig)
  108 + rerank: RerankConfig = field(default_factory=RerankConfig)
  109 + spu_config: SPUConfig = field(default_factory=SPUConfig)
  110 + es_index_name: str = "search_products"
  111 + es_settings: Dict[str, Any] = field(default_factory=dict)
  112 +
  113 +
  114 +@dataclass(frozen=True)
  115 +class TranslationServiceConfig:
  116 + """Translator service configuration."""
  117 +
  118 + endpoint: str
  119 + timeout_sec: float
  120 + default_model: str
  121 + default_scene: str
  122 + cache: Dict[str, Any]
  123 + capabilities: Dict[str, Dict[str, Any]]
  124 +
  125 + def as_dict(self) -> Dict[str, Any]:
  126 + return {
  127 + "service_url": self.endpoint,
  128 + "timeout_sec": self.timeout_sec,
  129 + "default_model": self.default_model,
  130 + "default_scene": self.default_scene,
  131 + "cache": self.cache,
  132 + "capabilities": self.capabilities,
  133 + }
  134 +
  135 +
  136 +@dataclass(frozen=True)
  137 +class EmbeddingServiceConfig:
  138 + """Embedding service configuration."""
  139 +
  140 + provider: str
  141 + providers: Dict[str, Any]
  142 + backend: str
  143 + backends: Dict[str, Dict[str, Any]]
  144 + image_backend: str
  145 + image_backends: Dict[str, Dict[str, Any]]
  146 +
  147 + def get_provider_config(self) -> Dict[str, Any]:
  148 + return dict(self.providers.get(self.provider, {}) or {})
  149 +
  150 + def get_backend_config(self) -> Dict[str, Any]:
  151 + return dict(self.backends.get(self.backend, {}) or {})
  152 +
  153 + def get_image_backend_config(self) -> Dict[str, Any]:
  154 + return dict(self.image_backends.get(self.image_backend, {}) or {})
  155 +
  156 +
  157 +@dataclass(frozen=True)
  158 +class RerankServiceConfig:
  159 + """Reranker service configuration."""
  160 +
  161 + provider: str
  162 + providers: Dict[str, Any]
  163 + backend: str
  164 + backends: Dict[str, Dict[str, Any]]
  165 + request: Dict[str, Any]
  166 +
  167 + def get_provider_config(self) -> Dict[str, Any]:
  168 + return dict(self.providers.get(self.provider, {}) or {})
  169 +
  170 + def get_backend_config(self) -> Dict[str, Any]:
  171 + return dict(self.backends.get(self.backend, {}) or {})
  172 +
  173 +
  174 +@dataclass(frozen=True)
  175 +class ServicesConfig:
  176 + """All service-level configuration."""
  177 +
  178 + translation: TranslationServiceConfig
  179 + embedding: EmbeddingServiceConfig
  180 + rerank: RerankServiceConfig
  181 +
  182 +
  183 +@dataclass(frozen=True)
  184 +class TenantCatalogConfig:
  185 + """Tenant catalog configuration."""
  186 +
  187 + default: Dict[str, Any]
  188 + tenants: Dict[str, Dict[str, Any]]
  189 +
  190 + def get_raw(self) -> Dict[str, Any]:
  191 + return {
  192 + "default": dict(self.default),
  193 + "tenants": {str(key): dict(value) for key, value in self.tenants.items()},
  194 + }
  195 +
  196 +
  197 +@dataclass(frozen=True)
  198 +class ElasticsearchSettings:
  199 + host: str = "http://localhost:9200"
  200 + username: Optional[str] = None
  201 + password: Optional[str] = None
  202 +
  203 +
  204 +@dataclass(frozen=True)
  205 +class RedisSettings:
  206 + host: str = "localhost"
  207 + port: int = 6479
  208 + snapshot_db: int = 0
  209 + password: Optional[str] = None
  210 + socket_timeout: int = 1
  211 + socket_connect_timeout: int = 1
  212 + retry_on_timeout: bool = False
  213 + cache_expire_days: int = 720
  214 + embedding_cache_prefix: str = "embedding"
  215 + anchor_cache_prefix: str = "product_anchors"
  216 + anchor_cache_expire_days: int = 30
  217 +
  218 +
  219 +@dataclass(frozen=True)
  220 +class DatabaseSettings:
  221 + host: Optional[str] = None
  222 + port: int = 3306
  223 + database: Optional[str] = None
  224 + username: Optional[str] = None
  225 + password: Optional[str] = None
  226 +
  227 +
  228 +@dataclass(frozen=True)
  229 +class SecretsConfig:
  230 + dashscope_api_key: Optional[str] = None
  231 + deepl_auth_key: Optional[str] = None
  232 +
  233 +
  234 +@dataclass(frozen=True)
  235 +class InfrastructureConfig:
  236 + elasticsearch: ElasticsearchSettings
  237 + redis: RedisSettings
  238 + database: DatabaseSettings
  239 + secrets: SecretsConfig
  240 +
  241 +
  242 +@dataclass(frozen=True)
  243 +class RuntimeConfig:
  244 + environment: str = "prod"
  245 + index_namespace: str = ""
  246 + api_host: str = "0.0.0.0"
  247 + api_port: int = 6002
  248 + indexer_host: str = "0.0.0.0"
  249 + indexer_port: int = 6004
  250 + embedding_host: str = "127.0.0.1"
  251 + embedding_port: int = 6005
  252 + embedding_text_port: int = 6005
  253 + embedding_image_port: int = 6008
  254 + translator_host: str = "127.0.0.1"
  255 + translator_port: int = 6006
  256 + reranker_host: str = "127.0.0.1"
  257 + reranker_port: int = 6007
  258 +
  259 +
  260 +@dataclass(frozen=True)
  261 +class AssetsConfig:
  262 + query_rewrite_dictionary_path: Path
  263 +
  264 +
  265 +@dataclass(frozen=True)
  266 +class ConfigMetadata:
  267 + loaded_files: Tuple[str, ...]
  268 + config_hash: str
  269 + deprecated_keys: Tuple[str, ...] = field(default_factory=tuple)
  270 +
  271 +
  272 +@dataclass(frozen=True)
  273 +class AppConfig:
  274 + """Root application configuration."""
  275 +
  276 + runtime: RuntimeConfig
  277 + infrastructure: InfrastructureConfig
  278 + search: SearchConfig
  279 + services: ServicesConfig
  280 + tenants: TenantCatalogConfig
  281 + assets: AssetsConfig
  282 + metadata: ConfigMetadata
  283 +
  284 + def sanitized_dict(self) -> Dict[str, Any]:
  285 + data = asdict(self)
  286 + data["infrastructure"]["elasticsearch"]["password"] = _mask_secret(
  287 + data["infrastructure"]["elasticsearch"].get("password")
  288 + )
  289 + data["infrastructure"]["database"]["password"] = _mask_secret(
  290 + data["infrastructure"]["database"].get("password")
  291 + )
  292 + data["infrastructure"]["redis"]["password"] = _mask_secret(
  293 + data["infrastructure"]["redis"].get("password")
  294 + )
  295 + data["infrastructure"]["secrets"]["dashscope_api_key"] = _mask_secret(
  296 + data["infrastructure"]["secrets"].get("dashscope_api_key")
  297 + )
  298 + data["infrastructure"]["secrets"]["deepl_auth_key"] = _mask_secret(
  299 + data["infrastructure"]["secrets"].get("deepl_auth_key")
  300 + )
  301 + return data
  302 +
  303 +
  304 +def _mask_secret(value: Optional[str]) -> Optional[str]:
  305 + if not value:
  306 + return value
  307 + return "***"
... ...
config/services_config.py
  Show/Hide comments   View file @86d8358
1 1 """
2   -Services configuration - single source for translation, embedding, rerank.
  2 +Unified service configuration accessors.
3 3  
4   -Translation is modeled as:
5   -- one translator service endpoint used by business callers
6   -- multiple translation capabilities loaded inside the translator service
  4 +This module is now a thin adapter over ``config.loader.get_app_config`` and
  5 +contains no independent parsing or precedence logic.
7 6 """
8 7  
9 8 from __future__ import annotations
10 9  
11   -import os
12   -from dataclasses import dataclass, field
13   -from functools import lru_cache
14   -from pathlib import Path
15   -from typing import Any, Dict, List, Optional
16   -
17   -import yaml
18   -from translation.settings import TranslationConfig, build_translation_config, get_translation_cache
19   -
20   -
21   -@dataclass
22   -class ServiceConfig:
23   - """Config for one capability (embedding/rerank)."""
24   -
25   - provider: str
26   - providers: Dict[str, Any] = field(default_factory=dict)
27   -
28   - def get_provider_cfg(self) -> Dict[str, Any]:
29   - p = (self.provider or "").strip().lower()
30   - return self.providers.get(p, {}) if isinstance(self.providers, dict) else {}
31   -
32   -
33   -def _load_services_raw(config_path: Optional[Path] = None) -> Dict[str, Any]:
34   - if config_path is None:
35   - config_path = Path(__file__).parent / "config.yaml"
36   - path = Path(config_path)
37   - if not path.exists():
38   - raise FileNotFoundError(f"services config file not found: {path}")
39   - try:
40   - with open(path, "r", encoding="utf-8") as f:
41   - data = yaml.safe_load(f)
42   - except Exception as exc:
43   - raise RuntimeError(f"failed to parse services config from {path}: {exc}") from exc
44   - if not isinstance(data, dict):
45   - raise RuntimeError(f"invalid config format in {path}: expected mapping root")
46   - services = data.get("services")
47   - if not isinstance(services, dict):
48   - raise RuntimeError("config.yaml must contain a valid 'services' mapping")
49   - return services
50   -
51   -
52   -def _resolve_provider_name(env_name: str, config_provider: Any, capability: str) -> str:
53   - provider = os.getenv(env_name) or config_provider
54   - if not provider:
55   - raise ValueError(
56   - f"services.{capability}.provider is required "
57   - f"(or set env override {env_name})"
58   - )
59   - return str(provider).strip().lower()
60   -
61   -
62   -def _resolve_translation() -> TranslationConfig:
63   - raw = _load_services_raw()
64   - cfg = raw.get("translation", {}) if isinstance(raw.get("translation"), dict) else {}
65   - return build_translation_config(cfg)
66   -
67   -
68   -def _resolve_embedding() -> ServiceConfig:
69   - raw = _load_services_raw()
70   - cfg = raw.get("embedding", {}) if isinstance(raw.get("embedding"), dict) else {}
71   - providers = cfg.get("providers", {}) if isinstance(cfg.get("providers"), dict) else {}
72   -
73   - provider = _resolve_provider_name(
74   - env_name="EMBEDDING_PROVIDER",
75   - config_provider=cfg.get("provider"),
76   - capability="embedding",
77   - )
78   - if provider != "http":
79   - raise ValueError(f"Unsupported embedding provider: {provider}")
80   -
81   - env_text_url = os.getenv("EMBEDDING_TEXT_SERVICE_URL")
82   - env_image_url = os.getenv("EMBEDDING_IMAGE_SERVICE_URL")
83   - if provider == "http":
84   - providers = dict(providers)
85   - http_cfg = dict(providers.get("http", {}))
86   - if env_text_url:
87   - http_cfg["text_base_url"] = env_text_url.rstrip("/")
88   - if env_image_url:
89   - http_cfg["image_base_url"] = env_image_url.rstrip("/")
90   - if not http_cfg.get("text_base_url"):
91   - raise ValueError("services.embedding.providers.http.text_base_url is required")
92   - if not http_cfg.get("image_base_url"):
93   - raise ValueError("services.embedding.providers.http.image_base_url is required")
94   - providers["http"] = http_cfg
95   -
96   - return ServiceConfig(provider=provider, providers=providers)
97   -
98   -
99   -def _resolve_rerank() -> ServiceConfig:
100   - raw = _load_services_raw()
101   - cfg = raw.get("rerank", {}) if isinstance(raw.get("rerank"), dict) else {}
102   - providers = cfg.get("providers", {}) if isinstance(cfg.get("providers"), dict) else {}
103   -
104   - provider = _resolve_provider_name(
105   - env_name="RERANK_PROVIDER",
106   - config_provider=cfg.get("provider"),
107   - capability="rerank",
108   - )
109   - if provider != "http":
110   - raise ValueError(f"Unsupported rerank provider: {provider}")
111   -
112   - env_url = os.getenv("RERANKER_SERVICE_URL")
113   - if env_url:
114   - url = env_url.rstrip("/")
115   - if not url.endswith("/rerank"):
116   - url = f"{url}/rerank" if "/rerank" not in url else url
117   - providers = dict(providers)
118   - providers["http"] = dict(providers.get("http", {}))
119   - providers["http"]["base_url"] = url.replace("/rerank", "")
120   - providers["http"]["service_url"] = url
121   -
122   - return ServiceConfig(provider=provider, providers=providers)
123   -
124   -
125   -def get_rerank_backend_config() -> tuple[str, dict]:
126   - raw = _load_services_raw()
127   - cfg = raw.get("rerank", {}) if isinstance(raw.get("rerank"), dict) else {}
128   - backends = cfg.get("backends", {}) if isinstance(cfg.get("backends"), dict) else {}
129   - name = os.getenv("RERANK_BACKEND") or cfg.get("backend")
130   - if not name:
131   - raise ValueError("services.rerank.backend is required (or env RERANK_BACKEND)")
132   - name = str(name).strip().lower()
133   - backend_cfg = backends.get(name, {}) if isinstance(backends.get(name), dict) else {}
134   - if not backend_cfg:
135   - raise ValueError(f"services.rerank.backends.{name} is required")
136   - return name, backend_cfg
137   -
138   -
139   -def get_embedding_backend_config() -> tuple[str, dict]:
140   - raw = _load_services_raw()
141   - cfg = raw.get("embedding", {}) if isinstance(raw.get("embedding"), dict) else {}
142   - backends = cfg.get("backends", {}) if isinstance(cfg.get("backends"), dict) else {}
143   - name = os.getenv("EMBEDDING_BACKEND") or cfg.get("backend")
144   - if not name:
145   - raise ValueError("services.embedding.backend is required (or env EMBEDDING_BACKEND)")
146   - name = str(name).strip().lower()
147   - backend_cfg = backends.get(name, {}) if isinstance(backends.get(name), dict) else {}
148   - if not backend_cfg:
149   - raise ValueError(f"services.embedding.backends.{name} is required")
150   - return name, backend_cfg
151   -
152   -
153   -@lru_cache(maxsize=1)
154   -def get_translation_config() -> TranslationConfig:
155   - return _resolve_translation()
156   -
157   -
158   -@lru_cache(maxsize=1)
159   -def get_embedding_config() -> ServiceConfig:
160   - return _resolve_embedding()
161   -
162   -
163   -@lru_cache(maxsize=1)
164   -def get_rerank_config() -> ServiceConfig:
165   - return _resolve_rerank()
  10 +from typing import Any, Dict, Tuple
  11 +
  12 +from config.loader import get_app_config
  13 +from config.schema import EmbeddingServiceConfig, RerankServiceConfig, TranslationServiceConfig
  14 +
  15 +
  16 +def get_translation_config() -> Dict[str, Any]:
  17 + return get_app_config().services.translation.as_dict()
  18 +
  19 +
  20 +def get_embedding_config() -> EmbeddingServiceConfig:
  21 + return get_app_config().services.embedding
  22 +
  23 +
  24 +def get_rerank_config() -> RerankServiceConfig:
  25 + return get_app_config().services.rerank
166 26  
167 27  
168 28 def get_translation_base_url() -> str:
169   - return str(get_translation_config()["service_url"])
  29 + return get_app_config().services.translation.endpoint
170 30  
171 31  
172 32 def get_translation_cache_config() -> Dict[str, Any]:
173   - return get_translation_cache(get_translation_config())
  33 + return dict(get_app_config().services.translation.cache)
174 34  
175 35  
176 36 def get_embedding_text_base_url() -> str:
177   - provider_cfg = get_embedding_config().providers.get("http", {})
178   - base = os.getenv("EMBEDDING_TEXT_SERVICE_URL") or provider_cfg.get("text_base_url")
  37 + provider_cfg = get_app_config().services.embedding.get_provider_config()
  38 + base = provider_cfg.get("text_base_url")
179 39 if not base:
180   - raise ValueError("Embedding text HTTP base_url is not configured")
  40 + raise ValueError("Embedding text base_url is not configured")
181 41 return str(base).rstrip("/")
182 42  
183 43  
184 44 def get_embedding_image_base_url() -> str:
185   - provider_cfg = get_embedding_config().providers.get("http", {})
186   - base = os.getenv("EMBEDDING_IMAGE_SERVICE_URL") or provider_cfg.get("image_base_url")
  45 + provider_cfg = get_app_config().services.embedding.get_provider_config()
  46 + base = provider_cfg.get("image_base_url")
187 47 if not base:
188   - raise ValueError("Embedding image HTTP base_url is not configured")
  48 + raise ValueError("Embedding image base_url is not configured")
189 49 return str(base).rstrip("/")
190 50  
191 51  
  52 +def get_embedding_backend_config() -> Tuple[str, Dict[str, Any]]:
  53 + cfg = get_app_config().services.embedding
  54 + return cfg.backend, cfg.get_backend_config()
  55 +
  56 +
  57 +def get_embedding_image_backend_config() -> Tuple[str, Dict[str, Any]]:
  58 + cfg = get_app_config().services.embedding
  59 + return cfg.image_backend, cfg.get_image_backend_config()
  60 +
  61 +
  62 +def get_rerank_backend_config() -> Tuple[str, Dict[str, Any]]:
  63 + cfg = get_app_config().services.rerank
  64 + return cfg.backend, cfg.get_backend_config()
  65 +
  66 +
192 67 def get_rerank_base_url() -> str:
193   - base = (
194   - os.getenv("RERANKER_SERVICE_URL")
195   - or get_rerank_config().providers.get("http", {}).get("service_url")
196   - or get_rerank_config().providers.get("http", {}).get("base_url")
197   - )
  68 + provider_cfg = get_app_config().services.rerank.get_provider_config()
  69 + base = provider_cfg.get("service_url") or provider_cfg.get("base_url")
198 70 if not base:
199   - raise ValueError("Rerank HTTP base_url is not configured")
  71 + raise ValueError("Rerank service URL is not configured")
200 72 return str(base).rstrip("/")
201 73  
202 74  
203 75 def get_rerank_service_url() -> str:
204   - """Backward-compatible alias."""
205 76 return get_rerank_base_url()
... ...
config/tenant_config_loader.py
  Show/Hide comments   View file @86d8358
... ... @@ -2,12 +2,13 @@
2 2 ็งŸๆˆท้…็ฝฎๅŠ ่ฝฝๅ™จใ€‚
3 3  
4 4 ไปŽ็ปŸไธ€้…็ฝฎๆ–‡ไปถ๏ผˆconfig.yaml๏ผ‰ๅŠ ่ฝฝ็งŸๆˆท้…็ฝฎ๏ผŒๅŒ…ๆ‹ฌไธป่ฏญ่จ€ๅ’Œ็ดขๅผ•่ฏญ่จ€๏ผˆindex_languages๏ผ‰ใ€‚
5   -ๆ”ฏๆŒๆ—ง้…็ฝฎ translate_to_en / translate_to_zh ็š„ๅ…ผๅฎน่งฃๆžใ€‚
6 5 """
7 6  
8 7 import logging
9 8 from typing import Dict, Any, Optional, List
10 9  
  10 +from config.loader import get_app_config
  11 +
11 12 logger = logging.getLogger(__name__)
12 13  
13 14 # ๆ”ฏๆŒ็š„็ดขๅผ•่ฏญ่จ€๏ผšcode -> display name๏ผˆไพ›ๅ•†ๅฎถๅ‹พ้€‰ไธปๅธ‚ๅœบ่ฏญ่จ€็ญ‰ๅœบๆ™ฏไฝฟ็”จ๏ผ‰
... ... @@ -83,25 +84,13 @@ def resolve_index_languages(
83 84 ) -> List[str]:
84 85 """
85 86 ไปŽ็งŸๆˆท้…็ฝฎ่งฃๆž index_languagesใ€‚
86   - ่‹ฅๅญ˜ๅœจ index_languages ๅˆ™็”จไน‹๏ผ›ๅฆๅˆ™ๆŒ‰ๆ—ง้…็ฝฎ translate_to_en / translate_to_zh ๆŽจๅฏผใ€‚
  87 + ่‹ฅ้…็ฝฎ็ผบๅคฑๆˆ–้žๆณ•๏ผŒๅˆ™ๅ›ž้€€ๅˆฐ้ป˜่ฎค้…็ฝฎใ€‚
87 88 """
88   - if "index_languages" in tenant_config:
89   - normalized = normalize_index_languages(
90   - tenant_config["index_languages"],
91   - tenant_config.get("primary_language") or "en",
92   - )
93   - return normalized if normalized else list(default_index_languages)
94   - primary = (tenant_config.get("primary_language") or "en").strip().lower()
95   - to_en = bool(tenant_config.get("translate_to_en"))
96   - to_zh = bool(tenant_config.get("translate_to_zh"))
97   - langs: List[str] = []
98   - if primary and primary in SOURCE_LANG_CODE_MAP:
99   - langs.append(primary)
100   - for code in ("en", "zh"):
101   - if code not in langs and ((code == "en" and to_en) or (code == "zh" and to_zh)):
102   - if code in SOURCE_LANG_CODE_MAP:
103   - langs.append(code)
104   - return langs if langs else list(default_index_languages)
  89 + normalized = normalize_index_languages(
  90 + tenant_config.get("index_languages"),
  91 + tenant_config.get("primary_language") or "en",
  92 + )
  93 + return normalized if normalized else list(default_index_languages)
105 94  
106 95  
107 96 class TenantConfigLoader:
... ... @@ -122,15 +111,8 @@ class TenantConfigLoader:
122 111 return self._config
123 112  
124 113 try:
125   - from config import ConfigLoader
126   -
127   - config_loader = ConfigLoader()
128   - search_config = config_loader.load_config()
129   - tenant_cfg = search_config.tenant_config
130   - if not isinstance(tenant_cfg, dict):
131   - raise RuntimeError("tenant_config must be an object")
132   -
133   - default_cfg = tenant_cfg.get("default")
  114 + tenant_cfg = get_app_config().tenants
  115 + default_cfg = tenant_cfg.default
134 116 if not isinstance(default_cfg, dict):
135 117 raise RuntimeError("tenant_config.default must be configured in config.yaml")
136 118 default_primary = (default_cfg.get("primary_language") or "en").strip().lower()
... ... @@ -143,7 +125,7 @@ class TenantConfigLoader:
143 125 "tenant_config.default.index_languages must include at least one supported language"
144 126 )
145 127  
146   - tenants_cfg = tenant_cfg.get("tenants", {})
  128 + tenants_cfg = tenant_cfg.tenants
147 129 if not isinstance(tenants_cfg, dict):
148 130 raise RuntimeError("tenant_config.tenants must be an object")
149 131  
... ...
config/utils.py
  Show/Hide comments   View file @86d8358
1 1 """Configuration helper functions for dynamic multi-language search fields."""
2 2  
3 3 from typing import Dict, List
4   -from .config_loader import SearchConfig
  4 +
  5 +from config.schema import SearchConfig
5 6  
6 7  
7 8 def _format_field_with_boost(field_name: str, boost: float) -> str:
... ...
docs/config-system-review-and-redesign.md 0 โ†’ 100644
  Show/Hide comments   View file @86d8358
... ... @@ -0,0 +1,738 @@
  1 +# Configuration System Review And Redesign
  2 +
  3 +## 1. Goal
  4 +
  5 +This document reviews the current configuration system and proposes a practical redesign for long-term maintainability.
  6 +
  7 +The target is a configuration system that is:
  8 +
  9 +- unified in loading and ownership
  10 +- clear in boundaries and precedence
  11 +- visible in effective behavior
  12 +- easy to evolve across development, deployment, and operations
  13 +
  14 +This review is based on the current implementation, not only on the intended architecture in docs.
  15 +
  16 +## 2. Project Context
  17 +
  18 +The repo already defines the right architectural direction:
  19 +
  20 +- `config/config.yaml` should be the main configuration source for search behavior and service wiring
  21 +- `.env` should mainly carry deployment-specific values and secrets
  22 +- provider/backend expansion should stay centralized instead of spreading through business code
  23 +
  24 +That direction is described in:
  25 +
  26 +- [`README.md`](/data/saas-search/README.md)
  27 +- [`docs/DEVELOPER_GUIDE.md`](/data/saas-search/docs/DEVELOPER_GUIDE.md)
  28 +- [`docs/QUICKSTART.md`](/data/saas-search/docs/QUICKSTART.md)
  29 +- [`translation/README.md`](/data/saas-search/translation/README.md)
  30 +
  31 +The problem is not the architectural intent. The problem is that the current implementation only partially follows it.
  32 +
  33 +## 3. Current-State Review
  34 +
  35 +### 3.1 What exists today
  36 +
  37 +The current system effectively has several configuration channels:
  38 +
  39 +- `config/config.yaml`
  40 + - search behavior
  41 + - rerank behavior
  42 + - services registry
  43 + - tenant config
  44 +- `config/config_loader.py`
  45 + - parses search behavior and tenant config into `SearchConfig`
  46 + - also injects some defaults from code
  47 +- `config/services_config.py`
  48 + - reparses `config/config.yaml` again, independently
  49 + - resolves translation, embedding, rerank service config
  50 + - also applies env overrides
  51 +- `config/env_config.py`
  52 + - loads `.env`
  53 + - defines ES, Redis, DB, host/port, service URLs, namespace, model path defaults
  54 +- service-local config modules
  55 + - [`embeddings/config.py`](/data/saas-search/embeddings/config.py)
  56 + - [`reranker/config.py`](/data/saas-search/reranker/config.py)
  57 +- startup scripts
  58 + - derive defaults from shell env, Python config, and YAML in different combinations
  59 +- inline fallbacks in business logic
  60 + - query parsing
  61 + - indexing
  62 + - service startup
  63 +
  64 +### 3.2 Main findings
  65 +
  66 +#### Finding A: there is no single loader for the full effective configuration
  67 +
  68 +`ConfigLoader` and `services_config` both parse `config/config.yaml`, but they do so separately and with different responsibilities.
  69 +
  70 +- [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L148)
  71 +- [`config/services_config.py`](/data/saas-search/config/services_config.py#L33)
  72 +
  73 +Impact:
  74 +
  75 +- the same file is loaded twice through different code paths
  76 +- search config and services config can drift in interpretation
  77 +- alternative config paths are hard to support cleanly
  78 +- tests and tools cannot ask one place for the full effective config tree
  79 +
  80 +#### Finding B: precedence is not explicit, stable, or globally enforced
  81 +
  82 +Current precedence differs by subsystem:
  83 +
  84 +- search behavior mostly comes from YAML plus code defaults
  85 +- embedding and rerank allow env overrides for provider/backend/url
  86 +- translation intentionally blocks some env overrides
  87 +- startup scripts still choose host/port and mode via env
  88 +- some values are reconstructed from other env vars
  89 +
  90 +Examples:
  91 +
  92 +- env override for embedding provider/url/backend:
  93 + - [`config/services_config.py`](/data/saas-search/config/services_config.py#L52)
  94 + - [`config/services_config.py`](/data/saas-search/config/services_config.py#L68)
  95 + - [`config/services_config.py`](/data/saas-search/config/services_config.py#L139)
  96 +- host/port and service URL reconstruction:
  97 + - [`config/env_config.py`](/data/saas-search/config/env_config.py#L55)
  98 + - [`config/env_config.py`](/data/saas-search/config/env_config.py#L75)
  99 +- translator host/port still driven by startup env:
  100 + - [`scripts/start_translator.sh`](/data/saas-search/scripts/start_translator.sh#L28)
  101 +
  102 +Impact:
  103 +
  104 +- operators cannot reliably predict the effective configuration by reading one file
  105 +- the same setting category behaves differently across services
  106 +- incidents become harder to debug because source-of-truth depends on the code path
  107 +
  108 +#### Finding C: defaults are duplicated across YAML and code
  109 +
  110 +There are several layers of default values:
  111 +
  112 +- dataclass defaults in `QueryConfig`
  113 +- fallback defaults in `ConfigLoader._parse_config`
  114 +- defaults in `config.yaml`
  115 +- defaults in `env_config.py`
  116 +- defaults in `embeddings/config.py`
  117 +- defaults in `reranker/config.py`
  118 +- defaults in startup scripts
  119 +
  120 +Examples:
  121 +
  122 +- query defaults duplicated in dataclass and parser:
  123 + - [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L24)
  124 + - [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L240)
  125 +- embedding defaults duplicated in YAML, `services_config`, `embeddings/config.py`, and startup script:
  126 + - [`config/config.yaml`](/data/saas-search/config/config.yaml#L196)
  127 + - [`embeddings/config.py`](/data/saas-search/embeddings/config.py#L14)
  128 + - [`scripts/start_embedding_service.sh`](/data/saas-search/scripts/start_embedding_service.sh#L29)
  129 +- reranker defaults duplicated in YAML and `reranker/config.py`:
  130 + - [`config/config.yaml`](/data/saas-search/config/config.yaml#L214)
  131 + - [`reranker/config.py`](/data/saas-search/reranker/config.py#L6)
  132 +
  133 +Impact:
  134 +
  135 +- changing a default is risky because there may be multiple hidden copies
  136 +- code review cannot easily tell whether a value is authoritative or dead legacy
  137 +- โ€œsame configโ€ may behave differently across processes
  138 +
  139 +#### Finding D: config is still embedded in runtime logic
  140 +
  141 +Some important behavior remains encoded as inline fallback logic rather than declared config.
  142 +
  143 +Examples:
  144 +
  145 +- query-time translation target languages fallback to `["en", "zh"]`:
  146 + - [`query/query_parser.py`](/data/saas-search/query/query_parser.py#L339)
  147 +- indexer text handling and LLM enrichment also fallback to `["en", "zh"]`:
  148 + - [`indexer/document_transformer.py`](/data/saas-search/indexer/document_transformer.py#L216)
  149 + - [`indexer/document_transformer.py`](/data/saas-search/indexer/document_transformer.py#L310)
  150 + - [`indexer/document_transformer.py`](/data/saas-search/indexer/document_transformer.py#L649)
  151 +
  152 +Impact:
  153 +
  154 +- configuration is not fully visible in config files
  155 +- behavior can silently change when tenant config is missing or malformed
  156 +- โ€œdefault behaviorโ€ is spread across business modules
  157 +
  158 +#### Finding E: some configuration assets are not managed as first-class config
  159 +
  160 +Query rewrite is configured through an external file, but the file path is hardcoded and currently inconsistent with the repository content.
  161 +
  162 +- loader expects:
  163 + - [`config/config_loader.py`](/data/saas-search/config/config_loader.py#L162)
  164 +- repo currently contains:
  165 + - [`config/query_rewrite.dict`](/data/saas-search/config/query_rewrite.dict)
  166 +
  167 +There is also an admin API that mutates rewrite rules in memory only:
  168 +
  169 +- [`api/routes/admin.py`](/data/saas-search/api/routes/admin.py#L68)
  170 +- [`query/query_parser.py`](/data/saas-search/query/query_parser.py#L622)
  171 +
  172 +Impact:
  173 +
  174 +- rewrite rules are neither cleanly file-backed nor fully runtime-managed
  175 +- restart behavior is unclear
  176 +- configuration visibility and persistence are weak
  177 +
  178 +#### Finding F: visibility is limited
  179 +
  180 +The system exposes only a small sanitized subset at `/admin/config`.
  181 +
  182 +- [`api/routes/admin.py`](/data/saas-search/api/routes/admin.py#L42)
  183 +
  184 +At the same time, the true effective config includes:
  185 +
  186 +- tenant overlays
  187 +- env overrides
  188 +- service backend selections
  189 +- script-selected modes
  190 +- hidden defaults in code
  191 +
  192 +Impact:
  193 +
  194 +- there is no authoritative โ€œeffective configโ€ view
  195 +- debugging configuration mismatches requires source reading
  196 +- operators cannot easily verify what each process actually started with
  197 +
  198 +#### Finding G: the indexer does not really consume the unified config as a first-class dependency
  199 +
  200 +Indexer startup explicitly says config is loaded only for parity/logging and routes do not depend on it.
  201 +
  202 +- [`api/indexer_app.py`](/data/saas-search/api/indexer_app.py#L76)
  203 +
  204 +Impact:
  205 +
  206 +- configuration is not truly system-wide
  207 +- search-side and indexer-side behavior can drift
  208 +- the current โ€œunified configโ€ is only partially unified
  209 +
  210 +#### Finding H: docs still carry legacy and mixed mental models
  211 +
  212 +Most high-level docs describe the desired centralized model, but some implementation/docs still expose legacy concepts such as `translate_to_en` and `translate_to_zh`.
  213 +
  214 +- desired model:
  215 + - [`README.md`](/data/saas-search/README.md#L78)
  216 + - [`docs/DEVELOPER_GUIDE.md`](/data/saas-search/docs/DEVELOPER_GUIDE.md#L207)
  217 + - [`translation/README.md`](/data/saas-search/translation/README.md#L161)
  218 +- legacy tenant translation flags still documented:
  219 + - [`indexer/README.md`](/data/saas-search/indexer/README.md#L39)
  220 +
  221 +Impact:
  222 +
  223 +- new developers may follow old mental models
  224 +- cleanup work keeps getting deferred because old and new systems appear both โ€œsupportedโ€
  225 +
  226 +## 4. Design Principles For The Redesign
  227 +
  228 +The redesign should follow these rules.
  229 +
  230 +### 4.1 One logical configuration system
  231 +
  232 +It is acceptable to have multiple files, but not multiple loaders with overlapping ownership.
  233 +
  234 +There must be one loader pipeline that produces one typed `AppConfig`.
  235 +
  236 +### 4.2 Configuration files declare, parser code interprets, env provides runtime injection
  237 +
  238 +Responsibilities should be:
  239 +
  240 +- configuration files
  241 + - declare non-secret desired behavior and non-secret deployable settings
  242 +- parsing logic
  243 + - load, merge, validate, normalize, and expose typed config
  244 + - never invent hidden business behavior
  245 +- environment variables
  246 + - carry secrets and a small set of runtime/process values
  247 + - do not redefine business behavior casually
  248 +
  249 +### 4.3 One precedence rule for the whole system
  250 +
  251 +Every config category should follow the same merge model unless explicitly exempted.
  252 +
  253 +### 4.4 No silent implicit fallback for business behavior
  254 +
  255 +Fail fast at startup when required config is missing or invalid.
  256 +
  257 +Do not silently fall back to legacy behavior such as hardcoded language lists.
  258 +
  259 +### 4.5 Effective configuration must be observable
  260 +
  261 +Every service should be able to show:
  262 +
  263 +- config version or hash
  264 +- source files loaded
  265 +- environment name
  266 +- sanitized effective configuration
  267 +
  268 +## 5. Recommended Target Design
  269 +
  270 +## 5.1 Boundary model
  271 +
  272 +Use three clear layers.
  273 +
  274 +### Layer 1: repository-managed static config
  275 +
  276 +Purpose:
  277 +
  278 +- search behavior
  279 +- tenant behavior
  280 +- provider/backend registry
  281 +- non-secret service topology defaults
  282 +- feature switches
  283 +
  284 +Examples:
  285 +
  286 +- field boosts
  287 +- query strategy
  288 +- rerank fusion parameters
  289 +- tenant language plans
  290 +- translation capability registry
  291 +- embedding backend selection default
  292 +
  293 +### Layer 2: environment-specific overlays
  294 +
  295 +Purpose:
  296 +
  297 +- per-environment non-secret differences
  298 +- service endpoints by environment
  299 +- resource sizing defaults by environment
  300 +- dev/test/prod operational differences
  301 +
  302 +Examples:
  303 +
  304 +- local embedding URL vs production URL
  305 +- dev rerank backend vs prod rerank backend
  306 +- lower concurrency in local development
  307 +
  308 +### Layer 3: environment variables
  309 +
  310 +Purpose:
  311 +
  312 +- secrets
  313 +- bind host/port
  314 +- external infrastructure credentials
  315 +- container-orchestrator last-mile injection
  316 +
  317 +Examples:
  318 +
  319 +- `ES_HOST`, `ES_USERNAME`, `ES_PASSWORD`
  320 +- `DB_HOST`, `DB_USERNAME`, `DB_PASSWORD`
  321 +- `REDIS_HOST`, `REDIS_PASSWORD`
  322 +- `DASHSCOPE_API_KEY`, `DEEPL_AUTH_KEY`
  323 +- `API_HOST`, `API_PORT`, `INDEXER_PORT`, `TRANSLATION_PORT`
  324 +
  325 +Rule:
  326 +
  327 +- environment variables should not be the normal path for choosing business behavior such as translation model, embedding backend, or tenant language policy
  328 +- if an env override is allowed for a non-secret field, it must be explicitly listed and documented as an operational override, not a hidden convention
  329 +
  330 +## 5.2 Unified precedence
  331 +
  332 +Recommended precedence:
  333 +
  334 +1. schema defaults in code
  335 +2. `config/base.yaml`
  336 +3. `config/environments/<env>.yaml`
  337 +4. tenant overlay from `config/tenants/`
  338 +5. environment variables for the explicitly allowed runtime keys
  339 +6. CLI flags for the current process only
  340 +
  341 +Important rule:
  342 +
  343 +- only one module may implement this merge logic
  344 +- no business module may call `os.getenv()` directly for configuration
  345 +
  346 +## 5.3 Recommended directory structure
  347 +
  348 +```text
  349 +config/
  350 + schema.py
  351 + loader.py
  352 + sources.py
  353 + base.yaml
  354 + environments/
  355 + dev.yaml
  356 + test.yaml
  357 + prod.yaml
  358 + tenants/
  359 + _default.yaml
  360 + 1.yaml
  361 + 162.yaml
  362 + 170.yaml
  363 + dictionaries/
  364 + query_rewrite.dict
  365 + README.md
  366 +.env.example
  367 +```
  368 +
  369 +Notes:
  370 +
  371 +- `base.yaml` contains shared defaults and feature behavior
  372 +- `environments/*.yaml` contains environment-specific non-secret overrides
  373 +- `tenants/*.yaml` contains tenant-specific overrides only
  374 +- `dictionaries/` stores first-class config assets such as rewrite dictionaries
  375 +- `schema.py` defines the typed config model
  376 +- `loader.py` is the only entry point that loads and merges config
  377 +
  378 +If the team prefers fewer files, `tenants.yaml` is also acceptable. The key requirement is not โ€œone fileโ€, but โ€œone loading model with clear ownershipโ€.
  379 +
  380 +## 5.4 Typed configuration model
  381 +
  382 +Introduce one root object, for example:
  383 +
  384 +```python
  385 +class AppConfig(BaseModel):
  386 + runtime: RuntimeConfig
  387 + infrastructure: InfrastructureConfig
  388 + search: SearchConfig
  389 + services: ServicesConfig
  390 + tenants: TenantCatalogConfig
  391 + assets: ConfigAssets
  392 +```
  393 +
  394 +Suggested subtrees:
  395 +
  396 +- `runtime`
  397 + - environment name
  398 + - config revision/hash
  399 + - bind addresses/ports
  400 +- `infrastructure`
  401 + - ES
  402 + - DB
  403 + - Redis
  404 + - index namespace
  405 +- `search`
  406 + - field boosts
  407 + - query config
  408 + - function score
  409 + - rerank behavior
  410 + - spu config
  411 +- `services`
  412 + - translation
  413 + - embedding
  414 + - rerank
  415 +- `tenants`
  416 + - default tenant config
  417 + - tenant overrides
  418 +- `assets`
  419 + - rewrite dictionary path
  420 +
  421 +Benefits:
  422 +
  423 +- one validated object shared by backend, indexer, translator, embedding, reranker
  424 +- one place for defaults
  425 +- one place for schema evolution
  426 +
  427 +## 5.5 Loading flow
  428 +
  429 +Recommended loading flow:
  430 +
  431 +1. determine `APP_ENV` or `RUNTIME_ENV`
  432 +2. load schema defaults
  433 +3. load `config/base.yaml`
  434 +4. load `config/environments/<env>.yaml` if present
  435 +5. load tenant files
  436 +6. inject first-class assets such as rewrite dictionary
  437 +7. apply allowed env overrides
  438 +8. validate the final `AppConfig`
  439 +9. freeze and cache the config object
  440 +10. expose a sanitized effective-config view
  441 +
  442 +Important:
  443 +
  444 +- every process should call the same loader
  445 +- services should receive a resolved `AppConfig`, not re-open YAML independently
  446 +
  447 +## 5.6 Clear responsibility split
  448 +
  449 +### Configuration files are responsible for
  450 +
  451 +- what the system should do
  452 +- what providers/backends are available
  453 +- which features are enabled
  454 +- tenant language/index policies
  455 +- non-secret service topology
  456 +
  457 +### Parser/loader code is responsible for
  458 +
  459 +- locating sources
  460 +- merge precedence
  461 +- type validation
  462 +- normalization
  463 +- deprecation warnings
  464 +- producing the final immutable config object
  465 +
  466 +### Environment variables are responsible for
  467 +
  468 +- secrets
  469 +- bind addresses/ports
  470 +- infrastructure endpoints when the deployment platform injects them
  471 +- a very small set of documented operational overrides
  472 +
  473 +### Business code is not responsible for
  474 +
  475 +- inventing defaults for missing config
  476 +- loading YAML directly
  477 +- calling `os.getenv()` for normal application behavior
  478 +
  479 +## 5.7 How to handle service config
  480 +
  481 +Unify all service-facing config under one structure:
  482 +
  483 +```yaml
  484 +services:
  485 + translation:
  486 + endpoint: "http://translator:6006"
  487 + timeout_sec: 10
  488 + default_model: "llm"
  489 + default_scene: "general"
  490 + capabilities: ...
  491 + embedding:
  492 + endpoint:
  493 + text: "http://embedding:6005"
  494 + image: "http://embedding-image:6008"
  495 + backend: "tei"
  496 + backends: ...
  497 + rerank:
  498 + endpoint: "http://reranker:6007/rerank"
  499 + backend: "qwen3_vllm"
  500 + backends: ...
  501 +```
  502 +
  503 +Rules:
  504 +
  505 +- `endpoint` is how callers reach the service
  506 +- `backend` is how the service itself is implemented
  507 +- only the service process cares about `backend`
  508 +- only callers care about `endpoint`
  509 +- both still belong to the same config tree, because they are part of one system
  510 +
  511 +## 5.8 How to handle tenant config
  512 +
  513 +Tenant config should become explicit policy, not translation-era leftovers.
  514 +
  515 +Recommended tenant fields:
  516 +
  517 +- `primary_language`
  518 +- `index_languages`
  519 +- `search_languages`
  520 +- `translation_policy`
  521 +- `facet_policy`
  522 +- optional tenant-specific ranking overrides
  523 +
  524 +Avoid keeping `translate_to_en` and `translate_to_zh` as active concepts in the long-term model.
  525 +
  526 +If compatibility is needed, support them only in the loader as deprecated aliases and emit warnings.
  527 +
  528 +## 5.9 How to handle rewrite rules and similar assets
  529 +
  530 +Treat them as declared config assets.
  531 +
  532 +Recommended rules:
  533 +
  534 +- file path declared in config
  535 +- one canonical location under `config/dictionaries/`
  536 +- loader validates presence and format
  537 +- admin runtime updates either:
  538 + - are removed, or
  539 + - write back through a controlled persistence path
  540 +
  541 +Do not keep a hybrid model where startup loads one file and admin mutates only in memory.
  542 +
  543 +## 5.10 Observability improvements
  544 +
  545 +Add the following:
  546 +
  547 +- `config dump` CLI that prints sanitized effective config
  548 +- startup log with config hash, environment, and config file list
  549 +- `/admin/config/effective` endpoint returning sanitized effective config
  550 +- `/admin/config/meta` endpoint returning:
  551 + - environment
  552 + - config hash
  553 + - loaded source files
  554 + - deprecated keys in use
  555 +
  556 +This is important for operations and for multi-service debugging.
  557 +
  558 +## 6. Practical Refactor Plan
  559 +
  560 +The refactor should be incremental.
  561 +
  562 +### Phase 1: establish the new config core without changing behavior
  563 +
  564 +- create `config/schema.py`
  565 +- create `config/loader.py`
  566 +- move all current defaults into schema models
  567 +- make loader read current `config/config.yaml`
  568 +- make loader read `.env` only for approved keys
  569 +- expose one `get_app_config()`
  570 +
  571 +Result:
  572 +
  573 +- same behavior, but one typed root config becomes available
  574 +
  575 +### Phase 2: remove duplicate readers
  576 +
  577 +- make `services_config.py` a thin adapter over `get_app_config()`
  578 +- make `tenant_config_loader.py` read from `get_app_config()`
  579 +- stop reparsing YAML in `services_config.py`
  580 +- stop service modules from depending on legacy local config modules for behavior
  581 +
  582 +Result:
  583 +
  584 +- one parsing path
  585 +- fewer divergence risks
  586 +
  587 +### Phase 3: move hidden defaults out of business logic
  588 +
  589 +- remove hardcoded fallback language lists from query/indexer modules
  590 +- require tenant defaults to come from config schema only
  591 +- remove duplicate behavior defaults from service code
  592 +
  593 +Result:
  594 +
  595 +- behavior becomes visible and reviewable
  596 +
  597 +### Phase 4: clean service startup configuration
  598 +
  599 +- make startup scripts ask the unified loader for resolved values
  600 +- keep only bind host/port and secret injection in shell env
  601 +- retire or reduce `embeddings/config.py` and `reranker/config.py`
  602 +
  603 +Result:
  604 +
  605 +- startup behavior matches runtime config model
  606 +
  607 +### Phase 5: split config files by responsibility
  608 +
  609 +- keep a single root loader
  610 +- split current giant `config.yaml` into:
  611 + - `base.yaml`
  612 + - `environments/<env>.yaml`
  613 + - `tenants/*.yaml`
  614 + - `dictionaries/query_rewrite.dict`
  615 +
  616 +Result:
  617 +
  618 +- config remains unified logically, but is easier to read and maintain physically
  619 +
  620 +### Phase 6: deprecate legacy compatibility
  621 +
  622 +- deprecate `translate_to_en` and `translate_to_zh`
  623 +- deprecate env-based backend/provider selection except for explicitly approved keys
  624 +- remove old code paths after one or two release cycles
  625 +
  626 +Result:
  627 +
  628 +- the system becomes simpler instead of carrying two generations forever
  629 +
  630 +## 7. Concrete Rules To Adopt
  631 +
  632 +These rules should be documented and enforced in code review.
  633 +
  634 +### Rule 1
  635 +
  636 +Only `config/loader.py` may load config files or `.env`.
  637 +
  638 +### Rule 2
  639 +
  640 +Only `config/loader.py` may read `os.getenv()` for application config.
  641 +
  642 +### Rule 3
  643 +
  644 +Business modules receive typed config objects and do not read files or env directly.
  645 +
  646 +### Rule 4
  647 +
  648 +Each config key has one owner.
  649 +
  650 +Examples:
  651 +
  652 +- `search.query.knn_boost` belongs to search behavior config
  653 +- `services.embedding.backend` belongs to service implementation config
  654 +- `infrastructure.redis.password` belongs to env/secrets
  655 +
  656 +### Rule 5
  657 +
  658 +Every fallback must be either:
  659 +
  660 +- declared in schema defaults, or
  661 +- rejected at startup
  662 +
  663 +No hidden fallback in runtime logic.
  664 +
  665 +### Rule 6
  666 +
  667 +Every configuration asset must be visible in one of these places only:
  668 +
  669 +- config file
  670 +- env var
  671 +- generated runtime metadata
  672 +
  673 +Not inside parser code as an implicit constant.
  674 +
  675 +## 8. Recommended Naming Conventions
  676 +
  677 +Suggested conventions:
  678 +
  679 +- config keys use noun-based hierarchical names
  680 +- avoid mixing transport and implementation concepts in one field
  681 +- use `endpoint` for caller-facing addresses
  682 +- use `backend` for service-internal implementation choice
  683 +- use `enabled` only for true feature toggles
  684 +- use `default_*` only when a real selection happens at runtime
  685 +
  686 +Examples:
  687 +
  688 +- good: `services.rerank.endpoint`
  689 +- good: `services.rerank.backend`
  690 +- good: `tenants.default.index_languages`
  691 +- avoid: `service_url`, `base_url`, `provider`, `backend`, and script env all meaning slightly different things without a common model
  692 +
  693 +## 9. Highest-Priority Cleanup Items
  694 +
  695 +If the team wants the shortest path to improvement, start here:
  696 +
  697 +1. build one root `AppConfig`
  698 +2. make `services_config.py` stop reparsing YAML
  699 +3. declare rewrite dictionary path explicitly and fix the current mismatch
  700 +4. remove hardcoded `["en", "zh"]` fallbacks from query/indexer logic
  701 +5. replace `/admin/config` with an effective-config endpoint
  702 +6. retire `embeddings/config.py` and `reranker/config.py` as behavior sources
  703 +7. deprecate legacy tenant translation flags
  704 +
  705 +## 10. Expected Outcome
  706 +
  707 +After the redesign:
  708 +
  709 +- developers can answer โ€œwhere does this setting come from?โ€ in one step
  710 +- operators can see effective config without reading source code
  711 +- backend, indexer, translator, embedding, and reranker all share one model
  712 +- tenant behavior is explicit instead of partially implicit
  713 +- migration becomes safer because defaults and precedence are centralized
  714 +- adding a new provider/backend becomes configuration extension, not configuration archaeology
  715 +
  716 +## 11. Summary
  717 +
  718 +The current system has the right intent but not yet the right implementation shape.
  719 +
  720 +Today the main problems are:
  721 +
  722 +- duplicate config loaders
  723 +- inconsistent precedence
  724 +- duplicated defaults
  725 +- config hidden in runtime logic
  726 +- weak effective-config visibility
  727 +- leftover legacy concepts
  728 +
  729 +The recommended direction is:
  730 +
  731 +- one root typed config
  732 +- one loader pipeline
  733 +- explicit layered sources
  734 +- narrow env responsibility
  735 +- no hidden business fallbacks
  736 +- observable effective config
  737 +
  738 +That design is practical to implement incrementally in this repository and aligns well with the project's multi-tenant, multi-service, provider/backend-based architecture.
... ...
embeddings/config.py
  Show/Hide comments   View file @86d8358
1   -"""
2   -Embedding module configuration.
  1 +"""Embedding service compatibility config derived from unified app config."""
3 2  
4   -This module is intentionally a plain Python file (no env var parsing, no extra deps).
5   -Edit values here to configure:
6   -- server host/port
7   -- local model settings (paths/devices/batch sizes)
8   -"""
  3 +from __future__ import annotations
9 4  
10 5 from typing import Optional
11   -import os
  6 +
  7 +from config.loader import get_app_config
12 8  
13 9  
14 10 class EmbeddingConfig(object):
15   - # Server
16   - HOST = os.getenv("EMBEDDING_HOST", "0.0.0.0")
17   - PORT = int(os.getenv("EMBEDDING_PORT", 6005))
18   -
19   - # Text backend defaults
20   - TEXT_MODEL_ID = os.getenv("TEXT_MODEL_ID", "Qwen/Qwen3-Embedding-0.6B")
21   - # Keep TEXT_MODEL_DIR as an alias so code can refer to one canonical text model value.
22   - TEXT_MODEL_DIR = TEXT_MODEL_ID
23   - TEXT_DEVICE = os.getenv("TEXT_DEVICE", "cuda") # "cuda" or "cpu"
24   - TEXT_BATCH_SIZE = int(os.getenv("TEXT_BATCH_SIZE", "32"))
25   - TEXT_NORMALIZE_EMBEDDINGS = os.getenv("TEXT_NORMALIZE_EMBEDDINGS", "true").lower() in ("1", "true", "yes")
26   - TEI_BASE_URL = os.getenv("TEI_BASE_URL", "http://127.0.0.1:8080")
27   - TEI_TIMEOUT_SEC = int(os.getenv("TEI_TIMEOUT_SEC", "60"))
28   -
29   - # Image embeddings
30   - # Option A: clip-as-service (Jina CLIP server, recommended)
31   - USE_CLIP_AS_SERVICE = os.getenv("USE_CLIP_AS_SERVICE", "true").lower() in ("1", "true", "yes")
32   - CLIP_AS_SERVICE_SERVER = os.getenv("CLIP_AS_SERVICE_SERVER", "grpc://127.0.0.1:51000")
33   - CLIP_AS_SERVICE_MODEL_NAME = os.getenv("CLIP_AS_SERVICE_MODEL_NAME", "CN-CLIP/ViT-L-14")
34   -
35   - # Option B: local CN-CLIP (when USE_CLIP_AS_SERVICE=false)
36   - IMAGE_MODEL_NAME = os.getenv("IMAGE_MODEL_NAME", "ViT-L-14")
37   - IMAGE_DEVICE = None # type: Optional[str] # "cuda" / "cpu" / None(auto)
38   -
39   - # Service behavior
40   - IMAGE_BATCH_SIZE = 8
41   - IMAGE_NORMALIZE_EMBEDDINGS = os.getenv("IMAGE_NORMALIZE_EMBEDDINGS", "true").lower() in ("1", "true", "yes")
  11 + def __init__(self) -> None:
  12 + app_config = get_app_config()
  13 + runtime = app_config.runtime
  14 + services = app_config.services.embedding
  15 + text_backend = services.get_backend_config()
  16 + image_backend = services.get_image_backend_config()
  17 +
  18 + self.HOST = runtime.embedding_host
  19 + self.PORT = runtime.embedding_port
  20 +
  21 + self.TEXT_MODEL_ID = str(text_backend.get("model_id") or "Qwen/Qwen3-Embedding-0.6B")
  22 + self.TEXT_MODEL_DIR = self.TEXT_MODEL_ID
  23 + self.TEXT_DEVICE = str(text_backend.get("device") or "cuda")
  24 + self.TEXT_BATCH_SIZE = int(text_backend.get("batch_size", 32))
  25 + self.TEXT_NORMALIZE_EMBEDDINGS = bool(text_backend.get("normalize_embeddings", True))
  26 + self.TEI_BASE_URL = str(text_backend.get("base_url") or "http://127.0.0.1:8080")
  27 + self.TEI_TIMEOUT_SEC = int(text_backend.get("timeout_sec", 60))
  28 +
  29 + self.USE_CLIP_AS_SERVICE = services.image_backend == "clip_as_service"
  30 + self.CLIP_AS_SERVICE_SERVER = str(image_backend.get("server") or "grpc://127.0.0.1:51000")
  31 + self.CLIP_AS_SERVICE_MODEL_NAME = str(image_backend.get("model_name") or "CN-CLIP/ViT-L-14")
  32 +
  33 + self.IMAGE_MODEL_NAME = str(image_backend.get("model_name") or "ViT-L-14")
  34 + self.IMAGE_DEVICE = image_backend.get("device") # type: Optional[str]
  35 + self.IMAGE_BATCH_SIZE = int(image_backend.get("batch_size", 8))
  36 + self.IMAGE_NORMALIZE_EMBEDDINGS = bool(image_backend.get("normalize_embeddings", True))
42 37  
43 38  
44 39 CONFIG = EmbeddingConfig()
... ...
embeddings/image_encoder.py
  Show/Hide comments   View file @86d8358
... ... @@ -9,8 +9,8 @@ from PIL import Image
9 9  
10 10 logger = logging.getLogger(__name__)
11 11  
  12 +from config.loader import get_app_config
12 13 from config.services_config import get_embedding_image_base_url
13   -from config.env_config import REDIS_CONFIG
14 14 from embeddings.cache_keys import build_image_cache_key
15 15 from embeddings.redis_embedding_cache import RedisEmbeddingCache
16 16  
... ... @@ -24,10 +24,11 @@ class CLIPImageEncoder:
24 24  
25 25 def __init__(self, service_url: Optional[str] = None):
26 26 resolved_url = service_url or get_embedding_image_base_url()
  27 + redis_config = get_app_config().infrastructure.redis
27 28 self.service_url = str(resolved_url).rstrip("/")
28 29 self.endpoint = f"{self.service_url}/embed/image"
29 30 # Reuse embedding cache prefix, but separate namespace for images to avoid collisions.
30   - self.cache_prefix = str(REDIS_CONFIG.get("embedding_cache_prefix", "embedding")).strip() or "embedding"
  31 + self.cache_prefix = str(redis_config.embedding_cache_prefix).strip() or "embedding"
31 32 logger.info("Creating CLIPImageEncoder instance with service URL: %s", self.service_url)
32 33 self.cache = RedisEmbeddingCache(
33 34 key_prefix=self.cache_prefix,
... ...
embeddings/redis_embedding_cache.py
  Show/Hide comments   View file @86d8358
... ... @@ -20,7 +20,7 @@ try:
20 20 except ImportError: # pragma: no cover - runtime fallback for minimal envs
21 21 redis = None # type: ignore[assignment]
22 22  
23   -from config.env_config import REDIS_CONFIG
  23 +from config.loader import get_app_config
24 24 from embeddings.bf16 import decode_embedding_from_redis, encode_embedding_for_redis
25 25  
26 26 logger = logging.getLogger(__name__)
... ... @@ -37,7 +37,8 @@ class RedisEmbeddingCache:
37 37 ):
38 38 self.key_prefix = (key_prefix or "").strip() or "embedding"
39 39 self.namespace = (namespace or "").strip()
40   - self.expire_time = expire_time or timedelta(days=REDIS_CONFIG.get("cache_expire_days", 180))
  40 + redis_config = get_app_config().infrastructure.redis
  41 + self.expire_time = expire_time or timedelta(days=redis_config.cache_expire_days)
41 42  
42 43 if redis_client is not None:
43 44 self.redis_client = redis_client
... ... @@ -50,13 +51,13 @@ class RedisEmbeddingCache:
50 51  
51 52 try:
52 53 client = redis.Redis(
53   - host=REDIS_CONFIG.get("host", "localhost"),
54   - port=REDIS_CONFIG.get("port", 6479),
55   - password=REDIS_CONFIG.get("password"),
  54 + host=redis_config.host,
  55 + port=redis_config.port,
  56 + password=redis_config.password,
56 57 decode_responses=False,
57   - socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),
58   - socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),
59   - retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False),
  58 + socket_timeout=redis_config.socket_timeout,
  59 + socket_connect_timeout=redis_config.socket_connect_timeout,
  60 + retry_on_timeout=redis_config.retry_on_timeout,
60 61 health_check_interval=10,
61 62 )
62 63 client.ping()
... ...
embeddings/server.py
  Show/Hide comments   View file @86d8358
... ... @@ -470,16 +470,8 @@ def load_models():
470 470 if backend_name == "tei":
471 471 from embeddings.text_embedding_tei import TEITextModel
472 472  
473   - base_url = (
474   - os.getenv("TEI_BASE_URL")
475   - or backend_cfg.get("base_url")
476   - or CONFIG.TEI_BASE_URL
477   - )
478   - timeout_sec = int(
479   - os.getenv("TEI_TIMEOUT_SEC")
480   - or backend_cfg.get("timeout_sec")
481   - or CONFIG.TEI_TIMEOUT_SEC
482   - )
  473 + base_url = backend_cfg.get("base_url") or CONFIG.TEI_BASE_URL
  474 + timeout_sec = int(backend_cfg.get("timeout_sec") or CONFIG.TEI_TIMEOUT_SEC)
483 475 logger.info("Loading text backend: tei (base_url=%s)", base_url)
484 476 _text_model = TEITextModel(
485 477 base_url=str(base_url),
... ... @@ -488,11 +480,7 @@ def load_models():
488 480 elif backend_name == "local_st":
489 481 from embeddings.text_embedding_sentence_transformers import Qwen3TextModel
490 482  
491   - model_id = (
492   - os.getenv("TEXT_MODEL_ID")
493   - or backend_cfg.get("model_id")
494   - or CONFIG.TEXT_MODEL_ID
495   - )
  483 + model_id = backend_cfg.get("model_id") or CONFIG.TEXT_MODEL_ID
496 484 logger.info("Loading text backend: local_st (model=%s)", model_id)
497 485 _text_model = Qwen3TextModel(model_id=str(model_id))
498 486 _start_text_batch_worker()
... ...
embeddings/text_encoder.py
  Show/Hide comments   View file @86d8358
... ... @@ -9,13 +9,11 @@ import requests
9 9  
10 10 logger = logging.getLogger(__name__)
11 11  
  12 +from config.loader import get_app_config
12 13 from config.services_config import get_embedding_text_base_url
13 14 from embeddings.cache_keys import build_text_cache_key
14 15 from embeddings.redis_embedding_cache import RedisEmbeddingCache
15 16  
16   -# Try to import REDIS_CONFIG, but allow import to fail
17   -from config.env_config import REDIS_CONFIG
18   -
19 17  
20 18 class TextEmbeddingEncoder:
21 19 """
... ... @@ -24,10 +22,11 @@ class TextEmbeddingEncoder:
24 22  
25 23 def __init__(self, service_url: Optional[str] = None):
26 24 resolved_url = service_url or get_embedding_text_base_url()
  25 + redis_config = get_app_config().infrastructure.redis
27 26 self.service_url = str(resolved_url).rstrip("/")
28 27 self.endpoint = f"{self.service_url}/embed/text"
29   - self.expire_time = timedelta(days=REDIS_CONFIG.get("cache_expire_days", 180))
30   - self.cache_prefix = str(REDIS_CONFIG.get("embedding_cache_prefix", "embedding")).strip() or "embedding"
  28 + self.expire_time = timedelta(days=redis_config.cache_expire_days)
  29 + self.cache_prefix = str(redis_config.embedding_cache_prefix).strip() or "embedding"
31 30 logger.info("Creating TextEmbeddingEncoder instance with service URL: %s", self.service_url)
32 31  
33 32 self.cache = RedisEmbeddingCache(
... ...
indexer/document_transformer.py
  Show/Hide comments   View file @86d8358
... ... @@ -13,7 +13,6 @@ import numpy as np
13 13 import logging
14 14 import re
15 15 from typing import Dict, Any, Optional, List
16   -from config import ConfigLoader
17 16 from indexer.product_enrich import analyze_products
18 17  
19 18 logger = logging.getLogger(__name__)
... ...
indexer/incremental_service.py
  Show/Hide comments   View file @86d8358
... ... @@ -13,7 +13,7 @@ from indexer.mapping_generator import get_tenant_index_name
13 13 from indexer.indexer_logger import (
14 14 get_indexer_logger, log_index_request, log_index_result, log_spu_processing
15 15 )
16   -from config import ConfigLoader
  16 +from config import get_app_config
17 17 from translation import create_translation_client
18 18  
19 19 # Configure logger
... ... @@ -51,7 +51,7 @@ class IncrementalIndexerService:
51 51  
52 52 def _eager_init(self) -> None:
53 53 """Strict eager initialization. Any dependency failure should fail fast."""
54   - self._config = ConfigLoader("config/config.yaml").load_config()
  54 + self._config = get_app_config().search
55 55 self._searchable_option_dimensions = (
56 56 getattr(self._config.spu_config, "searchable_option_dimensions", None)
57 57 or ["option1", "option2", "option3"]
... ...
indexer/indexing_utils.py
  Show/Hide comments   View file @86d8358
... ... @@ -7,7 +7,7 @@
7 7 import logging
8 8 from typing import Dict, Any, Optional
9 9 from sqlalchemy import Engine, text
10   -from config import ConfigLoader
  10 +from config import get_app_config
11 11 from config.tenant_config_loader import get_tenant_config_loader
12 12 from indexer.document_transformer import SPUDocumentTransformer
13 13 from translation import create_translation_client
... ... @@ -92,8 +92,7 @@ def create_document_transformer(
92 92 or config is None
93 93 ):
94 94 if config is None:
95   - config_loader = ConfigLoader()
96   - config = config_loader.load_config()
  95 + config = get_app_config().search
97 96  
98 97 if searchable_option_dimensions is None:
99 98 searchable_option_dimensions = config.spu_config.searchable_option_dimensions
... ...
indexer/mapping_generator.py
  Show/Hide comments   View file @86d8358
... ... @@ -9,7 +9,7 @@ import json
9 9 import logging
10 10 from pathlib import Path
11 11  
12   -from config.env_config import ES_INDEX_NAMESPACE
  12 +from config.loader import get_app_config
13 13  
14 14 logger = logging.getLogger(__name__)
15 15  
... ... @@ -30,7 +30,7 @@ def get_tenant_index_name(tenant_id: str) -&gt; str:
30 30 ๅ…ถไธญ ES_INDEX_NAMESPACE ็”ฑ config.env_config.ES_INDEX_NAMESPACE ๆŽงๅˆถ๏ผŒ
31 31 ็”จไบŽๅŒบๅˆ† prod/uat/test ็ญ‰ไธๅŒ่ฟ่กŒ็Žฏๅขƒใ€‚
32 32 """
33   - prefix = ES_INDEX_NAMESPACE or ""
  33 + prefix = get_app_config().runtime.index_namespace or ""
34 34 return f"{prefix}search_products_tenant_{tenant_id}"
35 35  
36 36  
... ...
indexer/product_enrich.py
  Show/Hide comments   View file @86d8358
... ... @@ -20,7 +20,7 @@ import redis
20 20 import requests
21 21 from pathlib import Path
22 22  
23   -from config.env_config import REDIS_CONFIG
  23 +from config.loader import get_app_config
24 24 from config.tenant_config_loader import SOURCE_LANG_CODE_MAP
25 25 from indexer.product_enrich_prompts import (
26 26 SYSTEM_MESSAGE,
... ... @@ -91,19 +91,20 @@ logger.info(&quot;Verbose LLM logs are written to: %s&quot;, verbose_log_file)
91 91  
92 92  
93 93 # Redis ็ผ“ๅญ˜๏ผˆ็”จไบŽ anchors / ่ฏญไน‰ๅฑžๆ€ง๏ผ‰
94   -ANCHOR_CACHE_PREFIX = REDIS_CONFIG.get("anchor_cache_prefix", "product_anchors")
95   -ANCHOR_CACHE_EXPIRE_DAYS = int(REDIS_CONFIG.get("anchor_cache_expire_days", 30))
  94 +_REDIS_CONFIG = get_app_config().infrastructure.redis
  95 +ANCHOR_CACHE_PREFIX = _REDIS_CONFIG.anchor_cache_prefix
  96 +ANCHOR_CACHE_EXPIRE_DAYS = int(_REDIS_CONFIG.anchor_cache_expire_days)
96 97 _anchor_redis: Optional[redis.Redis] = None
97 98  
98 99 try:
99 100 _anchor_redis = redis.Redis(
100   - host=REDIS_CONFIG.get("host", "localhost"),
101   - port=REDIS_CONFIG.get("port", 6479),
102   - password=REDIS_CONFIG.get("password"),
  101 + host=_REDIS_CONFIG.host,
  102 + port=_REDIS_CONFIG.port,
  103 + password=_REDIS_CONFIG.password,
103 104 decode_responses=True,
104   - socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),
105   - socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),
106   - retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False),
  105 + socket_timeout=_REDIS_CONFIG.socket_timeout,
  106 + socket_connect_timeout=_REDIS_CONFIG.socket_connect_timeout,
  107 + retry_on_timeout=_REDIS_CONFIG.retry_on_timeout,
107 108 health_check_interval=10,
108 109 )
109 110 _anchor_redis.ping()
... ...
... ... @@ -16,8 +16,7 @@ import json
16 16 # Add parent directory to path
17 17 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
18 18  
19   -from config import ConfigLoader
20   -from config.env_config import ES_CONFIG
  19 +from config import get_app_config
21 20 from utils import ESClient
22 21 from search import Searcher
23 22 from suggestion import SuggestionIndexBuilder
... ... @@ -61,8 +60,7 @@ def cmd_serve_indexer(args):
61 60 def cmd_search(args):
62 61 """Test search from command line."""
63 62 # Load config
64   - config_loader = ConfigLoader("config/config.yaml")
65   - config = config_loader.load_config()
  63 + config = get_app_config().search
66 64  
67 65 # Initialize ES and searcher
68 66 es_client = ESClient(hosts=[args.es_host])
... ... @@ -106,8 +104,9 @@ def cmd_search(args):
106 104 def cmd_build_suggestions(args):
107 105 """Build/update suggestion index for a tenant."""
108 106 # Initialize ES client with optional authentication
109   - es_username = os.getenv("ES_USERNAME") or ES_CONFIG.get("username")
110   - es_password = os.getenv("ES_PASSWORD") or ES_CONFIG.get("password")
  107 + es_cfg = get_app_config().infrastructure.elasticsearch
  108 + es_username = es_cfg.username
  109 + es_password = es_cfg.password
111 110 if es_username and es_password:
112 111 es_client = ESClient(hosts=[args.es_host], username=es_username, password=es_password)
113 112 else:
... ... @@ -117,11 +116,12 @@ def cmd_build_suggestions(args):
117 116 return 1
118 117  
119 118 # Build DB config directly from environment to avoid dotenv dependency
120   - db_host = os.getenv("DB_HOST")
121   - db_port = int(os.getenv("DB_PORT", "3306"))
122   - db_name = os.getenv("DB_DATABASE")
123   - db_user = os.getenv("DB_USERNAME")
124   - db_pass = os.getenv("DB_PASSWORD")
  119 + db_cfg = get_app_config().infrastructure.database
  120 + db_host = db_cfg.host
  121 + db_port = db_cfg.port
  122 + db_name = db_cfg.database
  123 + db_user = db_cfg.username
  124 + db_pass = db_cfg.password
125 125 if not all([db_host, db_name, db_user, db_pass]):
126 126 print("ERROR: DB_HOST/DB_PORT/DB_DATABASE/DB_USERNAME/DB_PASSWORD must be set in environment")
127 127 return 1
... ... @@ -170,7 +170,7 @@ def main():
170 170 serve_parser = subparsers.add_parser('serve', help='Start API service (multi-tenant)')
171 171 serve_parser.add_argument('--host', default='0.0.0.0', help='Host to bind to')
172 172 serve_parser.add_argument('--port', type=int, default=6002, help='Port to bind to')
173   - serve_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host')
  173 + serve_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
174 174 serve_parser.add_argument('--reload', action='store_true', help='Enable auto-reload')
175 175  
176 176 # Serve-indexer command
... ... @@ -180,14 +180,14 @@ def main():
180 180 )
181 181 serve_indexer_parser.add_argument('--host', default='0.0.0.0', help='Host to bind to')
182 182 serve_indexer_parser.add_argument('--port', type=int, default=6004, help='Port to bind to')
183   - serve_indexer_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host')
  183 + serve_indexer_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
184 184 serve_indexer_parser.add_argument('--reload', action='store_true', help='Enable auto-reload')
185 185  
186 186 # Search command
187 187 search_parser = subparsers.add_parser('search', help='Test search from command line')
188 188 search_parser.add_argument('query', help='Search query')
189 189 search_parser.add_argument('--tenant-id', required=True, help='Tenant ID (required)')
190   - search_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host')
  190 + search_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
191 191 search_parser.add_argument('--size', type=int, default=10, help='Number of results')
192 192 search_parser.add_argument('--no-translation', action='store_true', help='Disable translation')
193 193 search_parser.add_argument('--no-embedding', action='store_true', help='Disable embeddings')
... ... @@ -199,7 +199,7 @@ def main():
199 199 help='Build tenant suggestion index (full/incremental)'
200 200 )
201 201 suggest_build_parser.add_argument('--tenant-id', required=True, help='Tenant ID')
202   - suggest_build_parser.add_argument('--es-host', default=ES_CONFIG.get('host', 'http://localhost:9200'), help='Elasticsearch host')
  202 + suggest_build_parser.add_argument('--es-host', default=get_app_config().infrastructure.elasticsearch.host, help='Elasticsearch host')
203 203 suggest_build_parser.add_argument(
204 204 '--mode',
205 205 choices=['full', 'incremental'],
... ...
query/query_parser.py
  Show/Hide comments   View file @86d8358
... ... @@ -336,13 +336,13 @@ class QueryParser:
336 336 translations = {}
337 337 translation_futures = {}
338 338 translation_executor = None
339   - index_langs = ["en", "zh"]
  339 + index_langs: List[str] = []
340 340 try:
341 341 # ๆ นๆฎ็งŸๆˆท้…็ฝฎ็š„ index_languages ๅ†ณๅฎš็ฟป่ฏ‘็›ฎๆ ‡่ฏญ่จ€
342 342 from config.tenant_config_loader import get_tenant_config_loader
343 343 tenant_loader = get_tenant_config_loader()
344 344 tenant_cfg = tenant_loader.get_tenant_config(tenant_id or "default")
345   - raw_index_langs = tenant_cfg.get("index_languages") or ["en", "zh"]
  345 + raw_index_langs = tenant_cfg.get("index_languages") or []
346 346 index_langs = []
347 347 seen_langs = set()
348 348 for lang in raw_index_langs:
... ...
reranker/backends/dashscope_rerank.py
  Show/Hide comments   View file @86d8358
... ... @@ -63,43 +63,19 @@ class DashScopeRerankBackend:
63 63 - max_retries: int, default 1
64 64 - retry_backoff_sec: float, default 0.2
65 65  
66   - Env overrides:
67   - - RERANK_DASHSCOPE_ENDPOINT
68   - - RERANK_DASHSCOPE_MODEL
69   - - RERANK_DASHSCOPE_TIMEOUT_SEC
70   - - RERANK_DASHSCOPE_TOP_N_CAP
71   - - RERANK_DASHSCOPE_BATCHSIZE
72 66 """
73 67  
74 68 def __init__(self, config: Dict[str, Any]) -> None:
75 69 self._config = config or {}
76   - self._model_name = str(
77   - os.getenv("RERANK_DASHSCOPE_MODEL")
78   - or self._config.get("model_name")
79   - or "qwen3-rerank"
80   - )
  70 + self._model_name = str(self._config.get("model_name") or "qwen3-rerank")
81 71 self._endpoint = str(
82   - os.getenv("RERANK_DASHSCOPE_ENDPOINT")
83   - or self._config.get("endpoint")
84   - or "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
  72 + self._config.get("endpoint") or "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
85 73 ).strip()
86 74 self._api_key_env = str(self._config.get("api_key_env") or "").strip()
87 75 self._api_key = str(os.getenv(self._api_key_env) or "").strip().strip('"').strip("'")
88   - self._timeout_sec = float(
89   - os.getenv("RERANK_DASHSCOPE_TIMEOUT_SEC")
90   - or self._config.get("timeout_sec")
91   - or 15.0
92   - )
93   - self._top_n_cap = int(
94   - os.getenv("RERANK_DASHSCOPE_TOP_N_CAP")
95   - or self._config.get("top_n_cap")
96   - or 0
97   - )
98   - self._batchsize = int(
99   - os.getenv("RERANK_DASHSCOPE_BATCHSIZE")
100   - or self._config.get("batchsize")
101   - or 0
102   - )
  76 + self._timeout_sec = float(self._config.get("timeout_sec") or 15.0)
  77 + self._top_n_cap = int(self._config.get("top_n_cap") or 0)
  78 + self._batchsize = int(self._config.get("batchsize") or 0)
103 79 self._instruct = str(self._config.get("instruct") or "").strip()
104 80 self._max_retries = int(self._config.get("max_retries", 1))
105 81 self._retry_backoff_sec = float(self._config.get("retry_backoff_sec", 0.2))
... ...
reranker/config.py
  Show/Hide comments   View file @86d8358
1   -"""Reranker service configuration (simple Python config)."""
  1 +"""Reranker service compatibility config derived from unified app config."""
2 2  
3   -import os
  3 +from __future__ import annotations
  4 +
  5 +from config.loader import get_app_config
4 6  
5 7  
6 8 class RerankerConfig(object):
7   - # Server
8   - HOST = os.getenv("RERANKER_HOST", "0.0.0.0")
9   - PORT = int(os.getenv("RERANKER_PORT", 6007))
10   -
11   - # Model
12   - MODEL_NAME = "Qwen/Qwen3-Reranker-0.6B"
13   - DEVICE = None # None -> auto (cuda if available)
14   - USE_FP16 = True
15   - BATCH_SIZE = 64
16   - MAX_LENGTH = 512
17   - CACHE_DIR = "./model_cache"
18   - ENABLE_WARMUP = True
19   -
20   - # Request limits
21   - MAX_DOCS = 1000
22   -
23   - # Output
24   - NORMALIZE = True
  9 + def __init__(self) -> None:
  10 + app_config = get_app_config()
  11 + runtime = app_config.runtime
  12 + service = app_config.services.rerank
  13 + backend = service.get_backend_config()
  14 + request = service.request
  15 +
  16 + self.HOST = runtime.reranker_host
  17 + self.PORT = runtime.reranker_port
  18 +
  19 + self.MODEL_NAME = str(backend.get("model_name") or "Qwen/Qwen3-Reranker-0.6B")
  20 + self.DEVICE = backend.get("device")
  21 + self.USE_FP16 = bool(backend.get("use_fp16", True))
  22 + self.BATCH_SIZE = int(backend.get("batch_size", backend.get("infer_batch_size", 64)))
  23 + self.MAX_LENGTH = int(backend.get("max_length", 512))
  24 + self.CACHE_DIR = str(backend.get("cache_dir") or "./model_cache")
  25 + self.ENABLE_WARMUP = bool(backend.get("enable_warmup", True))
  26 +
  27 + self.MAX_DOCS = int(request.get("max_docs", 1000))
  28 + self.NORMALIZE = bool(request.get("normalize", True))
25 29  
26 30  
27 31 CONFIG = RerankerConfig()
... ...
suggestion/builder.py
  Show/Hide comments   View file @86d8358
... ... @@ -18,7 +18,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple
18 18  
19 19 from sqlalchemy import text
20 20  
21   -from config.env_config import ES_INDEX_NAMESPACE
  21 +from config.loader import get_app_config
22 22 from config.tenant_config_loader import get_tenant_config_loader
23 23 from suggestion.mapping import build_suggestion_mapping
24 24 from utils.es_client import ESClient
... ... @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
27 27  
28 28  
29 29 def _index_prefix() -> str:
30   - return ES_INDEX_NAMESPACE or ""
  30 + return get_app_config().runtime.index_namespace or ""
31 31  
32 32  
33 33 def get_suggestion_alias_name(tenant_id: str) -> str:
... ...
translation/backends/deepl.py
  Show/Hide comments   View file @86d8358
... ... @@ -3,7 +3,6 @@
3 3 from __future__ import annotations
4 4  
5 5 import logging
6   -import os
7 6 import re
8 7 from typing import List, Optional, Sequence, Tuple, Union
9 8  
... ... @@ -24,7 +23,7 @@ class DeepLTranslationBackend:
24 23 timeout: float,
25 24 glossary_id: Optional[str] = None,
26 25 ) -> None:
27   - self.api_key = api_key or os.getenv("DEEPL_AUTH_KEY")
  26 + self.api_key = api_key
28 27 self.api_url = api_url
29 28 self.timeout = float(timeout)
30 29 self.glossary_id = glossary_id
... ...
translation/backends/llm.py
  Show/Hide comments   View file @86d8358
... ... @@ -3,13 +3,11 @@
3 3 from __future__ import annotations
4 4  
5 5 import logging
6   -import os
7 6 import time
8 7 from typing import List, Optional, Sequence, Union
9 8  
10 9 from openai import OpenAI
11 10  
12   -from config.env_config import DASHSCOPE_API_KEY
13 11 from translation.languages import LANGUAGE_LABELS
14 12 from translation.prompts import TRANSLATION_PROMPTS
15 13 from translation.scenes import normalize_scene_name
... ... @@ -52,11 +50,13 @@ class LLMTranslationBackend:
52 50 model: str,
53 51 timeout_sec: float,
54 52 base_url: str,
  53 + api_key: Optional[str],
55 54 ) -> None:
56 55 self.capability_name = capability_name
57 56 self.model = model
58 57 self.timeout_sec = float(timeout_sec)
59 58 self.base_url = base_url
  59 + self.api_key = api_key
60 60 self.client = self._create_client()
61 61  
62 62 @property
... ... @@ -64,12 +64,11 @@ class LLMTranslationBackend:
64 64 return True
65 65  
66 66 def _create_client(self) -> Optional[OpenAI]:
67   - api_key = DASHSCOPE_API_KEY or os.getenv("DASHSCOPE_API_KEY")
68   - if not api_key:
  67 + if not self.api_key:
69 68 logger.warning("DASHSCOPE_API_KEY not set; llm translation unavailable")
70 69 return None
71 70 try:
72   - return OpenAI(api_key=api_key, base_url=self.base_url)
  71 + return OpenAI(api_key=self.api_key, base_url=self.base_url)
73 72 except Exception as exc:
74 73 logger.error("Failed to initialize llm translation client: %s", exc, exc_info=True)
75 74 return None
... ...
translation/backends/qwen_mt.py
  Show/Hide comments   View file @86d8358
... ... @@ -3,14 +3,12 @@
3 3 from __future__ import annotations
4 4  
5 5 import logging
6   -import os
7 6 import re
8 7 import time
9 8 from typing import List, Optional, Sequence, Union
10 9  
11 10 from openai import OpenAI
12 11  
13   -from config.env_config import DASHSCOPE_API_KEY
14 12 from translation.languages import QWEN_LANGUAGE_CODES
15 13  
16 14 logger = logging.getLogger(__name__)
... ... @@ -64,7 +62,7 @@ class QwenMTTranslationBackend:
64 62 @staticmethod
65 63 def _default_api_key(model: str) -> Optional[str]:
66 64 del model
67   - return DASHSCOPE_API_KEY or os.getenv("DASHSCOPE_API_KEY")
  65 + return None
68 66  
69 67 def translate(
70 68 self,
... ...
translation/cache.py
  Show/Hide comments   View file @86d8358
... ... @@ -6,9 +6,12 @@ import hashlib
6 6 import logging
7 7 from typing import Mapping, Optional
8 8  
9   -import redis
  9 +try:
  10 + import redis
  11 +except ImportError: # pragma: no cover - runtime fallback for minimal envs
  12 + redis = None # type: ignore[assignment]
10 13  
11   -from config.env_config import REDIS_CONFIG
  14 +from config.loader import get_app_config
12 15  
13 16 logger = logging.getLogger(__name__)
14 17  
... ... @@ -70,15 +73,19 @@ class TranslationCache:
70 73  
71 74 @staticmethod
72 75 def _init_redis_client() -> Optional[redis.Redis]:
  76 + if redis is None:
  77 + logger.warning("redis package is not installed; translation cache disabled")
  78 + return None
  79 + redis_config = get_app_config().infrastructure.redis
73 80 try:
74 81 client = redis.Redis(
75   - host=REDIS_CONFIG.get("host", "localhost"),
76   - port=REDIS_CONFIG.get("port", 6479),
77   - password=REDIS_CONFIG.get("password"),
  82 + host=redis_config.host,
  83 + port=redis_config.port,
  84 + password=redis_config.password,
78 85 decode_responses=True,
79   - socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),
80   - socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),
81   - retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False),
  86 + socket_timeout=redis_config.socket_timeout,
  87 + socket_connect_timeout=redis_config.socket_connect_timeout,
  88 + retry_on_timeout=redis_config.retry_on_timeout,
82 89 health_check_interval=10,
83 90 )
84 91 client.ping()
... ...
translation/client.py
  Show/Hide comments   View file @86d8358
... ... @@ -7,7 +7,7 @@ from typing import List, Optional, Sequence, Union
7 7  
8 8 import requests
9 9  
10   -from config.services_config import get_translation_config
  10 +from config.loader import get_app_config
11 11 from translation.settings import normalize_translation_model, normalize_translation_scene
12 12  
13 13 logger = logging.getLogger(__name__)
... ... @@ -24,7 +24,7 @@ class TranslationServiceClient:
24 24 default_scene: Optional[str] = None,
25 25 timeout_sec: Optional[float] = None,
26 26 ) -> None:
27   - cfg = get_translation_config()
  27 + cfg = get_app_config().services.translation.as_dict()
28 28 self.base_url = str(base_url or cfg["service_url"]).rstrip("/")
29 29 self.default_model = normalize_translation_model(cfg, default_model or cfg["default_model"])
30 30 self.default_scene = normalize_translation_scene(cfg, default_scene or cfg["default_scene"])
... ...
translation/service.py
  Show/Hide comments   View file @86d8358
... ... @@ -5,7 +5,8 @@ from __future__ import annotations
5 5 import logging
6 6 from typing import Dict, List, Optional
7 7  
8   -from config.services_config import get_translation_config
  8 +from config.loader import get_app_config
  9 +from config.schema import AppConfig
9 10 from translation.cache import TranslationCache
10 11 from translation.protocols import TranslateInput, TranslateOutput, TranslationBackendProtocol
11 12 from translation.settings import (
... ... @@ -22,8 +23,9 @@ logger = logging.getLogger(__name__)
22 23 class TranslationService:
23 24 """Owns translation backends and routes calls by model and scene."""
24 25  
25   - def __init__(self, config: Optional[TranslationConfig] = None) -> None:
26   - self.config = config or get_translation_config()
  26 + def __init__(self, config: Optional[TranslationConfig] = None, app_config: Optional[AppConfig] = None) -> None:
  27 + self._app_config = app_config or get_app_config()
  28 + self.config = config or self._app_config.services.translation.as_dict()
27 29 self._enabled_capabilities = self._collect_enabled_capabilities()
28 30 if not self._enabled_capabilities:
29 31 raise ValueError("No enabled translation backends found in services.translation.capabilities")
... ... @@ -85,7 +87,7 @@ class TranslationService:
85 87 capability_name=name,
86 88 model=str(cfg["model"]).strip(),
87 89 base_url=str(cfg["base_url"]).strip(),
88   - api_key=cfg.get("api_key"),
  90 + api_key=self._app_config.infrastructure.secrets.dashscope_api_key,
89 91 timeout=int(cfg["timeout_sec"]),
90 92 glossary_id=cfg.get("glossary_id"),
91 93 )
... ... @@ -94,7 +96,7 @@ class TranslationService:
94 96 from translation.backends.deepl import DeepLTranslationBackend
95 97  
96 98 return DeepLTranslationBackend(
97   - api_key=cfg.get("api_key"),
  99 + api_key=self._app_config.infrastructure.secrets.deepl_auth_key,
98 100 api_url=str(cfg["api_url"]).strip(),
99 101 timeout=float(cfg["timeout_sec"]),
100 102 glossary_id=cfg.get("glossary_id"),
... ... @@ -108,6 +110,7 @@ class TranslationService:
108 110 model=str(cfg["model"]).strip(),
109 111 timeout_sec=float(cfg["timeout_sec"]),
110 112 base_url=str(cfg["base_url"]).strip(),
  113 + api_key=self._app_config.infrastructure.secrets.dashscope_api_key,
111 114 )
112 115  
113 116 def _create_local_nllb_backend(self, *, name: str, cfg: Dict[str, object]) -> TranslationBackendProtocol:
... ...
utils/es_client.py
  Show/Hide comments   View file @86d8358
... ... @@ -5,10 +5,9 @@ Elasticsearch client wrapper.
5 5 from elasticsearch import Elasticsearch
6 6 from elasticsearch.helpers import bulk
7 7 from typing import Dict, Any, List, Optional
8   -import os
9 8 import logging
10 9  
11   -from config.env_config import ES_CONFIG
  10 +from config.loader import get_app_config
12 11  
13 12 logger = logging.getLogger(__name__)
14 13  
... ... @@ -33,7 +32,7 @@ class ESClient:
33 32 **kwargs: Additional ES client parameters
34 33 """
35 34 if hosts is None:
36   - hosts = [os.getenv('ES_HOST', 'http://localhost:9200')]
  35 + hosts = [get_app_config().infrastructure.elasticsearch.host]
37 36  
38 37 # Build client config
39 38 client_config = {
... ... @@ -325,16 +324,9 @@ def get_es_client_from_env() -&gt; ESClient:
325 324 Returns:
326 325 ESClient instance
327 326 """
328   - if ES_CONFIG:
329   - return ESClient(
330   - hosts=[ES_CONFIG['host']],
331   - username=ES_CONFIG.get('username'),
332   - password=ES_CONFIG.get('password')
333   - )
334   - else:
335   - # Fallback to env variables
336   - return ESClient(
337   - hosts=[os.getenv('ES_HOST', 'http://localhost:9200')],
338   - username=os.getenv('ES_USERNAME'),
339   - password=os.getenv('ES_PASSWORD')
340   - )
  327 + cfg = get_app_config().infrastructure.elasticsearch
  328 + return ESClient(
  329 + hosts=[cfg.host],
  330 + username=cfg.username,
  331 + password=cfg.password,
  332 + )
... ...