ai-saas / saas-search

Commit 5e4dc8e4d3206a9bd3326c0b202b4f972feb5ee7

Authored by tangwang
1 parent 4a37d233

翻译架构按“一个翻译服务 + 

多个独立翻译能力”重构。现在业务侧不再把翻译当 provider
选型，QueryParser 和 indexer 统一通过 6006 的 translator service client
调用；真正的能力选择、启用开关、model + scene 路由，都收口到服务端和新的
translation/ 目录里了。

这次的核心改动在
config/services_config.py、providers/translation.py、api/translator_app.py、config/config.yaml
和新的 translation/service.py。配置从旧的
services.translation.provider/providers 改成了 service_url +
default_model + default_scene + capabilities，每个能力可独立
enabled；服务端新增了统一的 backend 管理与懒加载，真实实现集中到
translation/backends/qwen_mt.py、translation/backends/llm.py、translation/backends/deepl.py，旧的
query/qwen_mt_translate.py、query/llm_translate.py、query/deepl_provider.py
只保留兼容导出。接口上，/translate 现在标准支持 scene，context
作为兼容别名继续可用，健康检查会返回默认模型、默认场景和已启用能力。
Showing 20 changed files with 1208 additions and 1118 deletions   Show diff stats
Inline Side-by-side
api/translator_app.py
  Show/Hide comments   View file @5e4dc8e
... ... @@ -83,24 +83,17 @@ Start the service:
83 83 uvicorn api.translator_app:app --host 0.0.0.0 --port 6006 --reload
84 84 """
85 85  
86   -import os
87   -import sys
88 86 import logging
89 87 import argparse
90 88 import uvicorn
91   -from typing import Dict, List, Optional, Sequence, Union
  89 +from typing import Dict, List, Optional, Union
92 90 from fastapi import FastAPI, HTTPException
93 91 from fastapi.responses import JSONResponse
94 92 from fastapi.middleware.cors import CORSMiddleware
95 93 from pydantic import BaseModel, Field
96 94  
97   -# Add parent directory to path
98   -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
99   -
100   -from query.qwen_mt_translate import Translator
101   -from query.llm_translate import LLMTranslatorProvider
102   -from query.deepl_provider import DeepLProvider
103 95 from config.services_config import get_translation_config
  96 +from translation.service import TranslationService
104 97  
105 98 # Configure logging
106 99 logging.basicConfig(
... ... @@ -109,54 +102,14 @@ logging.basicConfig(
109 102 )
110 103 logger = logging.getLogger(__name__)
111 104  
112   -# Global translator instances cache (keyed by model)
113   -_translators: Dict[str, object] = {}
114   -
  105 +_translation_service: Optional[TranslationService] = None
115 106  
116   -def _resolve_default_model() -> str:
117   - """
118   - Resolve translator model from services.translation config first.
119 107  
120   - Priority:
121   - 1) TRANSLATION_MODEL env (explicit runtime override)
122   - 2) services.translation.provider + providers.<provider>.model
123   - 3) qwen-mt
124   - """
125   - env_model = (os.getenv("TRANSLATION_MODEL") or "").strip()
126   - if env_model:
127   - return env_model
128   - try:
129   - cfg = get_translation_config()
130   - provider = (cfg.provider or "").strip().lower()
131   - provider_cfg = cfg.get_provider_cfg() if hasattr(cfg, "get_provider_cfg") else {}
132   - model = (provider_cfg.get("model") or "").strip().lower() if isinstance(provider_cfg, dict) else ""
133   - if provider == "llm":
134   - return "llm"
135   - if provider in {"qwen-mt", "direct", "http"}:
136   - return model or "qwen-mt"
137   - if provider == "deepl":
138   - return "deepl"
139   - except Exception:
140   - pass
141   - return "qwen-mt"
142   -
143   -
144   -def get_translator(model: str = "qwen") -> object:
145   - """Get or create translator instance for the specified model."""
146   - global _translators
147   - if model not in _translators:
148   - logger.info(f"Initializing translator with model: {model}...")
149   - normalized = (model or "qwen").strip().lower()
150   - if normalized in {"qwen", "qwen-mt", "qwen-mt-flash", "qwen-mt-flush"}:
151   - _translators[model] = Translator(model=normalized, use_cache=True, timeout=10)
152   - elif normalized == "deepl":
153   - _translators[model] = DeepLProvider(api_key=None, timeout=10.0)
154   - elif normalized == "llm":
155   - _translators[model] = LLMTranslatorProvider()
156   - else:
157   - raise ValueError(f"Unsupported model: {model}")
158   - logger.info(f"Translator initialized with model: {model}")
159   - return _translators[model]
  108 +def get_translation_service() -> TranslationService:
  109 + global _translation_service
  110 + if _translation_service is None:
  111 + _translation_service = TranslationService(get_translation_config())
  112 + return _translation_service
160 113  
161 114  
162 115 # Request/Response models
... ... @@ -166,7 +119,8 @@ class TranslationRequest(BaseModel):
166 119 target_lang: str = Field(..., description="Target language code (zh, en, ru, etc.)")
167 120 source_lang: Optional[str] = Field(None, description="Source language code (optional, auto-detect if not provided)")
168 121 model: Optional[str] = Field(None, description="Translation model: qwen-mt | deepl | llm")
169   - context: Optional[str] = Field(None, description="Optional translation scene or context")
  122 + scene: Optional[str] = Field(None, description="Translation scene, paired with model routing")
  123 + context: Optional[str] = Field(None, description="Deprecated alias of scene")
170 124 prompt: Optional[str] = Field(None, description="Optional prompt override")
171 125  
172 126 class Config:
... ... @@ -176,7 +130,7 @@ class TranslationRequest(BaseModel):
176 130 "target_lang": "en",
177 131 "source_lang": "zh",
178 132 "model": "llm",
179   - "context": "sku_name"
  133 + "scene": "sku_name"
180 134 }
181 135 }
182 136  
... ... @@ -192,6 +146,7 @@ class TranslationResponse(BaseModel):
192 146 )
193 147 status: str = Field(..., description="Translation status")
194 148 model: str = Field(..., description="Translation model used")
  149 + scene: str = Field(..., description="Translation scene used")
195 150  
196 151  
197 152 # Create FastAPI app
... ... @@ -217,10 +172,13 @@ app.add_middleware(
217 172 async def startup_event():
218 173 """Initialize translator on startup."""
219 174 logger.info("Starting Translation Service API on port 6006")
220   - default_model = _resolve_default_model()
221 175 try:
222   - get_translator(model=default_model)
223   - logger.info(f"Translation service ready with default model: {default_model}")
  176 + service = get_translation_service()
  177 + logger.info(
  178 + "Translation service ready | default_model=%s available_models=%s",
  179 + service.config.default_model,
  180 + service.available_models,
  181 + )
224 182 except Exception as e:
225 183 logger.error(f"Failed to initialize translator: {e}", exc_info=True)
226 184 raise
... ... @@ -230,17 +188,14 @@ async def startup_event():
230 188 async def health_check():
231 189 """Health check endpoint."""
232 190 try:
233   - # 仅做轻量级本地检查，避免在健康检查中触发潜在的阻塞初始化或外部依赖
234   - default_model = _resolve_default_model()
235   - # 如果启动事件成功，默认模型通常会已经初始化到缓存中
236   - translator = _translators.get(default_model) or next(iter(_translators.values()), None)
  191 + service = get_translation_service()
237 192 return {
238 193 "status": "healthy",
239 194 "service": "translation",
240   - "default_model": default_model,
241   - "available_models": list(_translators.keys()),
242   - "translator_initialized": translator is not None,
243   - "cache_enabled": bool(getattr(translator, "use_cache", False))
  195 + "default_model": service.config.default_model,
  196 + "default_scene": service.config.default_scene,
  197 + "available_models": service.available_models,
  198 + "enabled_capabilities": service.config.enabled_models,
244 199 }
245 200 except Exception as e:
246 201 logger.error(f"Health check failed: {e}")
... ... @@ -283,27 +238,22 @@ async def translate(request: TranslationRequest):
283 238 detail="target_lang is required"
284 239 )
285 240  
286   - # Validate model parameter
287   - model = request.model.lower() if request.model else _resolve_default_model().lower()
288   - if model not in ["qwen", "qwen-mt", "deepl", "llm"]:
289   - raise HTTPException(
290   - status_code=400,
291   - detail="Invalid model. Supported models: 'qwen-mt', 'deepl', 'llm'"
292   - )
293   -
294 241 try:
295   - # Get translator instance for the specified model
296   - translator = get_translator(model=model)
  242 + service = get_translation_service()
  243 + scene = (request.scene or request.context or service.config.default_scene).strip() or "general"
  244 + model = service.config.normalize_model_name(request.model or service.config.default_model)
  245 + translator = service.get_backend(model)
297 246 raw_text = request.text
298 247  
299 248 # 如果是列表，并且底层 provider 声明支持 batch，则直接传 list
300 249 if isinstance(raw_text, list) and getattr(translator, "supports_batch", False):
301 250 try:
302   - translated_list = translator.translate(
  251 + translated_list = service.translate(
303 252 text=raw_text,
304 253 target_lang=request.target_lang,
305 254 source_lang=request.source_lang,
306   - context=request.context,
  255 + model=model,
  256 + scene=scene,
307 257 prompt=request.prompt,
308 258 )
309 259 except Exception as exc:
... ... @@ -334,6 +284,7 @@ async def translate(request: TranslationRequest):
334 284 translated_text=normalized,
335 285 status="success",
336 286 model=str(getattr(translator, "model", model)),
  287 + scene=scene,
337 288 )
338 289  
339 290 # 否则：统一走逐条拆分逻辑（包括不支持 batch 的 provider）
... ... @@ -345,11 +296,12 @@ async def translate(request: TranslationRequest):
345 296 results.append(item) # type: ignore[arg-type]
346 297 continue
347 298 try:
348   - out = translator.translate(
  299 + out = service.translate(
349 300 text=str(item),
350 301 target_lang=request.target_lang,
351 302 source_lang=request.source_lang,
352   - context=request.context,
  303 + model=model,
  304 + scene=scene,
353 305 prompt=request.prompt,
354 306 )
355 307 except Exception as exc:
... ... @@ -365,14 +317,16 @@ async def translate(request: TranslationRequest):
365 317 translated_text=results,
366 318 status="success",
367 319 model=str(getattr(translator, "model", model)),
  320 + scene=scene,
368 321 )
369 322  
370 323 # 单文本模式：保持原有严格失败语义
371   - translated_text = translator.translate(
  324 + translated_text = service.translate(
372 325 text=raw_text,
373 326 target_lang=request.target_lang,
374 327 source_lang=request.source_lang,
375   - context=request.context,
  328 + model=model,
  329 + scene=scene,
376 330 prompt=request.prompt,
377 331 )
378 332  
... ... @@ -388,7 +342,8 @@ async def translate(request: TranslationRequest):
388 342 source_lang=request.source_lang,
389 343 translated_text=translated_text,
390 344 status="success",
391   - model=str(getattr(translator, "model", model))
  345 + model=str(getattr(translator, "model", model)),
  346 + scene=scene,
392 347 )
393 348  
394 349 except HTTPException:
... ...
config/config.yaml
  Show/Hide comments   View file @5e4dc8e
... ... @@ -107,9 +107,9 @@ rerank:
107 107 # 可扩展服务/provider 注册表（单一配置源）
108 108 services:
109 109 translation:
110   - provider: "llm" # qwen-mt | deepl | http | llm
111   - base_url: "http://127.0.0.1:6006"
112   - model: "qwen-flash"
  110 + service_url: "http://127.0.0.1:6006"
  111 + default_model: "llm"
  112 + default_scene: "general"
113 113 timeout_sec: 10.0
114 114 cache:
115 115 enabled: true
... ... @@ -119,20 +119,21 @@ services:
119 119 key_include_context: true
120 120 key_include_prompt: true
121 121 key_include_source_lang: true
122   - providers:
  122 + capabilities:
123 123 qwen-mt:
124   - model: "qwen-mt-flush"
125   - http:
126   - base_url: "http://127.0.0.1:6006"
127   - model: "qwen-mt-flush"
  124 + enabled: true
  125 + model: "qwen-mt-flash"
128 126 timeout_sec: 10.0
  127 + use_cache: true
129 128 llm:
  129 + enabled: true
130 130 model: "qwen-flash"
131 131 # 可选：覆盖 DashScope 兼容模式的 Endpoint 与超时
132 132 # base_url 留空则使用 DASHSCOPE_BASE_URL 或默认地域
133 133 base_url: ""
134 134 timeout_sec: 30.0
135 135 deepl:
  136 + enabled: false
136 137 model: "deepl"
137 138 timeout_sec: 10.0
138 139 # 可选：用于术语表翻译（由 query_config.translation_glossary_id 衔接）
... ...
config/env_config.py
  Show/Hide comments   View file @5e4dc8e
... ... @@ -65,8 +65,9 @@ EMBEDDING_HOST = os.getenv(&#39;EMBEDDING_HOST&#39;, &#39;127.0.0.1&#39;)
65 65 EMBEDDING_PORT = int(os.getenv('EMBEDDING_PORT', 6005))
66 66 TRANSLATION_HOST = os.getenv('TRANSLATION_HOST', '127.0.0.1')
67 67 TRANSLATION_PORT = int(os.getenv('TRANSLATION_PORT', 6006))
68   -TRANSLATION_PROVIDER = os.getenv('TRANSLATION_PROVIDER', 'direct')
69   -TRANSLATION_MODEL = os.getenv('TRANSLATION_MODEL', 'qwen')
  68 +TRANSLATION_PROVIDER = os.getenv('TRANSLATION_PROVIDER', 'direct') # deprecated
  69 +TRANSLATION_MODEL = os.getenv('TRANSLATION_MODEL', 'llm')
  70 +TRANSLATION_SCENE = os.getenv('TRANSLATION_SCENE', 'general')
70 71 RERANKER_HOST = os.getenv('RERANKER_HOST', '127.0.0.1')
71 72 RERANKER_PORT = int(os.getenv('RERANKER_PORT', 6007))
72 73 RERANK_PROVIDER = os.getenv('RERANK_PROVIDER', 'http')
... ...
config/services_config.py
  Show/Hide comments   View file @5e4dc8e
1 1 """
2   -Services configuration - single source for translation, embedding, rerank providers.
  2 +Services configuration - single source for translation, embedding, rerank.
3 3  
4   -All provider selection and endpoint resolution is centralized here.
5   -Priority: env vars > config.yaml.
  4 +Translation is modeled as:
  5 +- one translator service endpoint used by business callers
  6 +- multiple translation capabilities loaded inside the translator service
6 7 """
7 8  
8 9 from __future__ import annotations
... ... @@ -11,25 +12,60 @@ import os
11 12 from dataclasses import dataclass, field
12 13 from functools import lru_cache
13 14 from pathlib import Path
14   -from typing import Any, Dict, Optional
  15 +from typing import Any, Dict, List, Optional
15 16  
16 17 import yaml
17 18  
18 19  
19 20 @dataclass
20 21 class ServiceConfig:
21   - """Config for one capability (translation/embedding/rerank)."""
  22 + """Config for one capability (embedding/rerank)."""
  23 +
22 24 provider: str
23 25 providers: Dict[str, Any] = field(default_factory=dict)
24 26  
25 27 def get_provider_cfg(self) -> Dict[str, Any]:
26   - """Get config for current provider."""
27 28 p = (self.provider or "").strip().lower()
28 29 return self.providers.get(p, {}) if isinstance(self.providers, dict) else {}
29 30  
30 31  
  32 +@dataclass
  33 +class TranslationServiceConfig:
  34 + """Dedicated config model for the translation service."""
  35 +
  36 + service_url: str
  37 + timeout_sec: float
  38 + default_model: str
  39 + default_scene: str
  40 + capabilities: Dict[str, Dict[str, Any]] = field(default_factory=dict)
  41 + cache: Dict[str, Any] = field(default_factory=dict)
  42 +
  43 + def normalize_model_name(self, model: Optional[str]) -> str:
  44 + normalized = str(model or self.default_model).strip().lower()
  45 + aliases = {
  46 + "qwen": "qwen-mt",
  47 + "qwen-mt-flash": "qwen-mt",
  48 + "qwen-mt-flush": "qwen-mt",
  49 + "service": self.default_model,
  50 + "default": self.default_model,
  51 + }
  52 + return aliases.get(normalized, normalized)
  53 +
  54 + @property
  55 + def enabled_models(self) -> List[str]:
  56 + items: List[str] = []
  57 + for name, cfg in self.capabilities.items():
  58 + if isinstance(cfg, dict) and bool(cfg.get("enabled", False)):
  59 + items.append(str(name).strip().lower())
  60 + return items
  61 +
  62 + def get_capability_cfg(self, model: Optional[str]) -> Dict[str, Any]:
  63 + normalized = self.normalize_model_name(model)
  64 + value = self.capabilities.get(normalized)
  65 + return dict(value) if isinstance(value, dict) else {}
  66 +
  67 +
31 68 def _load_services_raw(config_path: Optional[Path] = None) -> Dict[str, Any]:
32   - """Load services block from config.yaml."""
33 69 if config_path is None:
34 70 config_path = Path(__file__).parent / "config.yaml"
35 71 path = Path(config_path)
... ... @@ -48,11 +84,7 @@ def _load_services_raw(config_path: Optional[Path] = None) -&gt; Dict[str, Any]:
48 84 return services
49 85  
50 86  
51   -def _resolve_provider_name(
52   - env_name: str,
53   - config_provider: Any,
54   - capability: str,
55   -) -> str:
  87 +def _resolve_provider_name(env_name: str, config_provider: Any, capability: str) -> str:
56 88 provider = os.getenv(env_name) or config_provider
57 89 if not provider:
58 90 raise ValueError(
... ... @@ -62,27 +94,70 @@ def _resolve_provider_name(
62 94 return str(provider).strip().lower()
63 95  
64 96  
65   -def _resolve_translation() -> ServiceConfig:
  97 +def _resolve_translation() -> TranslationServiceConfig:
66 98 raw = _load_services_raw()
67 99 cfg = raw.get("translation", {}) if isinstance(raw.get("translation"), dict) else {}
68   - providers = cfg.get("providers", {}) if isinstance(cfg.get("providers"), dict) else {}
69 100  
70   - provider = _resolve_provider_name(
71   - env_name="TRANSLATION_PROVIDER",
72   - config_provider=cfg.get("provider"),
73   - capability="translation",
  101 + service_url = (
  102 + os.getenv("TRANSLATION_SERVICE_URL")
  103 + or cfg.get("service_url")
  104 + or cfg.get("base_url")
  105 + or "http://127.0.0.1:6006"
74 106 )
75   - if provider not in ("qwen-mt", "deepl", "direct", "local", "inprocess", "http", "service", "llm"):
76   - raise ValueError(f"Unsupported translation provider: {provider}")
  107 + timeout_sec = float(os.getenv("TRANSLATION_TIMEOUT_SEC") or cfg.get("timeout_sec") or 10.0)
  108 +
  109 + raw_capabilities = cfg.get("capabilities")
  110 + if not isinstance(raw_capabilities, dict):
  111 + raw_capabilities = cfg.get("providers")
  112 + capabilities = raw_capabilities if isinstance(raw_capabilities, dict) else {}
  113 +
  114 + default_model = str(
  115 + os.getenv("TRANSLATION_MODEL")
  116 + or cfg.get("default_model")
  117 + or cfg.get("provider")
  118 + or "qwen-mt"
  119 + ).strip().lower()
  120 + default_scene = str(
  121 + os.getenv("TRANSLATION_SCENE")
  122 + or cfg.get("default_scene")
  123 + or "general"
  124 + ).strip() or "general"
  125 +
  126 + resolved_capabilities: Dict[str, Dict[str, Any]] = {}
  127 + for name, value in capabilities.items():
  128 + if not isinstance(value, dict):
  129 + continue
  130 + normalized = str(name or "").strip().lower()
  131 + if not normalized:
  132 + continue
  133 + copied = dict(value)
  134 + copied.setdefault("enabled", normalized == default_model)
  135 + resolved_capabilities[normalized] = copied
  136 +
  137 + aliases = {
  138 + "qwen": "qwen-mt",
  139 + "qwen-mt-flash": "qwen-mt",
  140 + "qwen-mt-flush": "qwen-mt",
  141 + }
  142 + default_model = aliases.get(default_model, default_model)
77 143  
78   - # Env override for http base_url
79   - env_url = os.getenv("TRANSLATION_SERVICE_URL")
80   - if env_url and provider in ("http", "service"):
81   - providers = dict(providers)
82   - providers["http"] = dict(providers.get("http", {}))
83   - providers["http"]["base_url"] = env_url.rstrip("/")
  144 + if default_model not in resolved_capabilities:
  145 + raise ValueError(
  146 + f"services.translation.default_model '{default_model}' is not defined in capabilities"
  147 + )
  148 + if not bool(resolved_capabilities[default_model].get("enabled", False)):
  149 + resolved_capabilities[default_model]["enabled"] = True
84 150  
85   - return ServiceConfig(provider=provider, providers=providers)
  151 + cache_cfg = cfg.get("cache", {}) if isinstance(cfg.get("cache"), dict) else {}
  152 +
  153 + return TranslationServiceConfig(
  154 + service_url=str(service_url).rstrip("/"),
  155 + timeout_sec=timeout_sec,
  156 + default_model=default_model,
  157 + default_scene=default_scene,
  158 + capabilities=resolved_capabilities,
  159 + cache=cache_cfg,
  160 + )
86 161  
87 162  
88 163 def _resolve_embedding() -> ServiceConfig:
... ... @@ -134,18 +209,10 @@ def _resolve_rerank() -&gt; ServiceConfig:
134 209  
135 210  
136 211 def get_rerank_backend_config() -> tuple[str, dict]:
137   - """
138   - Resolve reranker backend name and config for the reranker service process.
139   - Returns (backend_name, backend_cfg).
140   - Env RERANK_BACKEND overrides config.
141   - """
142 212 raw = _load_services_raw()
143 213 cfg = raw.get("rerank", {}) if isinstance(raw.get("rerank"), dict) else {}
144 214 backends = cfg.get("backends", {}) if isinstance(cfg.get("backends"), dict) else {}
145   - name = (
146   - os.getenv("RERANK_BACKEND")
147   - or cfg.get("backend")
148   - )
  215 + name = os.getenv("RERANK_BACKEND") or cfg.get("backend")
149 216 if not name:
150 217 raise ValueError("services.rerank.backend is required (or env RERANK_BACKEND)")
151 218 name = str(name).strip().lower()
... ... @@ -156,18 +223,10 @@ def get_rerank_backend_config() -&gt; tuple[str, dict]:
156 223  
157 224  
158 225 def get_embedding_backend_config() -> tuple[str, dict]:
159   - """
160   - Resolve embedding backend name and config for the embedding service process.
161   - Returns (backend_name, backend_cfg).
162   - Env EMBEDDING_BACKEND overrides config.
163   - """
164 226 raw = _load_services_raw()
165 227 cfg = raw.get("embedding", {}) if isinstance(raw.get("embedding"), dict) else {}
166 228 backends = cfg.get("backends", {}) if isinstance(cfg.get("backends"), dict) else {}
167   - name = (
168   - os.getenv("EMBEDDING_BACKEND")
169   - or cfg.get("backend")
170   - )
  229 + name = os.getenv("EMBEDDING_BACKEND") or cfg.get("backend")
171 230 if not name:
172 231 raise ValueError("services.embedding.backend is required (or env EMBEDDING_BACKEND)")
173 232 name = str(name).strip().lower()
... ... @@ -178,44 +237,26 @@ def get_embedding_backend_config() -&gt; tuple[str, dict]:
178 237  
179 238  
180 239 @lru_cache(maxsize=1)
181   -def get_translation_config() -> ServiceConfig:
182   - """Get translation service config."""
  240 +def get_translation_config() -> TranslationServiceConfig:
183 241 return _resolve_translation()
184 242  
185 243  
186 244 @lru_cache(maxsize=1)
187 245 def get_embedding_config() -> ServiceConfig:
188   - """Get embedding service config."""
189 246 return _resolve_embedding()
190 247  
191 248  
192 249 @lru_cache(maxsize=1)
193 250 def get_rerank_config() -> ServiceConfig:
194   - """Get rerank service config."""
195 251 return _resolve_rerank()
196 252  
197 253  
198 254 def get_translation_base_url() -> str:
199   - """Resolve translation HTTP base URL (for http provider)."""
200   - base = (
201   - os.getenv("TRANSLATION_SERVICE_URL")
202   - or get_translation_config().providers.get("http", {}).get("base_url")
203   - )
204   - if not base:
205   - raise ValueError("Translation HTTP base_url is not configured")
206   - return str(base).rstrip("/")
  255 + return get_translation_config().service_url
207 256  
208 257  
209 258 def get_translation_cache_config() -> Dict[str, Any]:
210   - """
211   - Resolve translation cache policy from services.translation.cache.
212   -
213   - All translation cache key/TTL behavior should be configured in config.yaml,
214   - not hardcoded in code.
215   - """
216   - raw = _load_services_raw()
217   - cfg = raw.get("translation", {}) if isinstance(raw.get("translation"), dict) else {}
218   - cache_cfg = cfg.get("cache", {}) if isinstance(cfg.get("cache"), dict) else {}
  259 + cache_cfg = get_translation_config().cache
219 260 return {
220 261 "enabled": bool(cache_cfg.get("enabled", True)),
221 262 "key_prefix": str(cache_cfg.get("key_prefix", "trans:v2")),
... ... @@ -228,31 +269,23 @@ def get_translation_cache_config() -&gt; Dict[str, Any]:
228 269  
229 270  
230 271 def get_embedding_base_url() -> str:
231   - """Resolve embedding HTTP base URL."""
232   - base = (
233   - os.getenv("EMBEDDING_SERVICE_URL")
234   - or get_embedding_config().providers.get("http", {}).get("base_url")
235   - )
  272 + base = os.getenv("EMBEDDING_SERVICE_URL") or get_embedding_config().providers.get("http", {}).get("base_url")
236 273 if not base:
237 274 raise ValueError("Embedding HTTP base_url is not configured")
238 275 return str(base).rstrip("/")
239 276  
240 277  
241   -def get_rerank_service_url() -> str:
242   - """Resolve rerank service URL (full path including /rerank)."""
  278 +def get_rerank_base_url() -> str:
243 279 base = (
244 280 os.getenv("RERANKER_SERVICE_URL")
245 281 or get_rerank_config().providers.get("http", {}).get("service_url")
246 282 or get_rerank_config().providers.get("http", {}).get("base_url")
247 283 )
248 284 if not base:
249   - raise ValueError("Rerank HTTP service_url/base_url is not configured")
250   - base = str(base).rstrip("/")
251   - return base if base.endswith("/rerank") else f"{base}/rerank"
  285 + raise ValueError("Rerank HTTP base_url is not configured")
  286 + return str(base).rstrip("/")
252 287  
253 288  
254   -def clear_services_cache() -> None:
255   - """Clear cached config (for tests)."""
256   - get_translation_config.cache_clear()
257   - get_embedding_config.cache_clear()
258   - get_rerank_config.cache_clear()
  289 +def get_rerank_service_url() -> str:
  290 + """Backward-compatible alias."""
  291 + return get_rerank_base_url()
... ...
docs/DEVELOPER_GUIDE.md
  Show/Hide comments   View file @5e4dc8e
... ... @@ -166,14 +166,14 @@ docs/ # 文档（含本指南）
166 166  
167 167 ### 4.8 providers
168 168  
169   -- **职责**：统一“能力”的调用方式：翻译、向量、重排均通过工厂函数（如 `create_translation_provider()`、`create_rerank_provider()`、`create_embedding_provider()`）获取实现，配置来自 `config/services_config`（即 `config.yaml` 的 `services` + 环境变量）。
170   -- **原则**：业务代码只依赖 Provider 接口，不依赖具体 URL 或后端类型；新增调用方式（如新 Provider 类型）在对应 `providers/<capability>.py` 中实现并在工厂中注册。
  169 +- **职责**：统一“能力”的调用方式。向量、重排仍是标准 provider 工厂；翻译侧的 `create_translation_provider()` 现在固定返回 translator service client，由 6006 服务统一承接后端选择与路由。
  170 +- **原则**：业务代码只依赖调用接口，不依赖具体 URL 或服务内后端类型；翻译能力新增时优先扩展 `translation/backends/` 与 `services.translation.capabilities`，而不是在业务侧新增 provider 分支。
171 171 - **详见**：本指南 §7.2；[QUICKSTART.md](./QUICKSTART.md) §3。
172 172  
173   -补充约定（翻译 provider）：
  173 +补充约定（翻译 client）：
174 174  
175 175 - `translate(text=...)` 支持 `str` 与 `List[str]` 两种输入；当输入为列表时，输出必须与输入 **等长且顺序对应**，失败位置为 `None`（HTTP JSON 表现为 `null`）。
176   -- provider 可暴露 `supports_batch: bool`（property）用于标识其是否支持直接批量调用；上层在处理 `text` 为列表时可优先走 batch，否则逐条拆分调用。
  176 +- client / backend 可暴露 `supports_batch: bool`（property）用于标识其是否支持直接批量调用；上层在处理 `text` 为列表时可优先走 batch，否则逐条拆分调用。
177 177  
178 178 ### 4.9 suggestion
179 179  
... ... @@ -197,18 +197,19 @@ docs/ # 文档（含本指南）
197 197 ### 5.2 配置驱动
198 198  
199 199 - 搜索行为（字段权重、搜索域、排序、function_score、重排融合参数等）来自 `config/config.yaml`，由 `ConfigLoader` 加载。
200   -- 能力访问（翻译/向量/重排的 provider、URL、后端类型）来自 `config.yaml` 的 `services` 块及环境变量，由 `config/services_config` 解析。
  200 +- 能力访问来自 `config.yaml` 的 `services` 块及环境变量，由 `config/services_config` 解析。
  201 +- 其中翻译单独采用“service + capabilities”模型：调用方只配 `service_url` / `default_model` / `default_scene`，服务内通过 `capabilities` 控制启用哪些翻译能力。
201 202 - 新增开关或参数时，优先在现有 config 结构下扩展，避免新增散落配置文件。
202 203  
203 204 ### 5.3 单一配置源与优先级
204 205  
205 206 - 同一类配置只在一个地方定义默认值；覆盖顺序约定为：**环境变量 > config 文件**。
206   -- 服务 URL、后端类型等均在 `services.<capability>` 下配置；环境变量用于部署态覆盖（如 `RERANKER_SERVICE_URL`、`RERANK_BACKEND`）。
  207 +- 服务 URL、后端类型等均在 `services.<capability>` 下配置；环境变量用于部署态覆盖（如 `TRANSLATION_SERVICE_URL`、`TRANSLATION_MODEL`、`RERANKER_SERVICE_URL`、`RERANK_BACKEND`）。
207 208  
208   -### 5.4 调用方与实现解耦（Provider + Backend）
  209 +### 5.4 调用方与实现解耦（Client + Backend）
209 210  
210   -- **调用方**：通过 Provider（如 `HttpRerankProvider`）访问能力，不依赖具体 URL 或服务内实现。
211   -- **服务内**：通过“后端”实现具体推理（如 BGE 与 Qwen3-vLLM）；后端实现协议、在配置与工厂中注册即可插拔。
  211 +- **调用方**：通过 client/provider 访问能力，不依赖具体 URL 或服务内实现；翻译调用方统一连 translator service。
  212 +- **服务内**：通过“后端”实现具体推理（如 qwen-mt、DeepL、LLM、本地模型；或 BGE 与 Qwen3-vLLM）；后端实现协议、在配置与工厂中注册即可插拔。
212 213 - 新增“一种调用方式”在 providers 中扩展；新增“一种推理实现”在对应服务的 backends 中扩展，并遵循本指南 §7。
213 214  
214 215 ### 5.5 协议契约
... ... @@ -246,7 +247,7 @@ docs/ # 文档（含本指南）
246 247  
247 248 ### 6.1 主配置文件
248 249  
249   -- **config/config.yaml**：搜索行为（field_boosts、query_config.search_fields、query_config.text_query_strategy，含翻译失败时的原文兜底 boost、ranking、function_score、rerank 融合参数）、SPU 配置、**services**（翻译/向量/重排的 provider 与 backends）、tenant_config 等。
  250 +- **config/config.yaml**：搜索行为（field_boosts、query_config.search_fields、query_config.text_query_strategy，含翻译失败时的原文兜底 boost、ranking、function_score、rerank 融合参数）、SPU 配置、**services**（翻译 service 与 capabilities、向量/重排的 provider 与 backends）、tenant_config 等。
250 251 - **.env**：敏感信息与部署态变量（DB、ES、Redis、API Key、端口等）；不提交敏感值，可提供 `.env.example` 模板。
251 252  
252 253 ### 6.2 services 块结构（能力统一约定）
... ... @@ -265,14 +266,31 @@ services:
265 266 qwen3_vllm: { ... }
266 267 ```
267 268  
  269 +翻译是特例，结构为：
  270 +
  271 +```yaml
  272 +services:
  273 + translation:
  274 + service_url: "http://127.0.0.1:6006"
  275 + default_model: "llm"
  276 + default_scene: "general"
  277 + timeout_sec: 10.0
  278 + capabilities:
  279 + llm: { enabled: true, model: "qwen-flash" }
  280 + qwen-mt: { enabled: true, model: "qwen-mt-flash" }
  281 + deepl: { enabled: false, timeout_sec: 10.0 }
  282 +```
  283 +
268 284 - **provider**：调用方如何访问（如 HTTP）。
269 285 - **backend / backends**：当能力由本仓库内服务提供时，该服务加载哪个后端及参数。
  286 +- **translation.service_url**：业务侧统一调用的翻译服务地址。
  287 +- **translation.capabilities**：翻译服务内部可启用的能力注册表。
270 288 - 解析入口：`config/services_config.py` 的 `get_*_config()` 及 `get_*_base_url()` / `get_rerank_service_url()` 等。
271 289  
272 290 ### 6.3 环境变量（常用）
273 291  
274 292 - 能力 URL：`TRANSLATION_SERVICE_URL`、`EMBEDDING_SERVICE_URL`、`RERANKER_SERVICE_URL`
275   -- 能力选择：`TRANSLATION_PROVIDER`、`EMBEDDING_PROVIDER`、`EMBEDDING_BACKEND`、`RERANK_PROVIDER`、`RERANK_BACKEND`
  293 +- 能力选择：`TRANSLATION_MODEL`、`TRANSLATION_SCENE`、`EMBEDDING_PROVIDER`、`EMBEDDING_BACKEND`、`RERANK_PROVIDER`、`RERANK_BACKEND`
276 294 - 环境与索引：`ES_HOST`、`ES_INDEX_NAMESPACE`、`RUNTIME_ENV`、DB 与 Redis 等
277 295  
278 296 详见 [QUICKSTART.md](./QUICKSTART.md) §1.6（.env 与生产凭证）、[Usage-Guide.md](./Usage-Guide.md)。
... ... @@ -293,6 +311,13 @@ services:
293 311 3. 在 `config/config.yaml` 的 `services.<capability>.providers` 下补充参数。
294 312 4. 不修改业务调用方（search/query/indexer 仍通过工厂获取实例）。
295 313  
  314 +翻译不再按这套扩展。新增翻译能力时：
  315 +
  316 +1. 在 `translation/backends/` 中实现新 backend。
  317 +2. 在 `translation/service.py` 中注册工厂。
  318 +3. 在 `services.translation.capabilities.<name>` 下增加配置，并用 `enabled` 控制是否启用。
  319 +4. 业务调用方保持不变，仍只通过 `create_translation_provider()` 调 6006。
  320 +
296 321 ### 7.3 新增 Backend（推理实现）
297 322  
298 323 1. **实现协议**：在对应目录（如 `reranker/backends/`、`embeddings/`）实现满足协议接口的类。
... ...
docs/TODO.txt
  Show/Hide comments   View file @5e4dc8e
... ... @@ -84,7 +84,7 @@ When sorting on a field, scores are not computed. By setting track_scores to tru
84 84 provider backend 两者的关系，如何配合。
85 85 translator的设计 ：
86 86  
87   -QueryParser 里面 并不是调用的6006，目前是把6006做了一个provider，然后translate的总体配置又有6006的baseurl，很混乱！！！
  87 +QueryParser 里面 并不是调用的6006，目前是把6006做了一个provider，然后translate的总体配置又有6006的baseurl，很混乱。
88 88  
89 89 config.yaml 里面的 翻译的配置 不是“6006 专用配置”，而是搜索服务的
90 90 6006本来之前是做一个provider。
... ...
docs/翻译模块说明.md
  Show/Hide comments   View file @5e4dc8e
... ... @@ -12,7 +12,9 @@ DASHSCOPE_API_KEY=sk-xxx
12 12 DEEPL_AUTH_KEY=xxx
13 13  
14 14 # 可选
15   -TRANSLATION_MODEL=qwen # 或 deepl
  15 +TRANSLATION_SERVICE_URL=http://127.0.0.1:6006
  16 +TRANSLATION_MODEL=llm # 默认能力；也可传 qwen-mt / deepl
  17 +TRANSLATION_SCENE=general
16 18 ```
17 19  
18 20 > **重要限速说明（Qwen 机翻）**
... ... @@ -21,9 +23,34 @@ TRANSLATION_MODEL=qwen # 或 deepl
21 23 > - 高并发场景需要在调用端做限流 / 去抖，或改为离线批量翻译
22 24 > - 如需更高吞吐，可考虑 DeepL 或自建翻译服务
23 25  
24   -## Provider 配置
25   -
26   -Provider 与 URL 在 `config/config.yaml` 的 `services.translation`。详见 [QUICKSTART.md](./QUICKSTART.md) §3 与 [DEVELOPER_GUIDE.md](./DEVELOPER_GUIDE.md) §7.2。
  26 +## 配置模型
  27 +
  28 +翻译已改为“一个翻译服务 + 多种翻译能力”的结构：
  29 +
  30 +- 业务侧（`QueryParser` / indexer）统一调用 `http://127.0.0.1:6006`
  31 +- 服务内按 `services.translation.capabilities` 加载并管理各翻译能力
  32 +- 每种能力独立配置 `enabled`、`model`、`timeout` 等参数
  33 +- 外部接口通过 `model + scene` 指定本次使用哪种能力、哪个场景
  34 +
  35 +配置入口在 `config/config.yaml -> services.translation`，核心字段示例：
  36 +
  37 +```yaml
  38 +services:
  39 + translation:
  40 + service_url: "http://127.0.0.1:6006"
  41 + default_model: "llm"
  42 + default_scene: "general"
  43 + timeout_sec: 10.0
  44 + capabilities:
  45 + qwen-mt:
  46 + enabled: true
  47 + model: "qwen-mt-flash"
  48 + llm:
  49 + enabled: true
  50 + model: "qwen-flash"
  51 + deepl:
  52 + enabled: false
  53 +```
27 54  
28 55 ## HTTP 接口契约（translator service，端口 6006）
29 56  
... ... @@ -41,8 +68,8 @@ Provider 与 URL 在 `config/config.yaml` 的 `services.translation`。详见 [Q
41 68 "text": "商品名称",
42 69 "target_lang": "en",
43 70 "source_lang": "zh",
44   - "model": "qwen",
45   - "context": "sku_name",
  71 + "model": "qwen-mt",
  72 + "scene": "sku_name",
46 73 "prompt": null
47 74 }
48 75 ```
... ... @@ -60,7 +87,8 @@ Provider 与 URL 在 `config/config.yaml` 的 `services.translation`。详见 [Q
60 87 "source_lang": "zh",
61 88 "translated_text": "Product name",
62 89 "status": "success",
63   - "model": "qwen"
  90 + "model": "qwen-mt",
  91 + "scene": "sku_name"
64 92 }
65 93 ```
66 94  
... ... @@ -73,17 +101,24 @@ Provider 与 URL 在 `config/config.yaml` 的 `services.translation`。详见 [Q
73 101 "source_lang": "zh",
74 102 "translated_text": ["Product name 1", null],
75 103 "status": "success",
76   - "model": "qwen"
  104 + "model": "qwen-mt",
  105 + "scene": "sku_name"
77 106 }
78 107 ```
79 108  
80 109 批量模式下，**单条失败用 `null` 占位**（即 `translated_text[i] = null`），保证长度与顺序一一对应，避免部分失败导致整批报错。
81 110  
  111 +说明：
  112 +
  113 +- `scene` 是标准字段，`context` 仅保留为兼容别名
  114 +- `model` 只能选择已在 `services.translation.capabilities` 中启用的能力
  115 +- `/health` 会返回 `default_model`、`default_scene` 与 `enabled_capabilities`
  116 +
82 117 ---
83 118  
84   -## 开发者接口约定（Provider / 代码调用）
  119 +## 开发者接口约定（代码调用）
85 120  
86   -除 HTTP 微服务外，代码侧（如 query/indexer）通常通过 `providers.translation.create_translation_provider()` 获取翻译 provider 实例并调用 `translate()`。
  121 +代码侧（如 query/indexer）仍通过 `providers.translation.create_translation_provider()` 获取实例并调用 `translate()`，但该实例现在固定是 **translator service client**，不再在业务侧做翻译 provider 选择。
87 122  
88 123 ### 输入输出形状（Shape）
89 124  
... ... @@ -94,13 +129,8 @@ Provider 与 URL 在 `config/config.yaml` 的 `services.translation`。详见 [Q
94 129  
95 130 ### 批量能力标识（supports_batch）
96 131  
97   -不同 provider 对批量的实现方式可能不同（例如：真正一次请求传多条，或内部循环逐条翻译并保持 shape）。
  132 +服务客户端与服务内后端都可以暴露 `supports_batch`。若后端不支持批量，服务端会逐条拆分并保持 shape。
98 133  
99 134 为便于上层（如 `api/translator_app.py`）做最优调用，provider 可暴露：
100 135  
101 136 - `supports_batch: bool`（property）
102   -
103   -上层在收到 `text` 为列表时：
104   -
105   -- **若 `supports_batch=True`**：可以直接将列表传给 `translate(text=[...])`
106   -- **若 `supports_batch=False`**：上层会逐条拆分调用（仍保证输出列表一一对应、失败为 `null`）
... ...
providers/translation.py
  Show/Hide comments   View file @5e4dc8e
1   -"""Translation provider factory and HTTP provider implementation."""
2   -from __future__ import annotations
3   -
4   -import logging
5   -from typing import Any, Dict, List, Optional, Sequence, Union
6   -import requests
7   -
8   -from config.services_config import get_translation_config, get_translation_base_url
9   -
10   -logger = logging.getLogger(__name__)
11   -
12   -
13   -class HttpTranslationProvider:
14   - """Translation via HTTP service."""
15   -
16   - def __init__(
17   - self,
18   - base_url: str,
19   - model: str = "qwen",
20   - timeout_sec: float = 10.0,
21   - ):
22   - self.base_url = (base_url or "").rstrip("/")
23   - self.model = model or "qwen"
24   - self.timeout_sec = float(timeout_sec or 10.0)
25   -
26   - @property
27   - def supports_batch(self) -> bool:
28   - """
29   - Whether this provider supports list input natively.
30   -
31   - 当前实现中，我们已经在 `_translate_once` 内处理了 list，
32   - 所以可以直接视为支持 batch。
33   - """
34   - return True
  1 +"""Translation client factory for business callers."""
35 2  
36   - def _translate_once(
37   - self,
38   - text: Union[str, Sequence[str]],
39   - target_lang: str,
40   - source_lang: Optional[str] = None,
41   - context: Optional[str] = None,
42   - prompt: Optional[str] = None,
43   - ) -> Union[Optional[str], List[Optional[str]]]:
44   - # 允许 text 为单个字符串或字符串列表
45   - if isinstance(text, (list, tuple)):
46   - # 上游约定：列表输入时，输出列表一一对应；失败位置为 None
47   - results: List[Optional[str]] = []
48   - for item in text:
49   - if item is None or not str(item).strip():
50   - # 空字符串/None 不视为失败，原样返回以保持语义
51   - results.append(item) # type: ignore[arg-type]
52   - continue
53   - try:
54   - single = self._translate_once(
55   - text=str(item),
56   - target_lang=target_lang,
57   - source_lang=source_lang,
58   - context=context,
59   - prompt=prompt,
60   - )
61   - results.append(single) # type: ignore[arg-type]
62   - except Exception:
63   - # 理论上不会进入，因为内部已捕获；兜底保持长度一致
64   - results.append(None)
65   - return results
  3 +from __future__ import annotations
66 4  
67   - if not text or not str(text).strip():
68   - return text # type: ignore[return-value]
69   - try:
70   - url = f"{self.base_url}/translate"
71   - payload = {
72   - "text": text,
73   - "target_lang": target_lang,
74   - "source_lang": source_lang or "auto",
75   - "model": self.model,
76   - }
77   - if context:
78   - payload["context"] = context
79   - if prompt:
80   - payload["prompt"] = prompt
81   - response = requests.post(url, json=payload, timeout=self.timeout_sec)
82   - if response.status_code != 200:
83   - logger.warning(
84   - "HTTP translator failed: status=%s body=%s",
85   - response.status_code,
86   - (response.text or "")[:200],
87   - )
88   - return None
89   - data = response.json()
90   - translated = data.get("translated_text")
91   - return translated if translated is not None else None
92   - except Exception as exc:
93   - logger.warning("HTTP translator request failed: %s", exc, exc_info=True)
94   - return None
  5 +from typing import Any
95 6  
96   - def translate(
97   - self,
98   - text: Union[str, Sequence[str]],
99   - target_lang: str,
100   - source_lang: Optional[str] = None,
101   - context: Optional[str] = None,
102   - prompt: Optional[str] = None,
103   - ) -> Union[Optional[str], List[Optional[str]]]:
104   - return self._translate_once(
105   - text=text,
106   - target_lang=target_lang,
107   - source_lang=source_lang,
108   - context=context,
109   - prompt=prompt,
110   - )
  7 +from config.services_config import get_translation_config
  8 +from translation.client import TranslationServiceClient
111 9  
112 10  
113   -def create_translation_provider(query_config: Any = None) -> Any:
  11 +def create_translation_provider(query_config: Any = None) -> TranslationServiceClient:
114 12 """
115   - Create translation provider from services config.
  13 + Create a translation client.
116 14  
117   - query_config: optional, for api_key/glossary_id/context (used by direct provider).
  15 + Translation is no longer selected via provider mechanism on the caller side.
  16 + Search / indexer always talk to the translator service, while the service
  17 + itself decides which translation capabilities are enabled and how to route.
118 18 """
119   - cfg = get_translation_config()
120   - provider = cfg.provider
121   - pc = cfg.get_provider_cfg()
122   -
123   - if provider in ("qwen-mt", "direct", "local", "inprocess"):
124   - from query.qwen_mt_translate import Translator
125   - model = pc.get("model") or "qwen-mt-flash"
126   - qc = query_config or _empty_query_config()
127   - return Translator(
128   - model=model,
129   - api_key=getattr(qc, "translation_api_key", None),
130   - use_cache=True,
131   - glossary_id=getattr(qc, "translation_glossary_id", None),
132   - translation_context=getattr(qc, "translation_context", "e-commerce product search"),
133   - )
134   -
135   - elif provider in ("http", "service"):
136   - base_url = get_translation_base_url()
137   - model = pc.get("model") or "qwen"
138   - timeout = pc.get("timeout_sec", 10.0)
139   - qc = query_config or _empty_query_config()
140   - return HttpTranslationProvider(
141   - base_url=base_url,
142   - model=model,
143   - timeout_sec=float(timeout),
144   - )
145   -
146   - elif provider == "llm":
147   - from query.llm_translate import LLMTranslatorProvider
148   - model = pc.get("model")
149   - timeout = float(pc.get("timeout_sec", 30.0))
150   - base_url = (pc.get("base_url") or "").strip() or None
151   - return LLMTranslatorProvider(
152   - model=model,
153   - timeout_sec=timeout,
154   - base_url=base_url,
155   - )
156 19  
157   - elif provider == "deepl":
158   - from query.deepl_provider import DeepLProvider
159   - qc = query_config or _empty_query_config()
160   - return DeepLProvider(
161   - api_key=getattr(qc, "translation_api_key", None),
162   - timeout=float(pc.get("timeout_sec", 10.0)),
163   - glossary_id=pc.get("glossary_id") or getattr(qc, "translation_glossary_id", None),
164   - )
165   -
166   - raise ValueError(f"Unsupported translation provider: {provider}")
167   -
168   -
169   -def _empty_query_config() -> Any:
170   - """Minimal object with default translation attrs."""
171   - class _QC:
172   - translation_api_key = None
173   - translation_glossary_id = None
174   - translation_context = "e-commerce product search"
175   - return _QC()
  20 + cfg = get_translation_config()
  21 + qc = query_config
  22 + default_scene = getattr(qc, "translation_context", None) if qc is not None else None
  23 + return TranslationServiceClient(
  24 + base_url=cfg.service_url,
  25 + default_model=cfg.default_model,
  26 + default_scene=default_scene or cfg.default_scene,
  27 + timeout_sec=cfg.timeout_sec,
  28 + )
... ...
query/deepl_provider.py
  Show/Hide comments   View file @5e4dc8e
1   -"""
2   -DeepL backend provider.
3   -
4   -This module only handles network calls to DeepL.
5   -It does not handle cache, async fanout, or fallback semantics.
6   -"""
7   -
8   -from __future__ import annotations
9   -
10   -import logging
11   -import os
12   -import re
13   -from typing import Dict, List, Optional, Sequence, Tuple, Union
14   -
15   -import requests
16   -from config.services_config import get_translation_config
17   -
18   -
19   -logger = logging.getLogger(__name__)
20   -
21   -DEFAULT_CONTEXTS: Dict[str, Dict[str, str]] = {
22   - "sku_name": {
23   - "zh": "商品SKU名称",
24   - "en": "product SKU name",
25   - },
26   - "ecommerce_search_query": {
27   - "zh": "电商",
28   - "en": "e-commerce",
29   - },
30   - "general": {
31   - "zh": "",
32   - "en": "",
33   - },
34   -}
35   -SCENE_NAMES = frozenset(DEFAULT_CONTEXTS.keys())
36   -
37   -
38   -def _merge_contexts(raw: object) -> Dict[str, Dict[str, str]]:
39   - merged: Dict[str, Dict[str, str]] = {
40   - scene: dict(lang_map) for scene, lang_map in DEFAULT_CONTEXTS.items()
41   - }
42   - if not isinstance(raw, dict):
43   - return merged
44   - for scene, lang_map in raw.items():
45   - if not isinstance(lang_map, dict):
46   - continue
47   - scene_name = str(scene or "").strip()
48   - if not scene_name:
49   - continue
50   - merged.setdefault(scene_name, {})
51   - for lang, value in lang_map.items():
52   - lang_key = str(lang or "").strip().lower()
53   - context_value = str(value or "").strip()
54   - if lang_key and context_value:
55   - merged[scene_name][lang_key] = context_value
56   - return merged
57   -
58   -
59   -class DeepLProvider:
60   - API_URL = "https://api.deepl.com/v2/translate" # Pro tier
61   - LANG_CODE_MAP = {
62   - "zh": "ZH",
63   - "en": "EN",
64   - "ru": "RU",
65   - "ar": "AR",
66   - "ja": "JA",
67   - "es": "ES",
68   - "de": "DE",
69   - "fr": "FR",
70   - "it": "IT",
71   - "pt": "PT",
72   - }
73   -
74   - def __init__(
75   - self,
76   - api_key: Optional[str],
77   - *,
78   - timeout: float = 10.0,
79   - glossary_id: Optional[str] = None,
80   - ) -> None:
81   - cfg = get_translation_config()
82   - provider_cfg = cfg.providers.get("deepl", {}) if isinstance(cfg.providers, dict) else {}
83   - self.api_key = api_key or os.getenv("DEEPL_AUTH_KEY")
84   - self.timeout = float(provider_cfg.get("timeout_sec") or timeout or 10.0)
85   - self.glossary_id = glossary_id or provider_cfg.get("glossary_id")
86   - self.model = "deepl"
87   - self.context_presets = _merge_contexts(provider_cfg.get("contexts"))
88   - if not self.api_key:
89   - logger.warning("DEEPL_AUTH_KEY not set; DeepL translation is unavailable")
90   -
91   - @property
92   - def supports_batch(self) -> bool:
93   - """
94   - DeepL HTTP API 本身支持一次传多条 text，这里先返回 False，
95   - 由上层逐条拆分，后续如果要真正批量，可调整实现。
96   - """
97   - return False
98   -
99   - def _resolve_request_context(
100   - self,
101   - target_lang: str,
102   - context: Optional[str],
103   - prompt: Optional[str],
104   - ) -> Optional[str]:
105   - if prompt:
106   - return prompt
107   - if context in SCENE_NAMES:
108   - scene_map = self.context_presets.get(context) or self.context_presets.get("default") or {}
109   - tgt = (target_lang or "").strip().lower()
110   - return scene_map.get(tgt) or scene_map.get("en")
111   - if context:
112   - return context
113   - scene_map = self.context_presets.get("default") or {}
114   - tgt = (target_lang or "").strip().lower()
115   - return scene_map.get(tgt) or scene_map.get("en")
116   -
117   - def translate(
118   - self,
119   - text: Union[str, Sequence[str]],
120   - target_lang: str,
121   - source_lang: Optional[str] = None,
122   - context: Optional[str] = None,
123   - prompt: Optional[str] = None,
124   - ) -> Union[Optional[str], List[Optional[str]]]:
125   - if isinstance(text, (list, tuple)):
126   - results: List[Optional[str]] = []
127   - for item in text:
128   - if item is None or not str(item).strip():
129   - results.append(item) # type: ignore[arg-type]
130   - continue
131   - out = self.translate(
132   - text=str(item),
133   - target_lang=target_lang,
134   - source_lang=source_lang,
135   - context=context,
136   - prompt=prompt,
137   - )
138   - results.append(out)
139   - return results
140   -
141   - if not self.api_key:
142   - return None
143   -
144   - target_code = self.LANG_CODE_MAP.get((target_lang or "").lower(), (target_lang or "").upper())
145   - headers = {
146   - "Authorization": f"DeepL-Auth-Key {self.api_key}",
147   - "Content-Type": "application/json",
148   - }
149   -
150   - api_context = self._resolve_request_context(target_lang, context, prompt)
151   - text_to_translate, needs_extraction = self._add_ecommerce_context(text, source_lang, api_context)
152   -
153   - payload = {
154   - "text": [text_to_translate],
155   - "target_lang": target_code,
156   - }
157   - if source_lang:
158   - payload["source_lang"] = self.LANG_CODE_MAP.get(source_lang.lower(), source_lang.upper())
159   - if api_context:
160   - payload["context"] = api_context
161   - if self.glossary_id:
162   - payload["glossary_id"] = self.glossary_id
163   -
164   - try:
165   - response = requests.post(self.API_URL, headers=headers, json=payload, timeout=self.timeout)
166   - if response.status_code != 200:
167   - logger.warning(
168   - "[deepl] Failed | status=%s tgt=%s body=%s",
169   - response.status_code,
170   - target_code,
171   - (response.text or "")[:200],
172   - )
173   - return None
174   -
175   - data = response.json()
176   - translations = data.get("translations") or []
177   - if not translations:
178   - return None
179   - translated = translations[0].get("text")
180   - if not translated:
181   - return None
182   - if needs_extraction:
183   - translated = self._extract_term_from_translation(translated, text, target_code)
184   - return translated
185   - except requests.Timeout:
186   - logger.warning("[deepl] Timeout | tgt=%s timeout=%.1fs", target_code, self.timeout)
187   - return None
188   - except Exception as exc:
189   - logger.warning("[deepl] Exception | tgt=%s error=%s", target_code, exc, exc_info=True)
190   - return None
191   -
192   - def _add_ecommerce_context(
193   - self,
194   - text: str,
195   - source_lang: Optional[str],
196   - context: Optional[str],
197   - ) -> Tuple[str, bool]:
198   - if not context or "e-commerce" not in context.lower():
199   - return text, False
200   - if (source_lang or "").lower() != "zh":
201   - return text, False
202   -
203   - term = (text or "").strip()
204   - if len(term.split()) == 1 and len(term) <= 2:
205   - return f"购买 {term}", True
206   - return text, False
207   -
208   - def _extract_term_from_translation(
209   - self,
210   - translated_text: str,
211   - original_text: str,
212   - target_lang_code: str,
213   - ) -> str:
214   - del original_text
215   - if target_lang_code != "EN":
216   - return translated_text
217   -
218   - words = translated_text.strip().split()
219   - if len(words) <= 1:
220   - return translated_text
221   - context_words = {"buy", "purchase", "product", "item", "commodity", "goods"}
222   - for word in reversed(words):
223   - normalized = re.sub(r"[.,!?;:]+$", "", word.lower())
224   - if normalized not in context_words:
225   - return normalized
226   - return re.sub(r"[.,!?;:]+$", "", words[-1].lower())
  1 +"""Backward-compatible import for DeepL translation backend."""
227 2  
  3 +from translation.backends.deepl import DeepLProvider, DeepLTranslationBackend
... ...
query/llm_translate.py
  Show/Hide comments   View file @5e4dc8e
1   -"""
2   -LLM-based translation backend (DashScope-compatible OpenAI API).
  1 +"""Backward-compatible import for LLM translation backend."""
3 2  
4   -Failure semantics are strict:
5   -- success: translated string
6   -- failure: None
7   -"""
  3 +from translation.backends.llm import LLMTranslationBackend, LLMTranslatorProvider, llm_translate
8 4  
9   -from __future__ import annotations
10   -
11   -import logging
12   -import os
13   -import time
14   -from typing import List, Optional, Sequence, Union
15   -
16   -from openai import OpenAI
17   -
18   -from config.env_config import DASHSCOPE_API_KEY
19   -from config.services_config import get_translation_config
20   -from config.translate_prompts import TRANSLATION_PROMPTS
21   -from config.tenant_config_loader import SOURCE_LANG_CODE_MAP, TARGET_LANG_CODE_MAP
22   -
23   -
24   -logger = logging.getLogger(__name__)
25   -
26   -
27   -DEFAULT_QWEN_BASE_URL = "https://dashscope-us.aliyuncs.com/compatible-mode/v1"
28   -DEFAULT_LLM_MODEL = "qwen-flash"
29   -
30   -
31   -def _build_prompt(
32   - text: str,
33   - *,
34   - source_lang: Optional[str],
35   - target_lang: str,
36   - scene: Optional[str],
37   -) -> str:
38   - """
39   - 从 config.translate_prompts.TRANSLATION_PROMPTS 中构建提示词。
40   -
41   - 要求：模板必须包含 {source_lang}（{src_lang_code}）{target_lang}（{tgt_lang_code}）。
42   - 这里统一使用 code 作为占位的 lang 与 label，外部接口仍然只传语言 code。
43   - """
44   - tgt = (target_lang or "").lower() or "en"
45   - src = (source_lang or "auto").lower()
46   -
47   - # 将业务上下文 scene 映射为模板分组名
48   - normalized_scene = (scene or "").strip() or "general"
49   - # 如果出现历史词，则报错，用于发现错误
50   - if normalized_scene in {"query", "ecommerce_search", "ecommerce_search_query"}:
51   - group_key = "ecommerce_search_query"
52   - elif normalized_scene in {"product_title", "sku_name"}:
53   - group_key = "sku_name"
54   - else:
55   - group_key = normalized_scene
56   - group = TRANSLATION_PROMPTS.get(group_key) or TRANSLATION_PROMPTS["general"]
57   -
58   - # 先按目标语言 code 取模板，取不到回退到英文
59   - template = group.get(tgt) or group.get("en")
60   - if not template:
61   - # 理论上不会发生，兜底一个简单模板
62   - template = (
63   - "You are a professional {source_lang} ({src_lang_code}) to "
64   - "{target_lang} ({tgt_lang_code}) translator, output only the translation: {text}"
65   - )
66   -
67   - # 目前不额外维护语言名称映射，直接使用 code 作为 label
68   - source_lang_label = SOURCE_LANG_CODE_MAP.get(src, src)
69   - target_lang_label = SOURCE_LANG_CODE_MAP.get(tgt, tgt)
70   -
71   - return template.format(
72   - source_lang=source_lang_label,
73   - src_lang_code=src,
74   - target_lang=target_lang_label,
75   - tgt_lang_code=tgt,
76   - text=text,
77   - )
78   -
79   -
80   -class LLMTranslatorProvider:
81   - def __init__(
82   - self,
83   - *,
84   - model: Optional[str] = None,
85   - timeout_sec: float = 30.0,
86   - base_url: Optional[str] = None,
87   - ) -> None:
88   - cfg = get_translation_config()
89   - llm_cfg = cfg.providers.get("llm", {}) if isinstance(cfg.providers, dict) else {}
90   - self.model = model or llm_cfg.get("model") or DEFAULT_LLM_MODEL
91   - self.timeout_sec = float(llm_cfg.get("timeout_sec") or timeout_sec or 30.0)
92   - self.base_url = (
93   - (base_url or "").strip()
94   - or (llm_cfg.get("base_url") or "").strip()
95   - or os.getenv("DASHSCOPE_BASE_URL")
96   - or DEFAULT_QWEN_BASE_URL
97   - )
98   - self.client = self._create_client()
99   -
100   - @property
101   - def supports_batch(self) -> bool:
102   - """Whether this provider efficiently supports list input."""
103   - # 我们在 translate 中已经原生支持 list，所以这里返回 True
104   - return True
105   -
106   - def _create_client(self) -> Optional[OpenAI]:
107   - api_key = DASHSCOPE_API_KEY or os.getenv("DASHSCOPE_API_KEY")
108   - if not api_key:
109   - logger.warning("DASHSCOPE_API_KEY not set; llm translation unavailable")
110   - return None
111   - try:
112   - return OpenAI(api_key=api_key, base_url=self.base_url)
113   - except Exception as exc:
114   - logger.error("Failed to initialize llm translation client: %s", exc, exc_info=True)
115   - return None
116   -
117   - def _translate_single(
118   - self,
119   - text: str,
120   - target_lang: str,
121   - source_lang: Optional[str] = None,
122   - context: Optional[str] = None,
123   - prompt: Optional[str] = None,
124   - ) -> Optional[str]:
125   - if not text or not str(text).strip():
126   - return text
127   - if not self.client:
128   - return None
129   -
130   - tgt = (target_lang or "").lower() or "en"
131   - src = (source_lang or "auto").lower()
132   - scene = context or "default"
133   - user_prompt = prompt or _build_prompt(
134   - text=text,
135   - source_lang=src,
136   - target_lang=tgt,
137   - scene=scene,
138   - )
139   - start = time.time()
140   - try:
141   - logger.info(
142   - "[llm] Request | src=%s tgt=%s model=%s prompt=%s",
143   - src,
144   - tgt,
145   - self.model,
146   - user_prompt,
147   - )
148   - completion = self.client.chat.completions.create(
149   - model=self.model,
150   - messages=[{"role": "user", "content": user_prompt}],
151   - timeout=self.timeout_sec,
152   - )
153   - content = (completion.choices[0].message.content or "").strip()
154   - latency_ms = (time.time() - start) * 1000
155   - if not content:
156   - logger.warning("[llm] Empty result | src=%s tgt=%s latency=%.1fms", src, tgt, latency_ms)
157   - return None
158   - logger.info(
159   - "[llm] Success | src=%s tgt=%s src_text=%s response=%s latency=%.1fms",
160   - src,
161   - tgt,
162   - text,
163   - content,
164   - latency_ms,
165   - )
166   - return content
167   - except Exception as exc:
168   - latency_ms = (time.time() - start) * 1000
169   - logger.warning(
170   - "[llm] Failed | src=%s tgt=%s latency=%.1fms error=%s",
171   - src,
172   - tgt,
173   - latency_ms,
174   - exc,
175   - exc_info=True,
176   - )
177   - return None
178   -
179   - def translate(
180   - self,
181   - text: Union[str, Sequence[str]],
182   - target_lang: str,
183   - source_lang: Optional[str] = None,
184   - context: Optional[str] = None,
185   - prompt: Optional[str] = None,
186   - ) -> Union[Optional[str], List[Optional[str]]]:
187   - """
188   - Translate a single string or a list of strings.
189   -
190   - - If input is a list, returns a list of the same length.
191   - - Per-item failures are returned as None.
192   - """
193   - if isinstance(text, (list, tuple)):
194   - results: List[Optional[str]] = []
195   - for item in text:
196   - # 保证一一对应，即使某个元素为空也占位
197   - if item is None:
198   - results.append(None)
199   - continue
200   - results.append(
201   - self._translate_single(
202   - text=str(item),
203   - target_lang=target_lang,
204   - source_lang=source_lang,
205   - context=context,
206   - prompt=prompt,
207   - )
208   - )
209   - return results
210   -
211   - return self._translate_single(
212   - text=str(text),
213   - target_lang=target_lang,
214   - source_lang=source_lang,
215   - context=context,
216   - prompt=prompt,
217   - )
218   -
219   -
220   -def llm_translate(
221   - text: Union[str, Sequence[str]],
222   - target_lang: str,
223   - *,
224   - source_lang: Optional[str] = None,
225   - source_lang_label: Optional[str] = None,
226   - target_lang_label: Optional[str] = None,
227   - timeout_sec: Optional[float] = None,
228   -) -> Union[Optional[str], List[Optional[str]]]:
229   - provider = LLMTranslatorProvider(timeout_sec=timeout_sec or 30.0)
230   - return provider.translate(
231   - text=text,
232   - target_lang=target_lang,
233   - source_lang=source_lang,
234   - context=None,
235   - )
236   -
237   -
238   -__all__ = ["LLMTranslatorProvider", "llm_translate"]
  5 +__all__ = ["LLMTranslationBackend", "LLMTranslatorProvider", "llm_translate"]
... ...
query/query_parser.py
  Show/Hide comments   View file @5e4dc8e
... ... @@ -133,7 +133,11 @@ class QueryParser:
133 133 if self._translator is None:
134 134 from config.services_config import get_translation_config
135 135 cfg = get_translation_config()
136   - logger.info("Initializing translator at QueryParser construction (provider=%s)...", cfg.provider)
  136 + logger.info(
  137 + "Initializing translator client at QueryParser construction (service_url=%s, default_model=%s)...",
  138 + cfg.service_url,
  139 + cfg.default_model,
  140 + )
137 141 self._translator = create_translation_provider(self.config.query_config)
138 142 self._translation_executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="query-translation")
139 143  
... ...
query/qwen_mt_translate.py
  Show/Hide comments   View file @5e4dc8e
1   -"""Qwen-MT translation orchestrator with cache and async helpers."""
  1 +"""Backward-compatible import for Qwen-MT translation backend."""
2 2  
3   -from __future__ import annotations
  3 +from translation.backends.qwen_mt import QwenMTTranslationBackend, Translator
4 4  
5   -import hashlib
6   -import logging
7   -import os
8   -import re
9   -import time
10   -from typing import Dict, List, Optional, Sequence, Union
11   -
12   -import redis
13   -from openai import OpenAI
14   -
15   -from config.env_config import DASHSCOPE_API_KEY, REDIS_CONFIG
16   -from config.services_config import get_translation_cache_config
17   -from config.tenant_config_loader import SOURCE_LANG_CODE_MAP, TARGET_LANG_CODE_MAP
18   -
19   -logger = logging.getLogger(__name__)
20   -
21   -
22   -class Translator:
23   - QWEN_DEFAULT_BASE_URL = "https://dashscope-us.aliyuncs.com/compatible-mode/v1"
24   - QWEN_MODEL = "qwen-mt-flash"
25   -
26   - def __init__(
27   - self,
28   - model: str = "qwen",
29   - api_key: Optional[str] = None,
30   - use_cache: bool = True,
31   - timeout: int = 10,
32   - glossary_id: Optional[str] = None,
33   - translation_context: Optional[str] = None,
34   - ):
35   - self.model = self._normalize_model(model)
36   - self.timeout = int(timeout)
37   - self.use_cache = bool(use_cache)
38   - self.glossary_id = glossary_id
39   - self.translation_context = translation_context or "e-commerce product search"
40   -
41   - cache_cfg = get_translation_cache_config()
42   - self.cache_prefix = str(cache_cfg.get("key_prefix", "trans:v2"))
43   - self.expire_seconds = int(cache_cfg.get("ttl_seconds", 360 * 24 * 3600))
44   - self.cache_sliding_expiration = bool(cache_cfg.get("sliding_expiration", True))
45   - self.cache_include_context = bool(cache_cfg.get("key_include_context", True))
46   - self.cache_include_prompt = bool(cache_cfg.get("key_include_prompt", True))
47   - self.cache_include_source_lang = bool(cache_cfg.get("key_include_source_lang", True))
48   -
49   - self.qwen_model_name = self._resolve_qwen_model_name(model)
50   - self._api_key = api_key or self._default_api_key(self.model)
51   - self._qwen_client: Optional[OpenAI] = None
52   - base_url = os.getenv("DASHSCOPE_BASE_URL") or self.QWEN_DEFAULT_BASE_URL
53   - if self._api_key:
54   - try:
55   - self._qwen_client = OpenAI(api_key=self._api_key, base_url=base_url)
56   - except Exception as exc:
57   - logger.warning("Failed to initialize qwen-mt client: %s", exc, exc_info=True)
58   - else:
59   - logger.warning("DASHSCOPE_API_KEY not set; qwen-mt translation unavailable")
60   -
61   - self.redis_client = None
62   - if self.use_cache and bool(cache_cfg.get("enabled", True)):
63   - self.redis_client = self._init_redis_client()
64   -
65   - @property
66   - def supports_batch(self) -> bool:
67   - """
68   - 标记该 provider 已支持列表输入。
69   -
70   - 当前实现为循环单条调用（带缓存），不是真正的并行批量请求，
71   - 但对上层来说可以直接传 list，返回 list。
72   - """
73   - return True
74   -
75   - @staticmethod
76   - def _normalize_model(model: str) -> str:
77   - m = (model or "qwen").strip().lower()
78   - if m.startswith("qwen"):
79   - return "qwen-mt"
80   - raise ValueError(f"Unsupported model: {model}. Supported models: 'qwen', 'qwen-mt', 'qwen-mt-flash'")
81   -
82   - @staticmethod
83   - def _resolve_qwen_model_name(model: str) -> str:
84   - m = (model or "qwen").strip().lower()
85   - if m in {"qwen", "qwen-mt"}:
86   - return "qwen-mt-flash"
87   - return m
88   -
89   - @staticmethod
90   - def _default_api_key(model: str) -> Optional[str]:
91   - del model
92   - return DASHSCOPE_API_KEY or os.getenv("DASHSCOPE_API_KEY")
93   -
94   - def _init_redis_client(self):
95   - try:
96   - client = redis.Redis(
97   - host=REDIS_CONFIG.get("host", "localhost"),
98   - port=REDIS_CONFIG.get("port", 6479),
99   - password=REDIS_CONFIG.get("password"),
100   - decode_responses=True,
101   - socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),
102   - socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),
103   - retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False),
104   - health_check_interval=10,
105   - )
106   - client.ping()
107   - return client
108   - except Exception as exc:
109   - logger.warning("Failed to initialize translation redis cache: %s", exc)
110   - return None
111   -
112   - def _build_cache_key(
113   - self,
114   - text: str,
115   - target_lang: str,
116   - source_lang: Optional[str],
117   - context: Optional[str],
118   - prompt: Optional[str],
119   - ) -> str:
120   - src = (source_lang or "auto").strip().lower() if self.cache_include_source_lang else "-"
121   - tgt = (target_lang or "").strip().lower()
122   - ctx = (context or "").strip() if self.cache_include_context else ""
123   - prm = (prompt or "").strip() if self.cache_include_prompt else ""
124   - payload = f"model={self.model}\nsrc={src}\ntgt={tgt}\nctx={ctx}\nprm={prm}\ntext={text}"
125   - digest = hashlib.sha256(payload.encode("utf-8")).hexdigest()
126   - return f"{self.cache_prefix}:{self.model}:{src}:{tgt}:{digest}"
127   -
128   - def translate(
129   - self,
130   - text: Union[str, Sequence[str]],
131   - target_lang: str,
132   - source_lang: Optional[str] = None,
133   - context: Optional[str] = None,
134   - prompt: Optional[str] = None,
135   - ) -> Union[Optional[str], List[Optional[str]]]:
136   - if isinstance(text, (list, tuple)):
137   - results: List[Optional[str]] = []
138   - for item in text:
139   - if item is None or not str(item).strip():
140   - results.append(item) # type: ignore[arg-type]
141   - continue
142   - # 对于 batch，这里沿用单条的缓存与规则，逐条调用
143   - out = self.translate(
144   - text=str(item),
145   - target_lang=target_lang,
146   - source_lang=source_lang,
147   - context=context,
148   - prompt=prompt,
149   - )
150   - results.append(out)
151   - return results
152   -
153   - if not text or not str(text).strip():
154   - return text # type: ignore[return-value]
155   -
156   - tgt = (target_lang or "").strip().lower()
157   - src = (source_lang or "").strip().lower() or None
158   - if tgt == "en" and self._is_english_text(text):
159   - return text
160   - if tgt == "zh" and (self._contains_chinese(text) or self._is_pure_number(text)):
161   - return text
162   -
163   - translation_context = context or self.translation_context
164   - cached = self._get_cached_translation_redis(text, tgt, src, translation_context, prompt)
165   - if cached is not None:
166   - return cached
167   -
168   - result = self._translate_qwen(text, tgt, src)
169   -
170   - if result is not None:
171   - self._set_cached_translation_redis(text, tgt, result, src, translation_context, prompt)
172   - return result
173   -
174   - def _translate_qwen(
175   - self,
176   - text: str,
177   - target_lang: str,
178   - source_lang: Optional[str],
179   - ) -> Optional[str]:
180   - if not self._qwen_client:
181   - return None
182   - tgt_norm = (target_lang or "").strip().lower()
183   - src_norm = (source_lang or "").strip().lower()
184   - tgt_qwen = self.SOURCE_LANG_CODE_MAP.get(tgt_norm, tgt_norm.capitalize())
185   - src_qwen = "auto" if not src_norm or src_norm == "auto" else self.SOURCE_LANG_CODE_MAP.get(src_norm, src_norm.capitalize())
186   - start = time.time()
187   - try:
188   - completion = self._qwen_client.chat.completions.create(
189   - model=self.qwen_model_name,
190   - messages=[{"role": "user", "content": text}],
191   - extra_body={
192   - "translation_options": {
193   - "source_lang": src_qwen,
194   - "target_lang": tgt_qwen,
195   - }
196   - },
197   - timeout=self.timeout,
198   - )
199   - content = (completion.choices[0].message.content or "").strip()
200   - if not content:
201   - return None
202   - logger.info("[qwen-mt] Success | src=%s tgt=%s latency=%.1fms", src_qwen, tgt_qwen, (time.time() - start) * 1000)
203   - return content
204   - except Exception as exc:
205   - logger.warning(
206   - "[qwen-mt] Failed | src=%s tgt=%s latency=%.1fms error=%s",
207   - src_qwen,
208   - tgt_qwen,
209   - (time.time() - start) * 1000,
210   - exc,
211   - exc_info=True,
212   - )
213   - return None
214   -
215   -
216   - def _get_cached_translation_redis(
217   - self,
218   - text: str,
219   - target_lang: str,
220   - source_lang: Optional[str] = None,
221   - context: Optional[str] = None,
222   - prompt: Optional[str] = None,
223   - ) -> Optional[str]:
224   - if not self.redis_client:
225   - return None
226   - key = self._build_cache_key(text, target_lang, source_lang, context, prompt)
227   - try:
228   - value = self.redis_client.get(key)
229   - if value and self.cache_sliding_expiration:
230   - self.redis_client.expire(key, self.expire_seconds)
231   - return value
232   - except Exception as exc:
233   - logger.warning("Redis get translation cache failed: %s", exc)
234   - return None
235   -
236   - def _set_cached_translation_redis(
237   - self,
238   - text: str,
239   - target_lang: str,
240   - translation: str,
241   - source_lang: Optional[str] = None,
242   - context: Optional[str] = None,
243   - prompt: Optional[str] = None,
244   - ) -> None:
245   - if not self.redis_client:
246   - return
247   - key = self._build_cache_key(text, target_lang, source_lang, context, prompt)
248   - try:
249   - self.redis_client.setex(key, self.expire_seconds, translation)
250   - except Exception as exc:
251   - logger.warning("Redis set translation cache failed: %s", exc)
252   -
253   - def _shop_lang_matches(self, shop_lang_lower: str, lang_code: str) -> bool:
254   - if not shop_lang_lower or not lang_code:
255   - return False
256   - if shop_lang_lower == lang_code:
257   - return True
258   - if lang_code == "zh" and "zh" in shop_lang_lower:
259   - return True
260   - if lang_code == "en" and "en" in shop_lang_lower:
261   - return True
262   - return False
263   -
264   - def get_translation_needs(self, detected_lang: str, supported_langs: List[str]) -> List[str]:
265   - if detected_lang in supported_langs:
266   - return [lang for lang in supported_langs if lang != detected_lang]
267   - return supported_langs
268   -
269   - def _is_english_text(self, text: str) -> bool:
270   - if not text or not text.strip():
271   - return True
272   - text_clean = re.sub(r"[\s\.,!?;:\-\'\"\(\)\[\]{}]", "", text)
273   - if not text_clean:
274   - return True
275   - ascii_count = sum(1 for c in text_clean if ord(c) < 128)
276   - return (ascii_count / len(text_clean)) > 0.8
277   -
278   - def _contains_chinese(self, text: str) -> bool:
279   - if not text:
280   - return False
281   - return bool(re.search(r"[\u4e00-\u9fff]", text))
282   -
283   - def _is_pure_number(self, text: str) -> bool:
284   - if not text or not text.strip():
285   - return False
286   - text_clean = re.sub(r"[\s\.,]", "", text.strip())
287   - return bool(text_clean) and text_clean.isdigit()
  5 +__all__ = ["QwenMTTranslationBackend", "Translator"]
... ...
translation/__init__.py 0 → 100644
  Show/Hide comments   View file @5e4dc8e
... ... @@ -0,0 +1,8 @@
  1 +"""Translation package."""
  2 +
  3 +__all__ = [
  4 + "client",
  5 + "service",
  6 + "protocols",
  7 + "backends",
  8 +]
... ...
translation/backends/__init__.py 0 → 100644
  Show/Hide comments   View file @5e4dc8e
... ... @@ -0,0 +1,11 @@
  1 +"""Translation backend registry."""
  2 +
  3 +from .deepl import DeepLTranslationBackend
  4 +from .llm import LLMTranslationBackend
  5 +from .qwen_mt import QwenMTTranslationBackend
  6 +
  7 +__all__ = [
  8 + "DeepLTranslationBackend",
  9 + "LLMTranslationBackend",
  10 + "QwenMTTranslationBackend",
  11 +]
... ...
translation/backends/deepl.py 0 → 100644
  Show/Hide comments   View file @5e4dc8e
... ... @@ -0,0 +1,220 @@
  1 +"""DeepL translation backend."""
  2 +
  3 +from __future__ import annotations
  4 +
  5 +import logging
  6 +import os
  7 +import re
  8 +from typing import Dict, List, Optional, Sequence, Tuple, Union
  9 +
  10 +import requests
  11 +
  12 +from config.services_config import get_translation_config
  13 +
  14 +logger = logging.getLogger(__name__)
  15 +
  16 +DEFAULT_CONTEXTS: Dict[str, Dict[str, str]] = {
  17 + "sku_name": {
  18 + "zh": "商品SKU名称",
  19 + "en": "product SKU name",
  20 + },
  21 + "ecommerce_search_query": {
  22 + "zh": "电商",
  23 + "en": "e-commerce",
  24 + },
  25 + "general": {
  26 + "zh": "",
  27 + "en": "",
  28 + },
  29 +}
  30 +SCENE_NAMES = frozenset(DEFAULT_CONTEXTS.keys())
  31 +
  32 +
  33 +def _merge_contexts(raw: object) -> Dict[str, Dict[str, str]]:
  34 + merged: Dict[str, Dict[str, str]] = {
  35 + scene: dict(lang_map) for scene, lang_map in DEFAULT_CONTEXTS.items()
  36 + }
  37 + if not isinstance(raw, dict):
  38 + return merged
  39 + for scene, lang_map in raw.items():
  40 + if not isinstance(lang_map, dict):
  41 + continue
  42 + scene_name = str(scene or "").strip()
  43 + if not scene_name:
  44 + continue
  45 + merged.setdefault(scene_name, {})
  46 + for lang, value in lang_map.items():
  47 + lang_key = str(lang or "").strip().lower()
  48 + context_value = str(value or "").strip()
  49 + if lang_key and context_value:
  50 + merged[scene_name][lang_key] = context_value
  51 + return merged
  52 +
  53 +
  54 +class DeepLTranslationBackend:
  55 + API_URL = "https://api.deepl.com/v2/translate"
  56 + LANG_CODE_MAP = {
  57 + "zh": "ZH",
  58 + "en": "EN",
  59 + "ru": "RU",
  60 + "ar": "AR",
  61 + "ja": "JA",
  62 + "es": "ES",
  63 + "de": "DE",
  64 + "fr": "FR",
  65 + "it": "IT",
  66 + "pt": "PT",
  67 + }
  68 +
  69 + def __init__(
  70 + self,
  71 + api_key: Optional[str],
  72 + *,
  73 + timeout: float = 10.0,
  74 + glossary_id: Optional[str] = None,
  75 + ) -> None:
  76 + cfg = get_translation_config()
  77 + provider_cfg = cfg.get_capability_cfg("deepl")
  78 + self.api_key = api_key or os.getenv("DEEPL_AUTH_KEY")
  79 + self.timeout = float(provider_cfg.get("timeout_sec") or timeout or 10.0)
  80 + self.glossary_id = glossary_id or provider_cfg.get("glossary_id")
  81 + self.model = "deepl"
  82 + self.context_presets = _merge_contexts(provider_cfg.get("contexts"))
  83 + if not self.api_key:
  84 + logger.warning("DEEPL_AUTH_KEY not set; DeepL translation is unavailable")
  85 +
  86 + @property
  87 + def supports_batch(self) -> bool:
  88 + return False
  89 +
  90 + def _resolve_request_context(
  91 + self,
  92 + target_lang: str,
  93 + context: Optional[str],
  94 + prompt: Optional[str],
  95 + ) -> Optional[str]:
  96 + if prompt:
  97 + return prompt
  98 + if context in SCENE_NAMES:
  99 + scene_map = self.context_presets.get(context) or self.context_presets.get("default") or {}
  100 + tgt = (target_lang or "").strip().lower()
  101 + return scene_map.get(tgt) or scene_map.get("en")
  102 + if context:
  103 + return context
  104 + scene_map = self.context_presets.get("default") or {}
  105 + tgt = (target_lang or "").strip().lower()
  106 + return scene_map.get(tgt) or scene_map.get("en")
  107 +
  108 + def translate(
  109 + self,
  110 + text: Union[str, Sequence[str]],
  111 + target_lang: str,
  112 + source_lang: Optional[str] = None,
  113 + context: Optional[str] = None,
  114 + prompt: Optional[str] = None,
  115 + ) -> Union[Optional[str], List[Optional[str]]]:
  116 + if isinstance(text, (list, tuple)):
  117 + results: List[Optional[str]] = []
  118 + for item in text:
  119 + if item is None or not str(item).strip():
  120 + results.append(item) # type: ignore[arg-type]
  121 + continue
  122 + out = self.translate(
  123 + text=str(item),
  124 + target_lang=target_lang,
  125 + source_lang=source_lang,
  126 + context=context,
  127 + prompt=prompt,
  128 + )
  129 + results.append(out)
  130 + return results
  131 +
  132 + if not self.api_key:
  133 + return None
  134 +
  135 + target_code = self.LANG_CODE_MAP.get((target_lang or "").lower(), (target_lang or "").upper())
  136 + headers = {
  137 + "Authorization": f"DeepL-Auth-Key {self.api_key}",
  138 + "Content-Type": "application/json",
  139 + }
  140 +
  141 + api_context = self._resolve_request_context(target_lang, context, prompt)
  142 + text_to_translate, needs_extraction = self._add_ecommerce_context(text, source_lang, api_context)
  143 +
  144 + payload = {
  145 + "text": [text_to_translate],
  146 + "target_lang": target_code,
  147 + }
  148 + if source_lang:
  149 + payload["source_lang"] = self.LANG_CODE_MAP.get(source_lang.lower(), source_lang.upper())
  150 + if api_context:
  151 + payload["context"] = api_context
  152 + if self.glossary_id:
  153 + payload["glossary_id"] = self.glossary_id
  154 +
  155 + try:
  156 + response = requests.post(self.API_URL, headers=headers, json=payload, timeout=self.timeout)
  157 + if response.status_code != 200:
  158 + logger.warning(
  159 + "[deepl] Failed | status=%s tgt=%s body=%s",
  160 + response.status_code,
  161 + target_code,
  162 + (response.text or "")[:200],
  163 + )
  164 + return None
  165 +
  166 + data = response.json()
  167 + translations = data.get("translations") or []
  168 + if not translations:
  169 + return None
  170 + translated = translations[0].get("text")
  171 + if not translated:
  172 + return None
  173 + if needs_extraction:
  174 + translated = self._extract_term_from_translation(translated, text, target_code)
  175 + return translated
  176 + except requests.Timeout:
  177 + logger.warning("[deepl] Timeout | tgt=%s timeout=%.1fs", target_code, self.timeout)
  178 + return None
  179 + except Exception as exc:
  180 + logger.warning("[deepl] Exception | tgt=%s error=%s", target_code, exc, exc_info=True)
  181 + return None
  182 +
  183 + def _add_ecommerce_context(
  184 + self,
  185 + text: str,
  186 + source_lang: Optional[str],
  187 + context: Optional[str],
  188 + ) -> Tuple[str, bool]:
  189 + if not context or "e-commerce" not in context.lower():
  190 + return text, False
  191 + if (source_lang or "").lower() != "zh":
  192 + return text, False
  193 +
  194 + term = (text or "").strip()
  195 + if len(term.split()) == 1 and len(term) <= 2:
  196 + return f"购买 {term}", True
  197 + return text, False
  198 +
  199 + def _extract_term_from_translation(
  200 + self,
  201 + translated_text: str,
  202 + original_text: str,
  203 + target_lang_code: str,
  204 + ) -> str:
  205 + del original_text
  206 + if target_lang_code != "EN":
  207 + return translated_text
  208 +
  209 + words = translated_text.strip().split()
  210 + if len(words) <= 1:
  211 + return translated_text
  212 + context_words = {"buy", "purchase", "product", "item", "commodity", "goods"}
  213 + for word in reversed(words):
  214 + normalized = re.sub(r"[.,!?;:]+$", "", word.lower())
  215 + if normalized not in context_words:
  216 + return normalized
  217 + return re.sub(r"[.,!?;:]+$", "", words[-1].lower())
  218 +
  219 +
  220 +DeepLProvider = DeepLTranslationBackend
... ...
translation/backends/llm.py 0 → 100644
  Show/Hide comments   View file @5e4dc8e
... ... @@ -0,0 +1,209 @@
  1 +"""LLM-based translation backend."""
  2 +
  3 +from __future__ import annotations
  4 +
  5 +import logging
  6 +import os
  7 +import time
  8 +from typing import List, Optional, Sequence, Union
  9 +
  10 +from openai import OpenAI
  11 +
  12 +from config.env_config import DASHSCOPE_API_KEY
  13 +from config.services_config import get_translation_config
  14 +from config.translate_prompts import TRANSLATION_PROMPTS
  15 +from config.tenant_config_loader import SOURCE_LANG_CODE_MAP
  16 +
  17 +logger = logging.getLogger(__name__)
  18 +
  19 +DEFAULT_QWEN_BASE_URL = "https://dashscope-us.aliyuncs.com/compatible-mode/v1"
  20 +DEFAULT_LLM_MODEL = "qwen-flash"
  21 +
  22 +
  23 +def _build_prompt(
  24 + text: str,
  25 + *,
  26 + source_lang: Optional[str],
  27 + target_lang: str,
  28 + scene: Optional[str],
  29 +) -> str:
  30 + tgt = (target_lang or "").lower() or "en"
  31 + src = (source_lang or "auto").lower()
  32 + normalized_scene = (scene or "").strip() or "general"
  33 + if normalized_scene in {"query", "ecommerce_search", "ecommerce_search_query"}:
  34 + group_key = "ecommerce_search_query"
  35 + elif normalized_scene in {"product_title", "sku_name"}:
  36 + group_key = "sku_name"
  37 + else:
  38 + group_key = normalized_scene
  39 + group = TRANSLATION_PROMPTS.get(group_key) or TRANSLATION_PROMPTS["general"]
  40 + template = group.get(tgt) or group.get("en")
  41 + if not template:
  42 + template = (
  43 + "You are a professional {source_lang} ({src_lang_code}) to "
  44 + "{target_lang} ({tgt_lang_code}) translator, output only the translation: {text}"
  45 + )
  46 +
  47 + source_lang_label = SOURCE_LANG_CODE_MAP.get(src, src)
  48 + target_lang_label = SOURCE_LANG_CODE_MAP.get(tgt, tgt)
  49 +
  50 + return template.format(
  51 + source_lang=source_lang_label,
  52 + src_lang_code=src,
  53 + target_lang=target_lang_label,
  54 + tgt_lang_code=tgt,
  55 + text=text,
  56 + )
  57 +
  58 +
  59 +class LLMTranslationBackend:
  60 + def __init__(
  61 + self,
  62 + *,
  63 + model: Optional[str] = None,
  64 + timeout_sec: float = 30.0,
  65 + base_url: Optional[str] = None,
  66 + ) -> None:
  67 + cfg = get_translation_config()
  68 + llm_cfg = cfg.get_capability_cfg("llm")
  69 + self.model = model or llm_cfg.get("model") or DEFAULT_LLM_MODEL
  70 + self.timeout_sec = float(llm_cfg.get("timeout_sec") or timeout_sec or 30.0)
  71 + self.base_url = (
  72 + (base_url or "").strip()
  73 + or (llm_cfg.get("base_url") or "").strip()
  74 + or os.getenv("DASHSCOPE_BASE_URL")
  75 + or DEFAULT_QWEN_BASE_URL
  76 + )
  77 + self.client = self._create_client()
  78 +
  79 + @property
  80 + def supports_batch(self) -> bool:
  81 + return True
  82 +
  83 + def _create_client(self) -> Optional[OpenAI]:
  84 + api_key = DASHSCOPE_API_KEY or os.getenv("DASHSCOPE_API_KEY")
  85 + if not api_key:
  86 + logger.warning("DASHSCOPE_API_KEY not set; llm translation unavailable")
  87 + return None
  88 + try:
  89 + return OpenAI(api_key=api_key, base_url=self.base_url)
  90 + except Exception as exc:
  91 + logger.error("Failed to initialize llm translation client: %s", exc, exc_info=True)
  92 + return None
  93 +
  94 + def _translate_single(
  95 + self,
  96 + text: str,
  97 + target_lang: str,
  98 + source_lang: Optional[str] = None,
  99 + context: Optional[str] = None,
  100 + prompt: Optional[str] = None,
  101 + ) -> Optional[str]:
  102 + if not text or not str(text).strip():
  103 + return text
  104 + if not self.client:
  105 + return None
  106 +
  107 + tgt = (target_lang or "").lower() or "en"
  108 + src = (source_lang or "auto").lower()
  109 + scene = context or "default"
  110 + user_prompt = prompt or _build_prompt(
  111 + text=text,
  112 + source_lang=src,
  113 + target_lang=tgt,
  114 + scene=scene,
  115 + )
  116 + start = time.time()
  117 + try:
  118 + logger.info(
  119 + "[llm] Request | src=%s tgt=%s model=%s prompt=%s",
  120 + src,
  121 + tgt,
  122 + self.model,
  123 + user_prompt,
  124 + )
  125 + completion = self.client.chat.completions.create(
  126 + model=self.model,
  127 + messages=[{"role": "user", "content": user_prompt}],
  128 + timeout=self.timeout_sec,
  129 + )
  130 + content = (completion.choices[0].message.content or "").strip()
  131 + latency_ms = (time.time() - start) * 1000
  132 + if not content:
  133 + logger.warning("[llm] Empty result | src=%s tgt=%s latency=%.1fms", src, tgt, latency_ms)
  134 + return None
  135 + logger.info(
  136 + "[llm] Success | src=%s tgt=%s src_text=%s response=%s latency=%.1fms",
  137 + src,
  138 + tgt,
  139 + text,
  140 + content,
  141 + latency_ms,
  142 + )
  143 + return content
  144 + except Exception as exc:
  145 + latency_ms = (time.time() - start) * 1000
  146 + logger.warning(
  147 + "[llm] Failed | src=%s tgt=%s latency=%.1fms error=%s",
  148 + src,
  149 + tgt,
  150 + latency_ms,
  151 + exc,
  152 + exc_info=True,
  153 + )
  154 + return None
  155 +
  156 + def translate(
  157 + self,
  158 + text: Union[str, Sequence[str]],
  159 + target_lang: str,
  160 + source_lang: Optional[str] = None,
  161 + context: Optional[str] = None,
  162 + prompt: Optional[str] = None,
  163 + ) -> Union[Optional[str], List[Optional[str]]]:
  164 + if isinstance(text, (list, tuple)):
  165 + results: List[Optional[str]] = []
  166 + for item in text:
  167 + if item is None:
  168 + results.append(None)
  169 + continue
  170 + results.append(
  171 + self._translate_single(
  172 + text=str(item),
  173 + target_lang=target_lang,
  174 + source_lang=source_lang,
  175 + context=context,
  176 + prompt=prompt,
  177 + )
  178 + )
  179 + return results
  180 +
  181 + return self._translate_single(
  182 + text=str(text),
  183 + target_lang=target_lang,
  184 + source_lang=source_lang,
  185 + context=context,
  186 + prompt=prompt,
  187 + )
  188 +
  189 +
  190 +LLMTranslatorProvider = LLMTranslationBackend
  191 +
  192 +
  193 +def llm_translate(
  194 + text: Union[str, Sequence[str]],
  195 + target_lang: str,
  196 + *,
  197 + source_lang: Optional[str] = None,
  198 + source_lang_label: Optional[str] = None,
  199 + target_lang_label: Optional[str] = None,
  200 + timeout_sec: Optional[float] = None,
  201 +) -> Union[Optional[str], List[Optional[str]]]:
  202 + del source_lang_label, target_lang_label
  203 + provider = LLMTranslationBackend(timeout_sec=timeout_sec or 30.0)
  204 + return provider.translate(
  205 + text=text,
  206 + target_lang=target_lang,
  207 + source_lang=source_lang,
  208 + context=None,
  209 + )
... ...
translation/backends/qwen_mt.py 0 → 100644
  Show/Hide comments   View file @5e4dc8e
... ... @@ -0,0 +1,260 @@
  1 +"""Qwen-MT translation backend with cache support."""
  2 +
  3 +from __future__ import annotations
  4 +
  5 +import hashlib
  6 +import logging
  7 +import os
  8 +import re
  9 +import time
  10 +from typing import List, Optional, Sequence, Union
  11 +
  12 +import redis
  13 +from openai import OpenAI
  14 +
  15 +from config.env_config import DASHSCOPE_API_KEY, REDIS_CONFIG
  16 +from config.services_config import get_translation_cache_config
  17 +from config.tenant_config_loader import SOURCE_LANG_CODE_MAP
  18 +
  19 +logger = logging.getLogger(__name__)
  20 +
  21 +
  22 +class QwenMTTranslationBackend:
  23 + QWEN_DEFAULT_BASE_URL = "https://dashscope-us.aliyuncs.com/compatible-mode/v1"
  24 + QWEN_MODEL = "qwen-mt-flash"
  25 + SOURCE_LANG_CODE_MAP = SOURCE_LANG_CODE_MAP
  26 +
  27 + def __init__(
  28 + self,
  29 + model: str = "qwen",
  30 + api_key: Optional[str] = None,
  31 + use_cache: bool = True,
  32 + timeout: int = 10,
  33 + glossary_id: Optional[str] = None,
  34 + translation_context: Optional[str] = None,
  35 + ):
  36 + self.model = self._normalize_model(model)
  37 + self.timeout = int(timeout)
  38 + self.use_cache = bool(use_cache)
  39 + self.glossary_id = glossary_id
  40 + self.translation_context = translation_context or "e-commerce product search"
  41 +
  42 + cache_cfg = get_translation_cache_config()
  43 + self.cache_prefix = str(cache_cfg.get("key_prefix", "trans:v2"))
  44 + self.expire_seconds = int(cache_cfg.get("ttl_seconds", 360 * 24 * 3600))
  45 + self.cache_sliding_expiration = bool(cache_cfg.get("sliding_expiration", True))
  46 + self.cache_include_context = bool(cache_cfg.get("key_include_context", True))
  47 + self.cache_include_prompt = bool(cache_cfg.get("key_include_prompt", True))
  48 + self.cache_include_source_lang = bool(cache_cfg.get("key_include_source_lang", True))
  49 +
  50 + self.qwen_model_name = self._resolve_qwen_model_name(model)
  51 + self._api_key = api_key or self._default_api_key(self.model)
  52 + self._qwen_client: Optional[OpenAI] = None
  53 + base_url = os.getenv("DASHSCOPE_BASE_URL") or self.QWEN_DEFAULT_BASE_URL
  54 + if self._api_key:
  55 + try:
  56 + self._qwen_client = OpenAI(api_key=self._api_key, base_url=base_url)
  57 + except Exception as exc:
  58 + logger.warning("Failed to initialize qwen-mt client: %s", exc, exc_info=True)
  59 + else:
  60 + logger.warning("DASHSCOPE_API_KEY not set; qwen-mt translation unavailable")
  61 +
  62 + self.redis_client = None
  63 + if self.use_cache and bool(cache_cfg.get("enabled", True)):
  64 + self.redis_client = self._init_redis_client()
  65 +
  66 + @property
  67 + def supports_batch(self) -> bool:
  68 + return True
  69 +
  70 + @staticmethod
  71 + def _normalize_model(model: str) -> str:
  72 + m = (model or "qwen").strip().lower()
  73 + if m.startswith("qwen"):
  74 + return "qwen-mt"
  75 + raise ValueError(f"Unsupported model: {model}. Supported models: 'qwen', 'qwen-mt', 'qwen-mt-flash'")
  76 +
  77 + @staticmethod
  78 + def _resolve_qwen_model_name(model: str) -> str:
  79 + m = (model or "qwen").strip().lower()
  80 + if m in {"qwen", "qwen-mt"}:
  81 + return "qwen-mt-flash"
  82 + return m
  83 +
  84 + @staticmethod
  85 + def _default_api_key(model: str) -> Optional[str]:
  86 + del model
  87 + return DASHSCOPE_API_KEY or os.getenv("DASHSCOPE_API_KEY")
  88 +
  89 + def _init_redis_client(self):
  90 + try:
  91 + client = redis.Redis(
  92 + host=REDIS_CONFIG.get("host", "localhost"),
  93 + port=REDIS_CONFIG.get("port", 6479),
  94 + password=REDIS_CONFIG.get("password"),
  95 + decode_responses=True,
  96 + socket_timeout=REDIS_CONFIG.get("socket_timeout", 1),
  97 + socket_connect_timeout=REDIS_CONFIG.get("socket_connect_timeout", 1),
  98 + retry_on_timeout=REDIS_CONFIG.get("retry_on_timeout", False),
  99 + health_check_interval=10,
  100 + )
  101 + client.ping()
  102 + return client
  103 + except Exception as exc:
  104 + logger.warning("Failed to initialize translation redis cache: %s", exc)
  105 + return None
  106 +
  107 + def _build_cache_key(
  108 + self,
  109 + text: str,
  110 + target_lang: str,
  111 + source_lang: Optional[str],
  112 + context: Optional[str],
  113 + prompt: Optional[str],
  114 + ) -> str:
  115 + src = (source_lang or "auto").strip().lower() if self.cache_include_source_lang else "-"
  116 + tgt = (target_lang or "").strip().lower()
  117 + ctx = (context or "").strip() if self.cache_include_context else ""
  118 + prm = (prompt or "").strip() if self.cache_include_prompt else ""
  119 + payload = f"model={self.model}\nsrc={src}\ntgt={tgt}\nctx={ctx}\nprm={prm}\ntext={text}"
  120 + digest = hashlib.sha256(payload.encode("utf-8")).hexdigest()
  121 + return f"{self.cache_prefix}:{self.model}:{src}:{tgt}:{digest}"
  122 +
  123 + def translate(
  124 + self,
  125 + text: Union[str, Sequence[str]],
  126 + target_lang: str,
  127 + source_lang: Optional[str] = None,
  128 + context: Optional[str] = None,
  129 + prompt: Optional[str] = None,
  130 + ) -> Union[Optional[str], List[Optional[str]]]:
  131 + if isinstance(text, (list, tuple)):
  132 + results: List[Optional[str]] = []
  133 + for item in text:
  134 + if item is None or not str(item).strip():
  135 + results.append(item) # type: ignore[arg-type]
  136 + continue
  137 + out = self.translate(
  138 + text=str(item),
  139 + target_lang=target_lang,
  140 + source_lang=source_lang,
  141 + context=context,
  142 + prompt=prompt,
  143 + )
  144 + results.append(out)
  145 + return results
  146 +
  147 + if not text or not str(text).strip():
  148 + return text # type: ignore[return-value]
  149 +
  150 + tgt = (target_lang or "").strip().lower()
  151 + src = (source_lang or "").strip().lower() or None
  152 + if tgt == "en" and self._is_english_text(text):
  153 + return text
  154 + if tgt == "zh" and (self._contains_chinese(text) or self._is_pure_number(text)):
  155 + return text
  156 +
  157 + translation_context = context or self.translation_context
  158 + cached = self._get_cached_translation_redis(text, tgt, src, translation_context, prompt)
  159 + if cached is not None:
  160 + return cached
  161 +
  162 + result = self._translate_qwen(text, tgt, src)
  163 +
  164 + if result is not None:
  165 + self._set_cached_translation_redis(text, tgt, result, src, translation_context, prompt)
  166 + return result
  167 +
  168 + def _translate_qwen(
  169 + self,
  170 + text: str,
  171 + target_lang: str,
  172 + source_lang: Optional[str],
  173 + ) -> Optional[str]:
  174 + if not self._qwen_client:
  175 + return None
  176 + tgt_norm = (target_lang or "").strip().lower()
  177 + src_norm = (source_lang or "").strip().lower()
  178 + tgt_qwen = self.SOURCE_LANG_CODE_MAP.get(tgt_norm, tgt_norm.capitalize())
  179 + src_qwen = "auto" if not src_norm or src_norm == "auto" else self.SOURCE_LANG_CODE_MAP.get(src_norm, src_norm.capitalize())
  180 + start = time.time()
  181 + try:
  182 + completion = self._qwen_client.chat.completions.create(
  183 + model=self.qwen_model_name,
  184 + messages=[{"role": "user", "content": text}],
  185 + extra_body={
  186 + "translation_options": {
  187 + "source_lang": src_qwen,
  188 + "target_lang": tgt_qwen,
  189 + }
  190 + },
  191 + timeout=self.timeout,
  192 + )
  193 + content = (completion.choices[0].message.content or "").strip()
  194 + if not content:
  195 + return None
  196 + logger.info("[qwen-mt] Success | src=%s tgt=%s latency=%.1fms", src_qwen, tgt_qwen, (time.time() - start) * 1000)
  197 + return content
  198 + except Exception as exc:
  199 + logger.warning(
  200 + "[qwen-mt] Failed | src=%s tgt=%s latency=%.1fms error=%s",
  201 + src_qwen,
  202 + tgt_qwen,
  203 + (time.time() - start) * 1000,
  204 + exc,
  205 + exc_info=True,
  206 + )
  207 + return None
  208 +
  209 + def _get_cached_translation_redis(
  210 + self,
  211 + text: str,
  212 + target_lang: str,
  213 + source_lang: Optional[str] = None,
  214 + context: Optional[str] = None,
  215 + prompt: Optional[str] = None,
  216 + ) -> Optional[str]:
  217 + if not self.redis_client:
  218 + return None
  219 + key = self._build_cache_key(text, target_lang, source_lang, context, prompt)
  220 + try:
  221 + value = self.redis_client.get(key)
  222 + if value and self.cache_sliding_expiration:
  223 + self.redis_client.expire(key, self.expire_seconds)
  224 + return value
  225 + except Exception as exc:
  226 + logger.warning("Redis get translation cache failed: %s", exc)
  227 + return None
  228 +
  229 + def _set_cached_translation_redis(
  230 + self,
  231 + text: str,
  232 + target_lang: str,
  233 + translation: str,
  234 + source_lang: Optional[str] = None,
  235 + context: Optional[str] = None,
  236 + prompt: Optional[str] = None,
  237 + ) -> None:
  238 + if not self.redis_client:
  239 + return
  240 + key = self._build_cache_key(text, target_lang, source_lang, context, prompt)
  241 + try:
  242 + self.redis_client.setex(key, self.expire_seconds, translation)
  243 + except Exception as exc:
  244 + logger.warning("Redis set translation cache failed: %s", exc)
  245 +
  246 + @staticmethod
  247 + def _contains_chinese(text: str) -> bool:
  248 + return bool(re.search(r"[\u4e00-\u9fff]", text or ""))
  249 +
  250 + @staticmethod
  251 + def _is_english_text(text: str) -> bool:
  252 + stripped = (text or "").strip()
  253 + return bool(stripped) and bool(re.fullmatch(r"[A-Za-z0-9\s\W]+", stripped)) and not QwenMTTranslationBackend._contains_chinese(stripped)
  254 +
  255 + @staticmethod
  256 + def _is_pure_number(text: str) -> bool:
  257 + return bool(re.fullmatch(r"[\d.\-+%/,: ]+", (text or "").strip()))
  258 +
  259 +
  260 +Translator = QwenMTTranslationBackend
... ...
translation/client.py 0 → 100644
  Show/Hide comments   View file @5e4dc8e
... ... @@ -0,0 +1,86 @@
  1 +"""HTTP client for the translation service."""
  2 +
  3 +from __future__ import annotations
  4 +
  5 +import logging
  6 +from typing import List, Optional, Sequence, Union
  7 +
  8 +import requests
  9 +
  10 +from config.services_config import get_translation_config
  11 +
  12 +logger = logging.getLogger(__name__)
  13 +
  14 +
  15 +class TranslationServiceClient:
  16 + """Business-side translation client that talks to the translator service."""
  17 +
  18 + def __init__(
  19 + self,
  20 + *,
  21 + base_url: Optional[str] = None,
  22 + default_model: Optional[str] = None,
  23 + default_scene: Optional[str] = None,
  24 + timeout_sec: Optional[float] = None,
  25 + ) -> None:
  26 + cfg = get_translation_config()
  27 + self.base_url = (base_url or cfg.service_url).rstrip("/")
  28 + self.default_model = cfg.normalize_model_name(default_model or cfg.default_model)
  29 + self.default_scene = (default_scene or cfg.default_scene or "general").strip() or "general"
  30 + self.timeout_sec = float(timeout_sec or cfg.timeout_sec or 10.0)
  31 +
  32 + @property
  33 + def model(self) -> str:
  34 + return self.default_model
  35 +
  36 + @property
  37 + def supports_batch(self) -> bool:
  38 + return True
  39 +
  40 + def translate(
  41 + self,
  42 + text: Union[str, Sequence[str]],
  43 + target_lang: str,
  44 + source_lang: Optional[str] = None,
  45 + context: Optional[str] = None,
  46 + prompt: Optional[str] = None,
  47 + model: Optional[str] = None,
  48 + scene: Optional[str] = None,
  49 + ) -> Union[Optional[str], List[Optional[str]]]:
  50 + if isinstance(text, tuple):
  51 + text = list(text)
  52 + payload = {
  53 + "text": text,
  54 + "target_lang": target_lang,
  55 + "source_lang": source_lang or "auto",
  56 + "model": (model or self.default_model),
  57 + "scene": (scene or context or self.default_scene),
  58 + }
  59 + if prompt:
  60 + payload["prompt"] = prompt
  61 + try:
  62 + response = requests.post(
  63 + f"{self.base_url}/translate",
  64 + json=payload,
  65 + timeout=self.timeout_sec,
  66 + )
  67 + if response.status_code != 200:
  68 + logger.warning(
  69 + "Translation service request failed: status=%s body=%s",
  70 + response.status_code,
  71 + (response.text or "")[:300],
  72 + )
  73 + return self._empty_result_for(text)
  74 + data = response.json()
  75 + return data.get("translated_text")
  76 + except Exception as exc:
  77 + logger.warning("Translation service request error: %s", exc, exc_info=True)
  78 + return self._empty_result_for(text)
  79 +
  80 + @staticmethod
  81 + def _empty_result_for(
  82 + text: Union[str, Sequence[str]],
  83 + ) -> Union[Optional[str], List[Optional[str]]]:
  84 + if isinstance(text, (list, tuple)):
  85 + return [None for _ in text]
  86 + return None
... ...
translation/protocols.py 0 → 100644
  Show/Hide comments   View file @5e4dc8e
... ... @@ -0,0 +1,30 @@
  1 +"""Protocols for translation service backends."""
  2 +
  3 +from __future__ import annotations
  4 +
  5 +from typing import List, Optional, Protocol, Sequence, Union, runtime_checkable
  6 +
  7 +
  8 +TranslateInput = Union[str, Sequence[str]]
  9 +TranslateOutput = Union[Optional[str], List[Optional[str]]]
  10 +
  11 +
  12 +@runtime_checkable
  13 +class TranslationBackendProtocol(Protocol):
  14 + """Shared protocol implemented by translation backends."""
  15 +
  16 + model: str
  17 +
  18 + @property
  19 + def supports_batch(self) -> bool:
  20 + ...
  21 +
  22 + def translate(
  23 + self,
  24 + text: TranslateInput,
  25 + target_lang: str,
  26 + source_lang: Optional[str] = None,
  27 + context: Optional[str] = None,
  28 + prompt: Optional[str] = None,
  29 + ) -> TranslateOutput:
  30 + ...
... ...
translation/service.py 0 → 100644
  Show/Hide comments   View file @5e4dc8e
... ... @@ -0,0 +1,103 @@
  1 +"""Translation service orchestration."""
  2 +
  3 +from __future__ import annotations
  4 +
  5 +import logging
  6 +from typing import Dict, List, Optional
  7 +
  8 +from config.services_config import TranslationServiceConfig, get_translation_config
  9 +from translation.protocols import TranslateInput, TranslateOutput, TranslationBackendProtocol
  10 +
  11 +logger = logging.getLogger(__name__)
  12 +
  13 +
  14 +class TranslationService:
  15 + """Owns translation backends and routes calls by model and scene."""
  16 +
  17 + def __init__(self, config: Optional[TranslationServiceConfig] = None) -> None:
  18 + self.config = config or get_translation_config()
  19 + self._backends: Dict[str, TranslationBackendProtocol] = {}
  20 + self._init_enabled_backends()
  21 +
  22 + def _init_enabled_backends(self) -> None:
  23 + registry = {
  24 + "qwen-mt": self._create_qwen_mt_backend,
  25 + "deepl": self._create_deepl_backend,
  26 + "llm": self._create_llm_backend,
  27 + }
  28 + for name in self.config.enabled_models:
  29 + factory = registry.get(name)
  30 + if factory is None:
  31 + logger.warning("Translation backend '%s' is enabled but not registered", name)
  32 + continue
  33 + self._backends[name] = factory()
  34 +
  35 + if not self._backends:
  36 + raise ValueError("No enabled translation backends found in services.translation.capabilities")
  37 +
  38 + def _create_qwen_mt_backend(self) -> TranslationBackendProtocol:
  39 + from translation.backends.qwen_mt import QwenMTTranslationBackend
  40 +
  41 + cfg = self.config.get_capability_cfg("qwen-mt")
  42 + return QwenMTTranslationBackend(
  43 + model=cfg.get("model") or "qwen-mt-flash",
  44 + api_key=cfg.get("api_key"),
  45 + use_cache=bool(cfg.get("use_cache", True)),
  46 + timeout=int(cfg.get("timeout_sec", 10)),
  47 + glossary_id=cfg.get("glossary_id"),
  48 + translation_context=cfg.get("translation_context"),
  49 + )
  50 +
  51 + def _create_deepl_backend(self) -> TranslationBackendProtocol:
  52 + from translation.backends.deepl import DeepLTranslationBackend
  53 +
  54 + cfg = self.config.get_capability_cfg("deepl")
  55 + return DeepLTranslationBackend(
  56 + api_key=cfg.get("api_key"),
  57 + timeout=float(cfg.get("timeout_sec", 10.0)),
  58 + glossary_id=cfg.get("glossary_id"),
  59 + )
  60 +
  61 + def _create_llm_backend(self) -> TranslationBackendProtocol:
  62 + from translation.backends.llm import LLMTranslationBackend
  63 +
  64 + cfg = self.config.get_capability_cfg("llm")
  65 + return LLMTranslationBackend(
  66 + model=cfg.get("model"),
  67 + timeout_sec=float(cfg.get("timeout_sec", 30.0)),
  68 + base_url=cfg.get("base_url"),
  69 + )
  70 +
  71 + @property
  72 + def available_models(self) -> List[str]:
  73 + return list(self._backends.keys())
  74 +
  75 + def get_backend(self, model: Optional[str] = None) -> TranslationBackendProtocol:
  76 + normalized = self.config.normalize_model_name(model)
  77 + backend = self._backends.get(normalized)
  78 + if backend is None:
  79 + raise ValueError(
  80 + f"Translation model '{normalized}' is not enabled. "
  81 + f"Available models: {', '.join(self.available_models) or 'none'}"
  82 + )
  83 + return backend
  84 +
  85 + def translate(
  86 + self,
  87 + text: TranslateInput,
  88 + target_lang: str,
  89 + source_lang: Optional[str] = None,
  90 + *,
  91 + model: Optional[str] = None,
  92 + scene: Optional[str] = None,
  93 + prompt: Optional[str] = None,
  94 + ) -> TranslateOutput:
  95 + backend = self.get_backend(model)
  96 + active_scene = (scene or self.config.default_scene or "general").strip() or "general"
  97 + return backend.translate(
  98 + text=text,
  99 + target_lang=target_lang,
  100 + source_lang=source_lang,
  101 + context=active_scene,
  102 + prompt=prompt,
  103 + )
... ...