Commit ff32d8945e2d83e4937b40241a902da6a4018820
1 parent
7746376c
rerank
Showing
9 changed files
with
170 additions
and
88 deletions
Show diff stats
api/models.py
| ... | ... | @@ -151,9 +151,23 @@ class SearchRequest(BaseModel): |
| 151 | 151 | min_score: Optional[float] = Field(None, ge=0, description="最小相关性分数阈值") |
| 152 | 152 | highlight: bool = Field(False, description="是否高亮搜索关键词(暂不实现)") |
| 153 | 153 | debug: bool = Field(False, description="是否返回调试信息") |
| 154 | - ai_search: bool = Field( | |
| 154 | + enable_rerank: bool = Field( | |
| 155 | 155 | False, |
| 156 | - description="是否开启 AI 搜索(调用本地重排服务对 ES 结果进行二次排序)" | |
| 156 | + description="是否开启重排(调用外部重排服务对 ES 结果进行二次排序)" | |
| 157 | + ) | |
| 158 | + rerank_query_template: Optional[str] = Field( | |
| 159 | + None, | |
| 160 | + description=( | |
| 161 | + "重排 query 模板(可选)。支持 {query} 占位符。" | |
| 162 | + "不传则使用服务端配置的 rerank_query_template。" | |
| 163 | + ), | |
| 164 | + ) | |
| 165 | + rerank_doc_template: Optional[str] = Field( | |
| 166 | + None, | |
| 167 | + description=( | |
| 168 | + "重排 doc 模板(可选)。支持 {title} {brief} {vendor} {description} {category_path} 占位符。" | |
| 169 | + "不传则使用服务端配置的 rerank_doc_template。" | |
| 170 | + ), | |
| 157 | 171 | ) |
| 158 | 172 | |
| 159 | 173 | # SKU筛选参数 | ... | ... |
api/routes/search.py
| ... | ... | @@ -84,7 +84,9 @@ async def search(request: SearchRequest, http_request: Request): |
| 84 | 84 | f"min_score: {request.min_score} | " |
| 85 | 85 | f"language: {request.language} | " |
| 86 | 86 | f"debug: {request.debug} | " |
| 87 | - f"ai_search: {request.ai_search} | " | |
| 87 | + f"enable_rerank: {request.enable_rerank} | " | |
| 88 | + f"rerank_query_template: {request.rerank_query_template} | " | |
| 89 | + f"rerank_doc_template: {request.rerank_doc_template} | " | |
| 88 | 90 | f"sku_filter_dimension: {request.sku_filter_dimension} | " |
| 89 | 91 | f"filters: {request.filters} | " |
| 90 | 92 | f"range_filters: {request.range_filters} | " |
| ... | ... | @@ -112,7 +114,9 @@ async def search(request: SearchRequest, http_request: Request): |
| 112 | 114 | debug=request.debug, |
| 113 | 115 | language=request.language, |
| 114 | 116 | sku_filter_dimension=request.sku_filter_dimension, |
| 115 | - ai_search=request.ai_search, | |
| 117 | + enable_rerank=request.enable_rerank, | |
| 118 | + rerank_query_template=request.rerank_query_template, | |
| 119 | + rerank_doc_template=request.rerank_doc_template, | |
| 116 | 120 | ) |
| 117 | 121 | |
| 118 | 122 | # Include performance summary in response | ... | ... |
config/config.yaml
| ... | ... | @@ -133,14 +133,19 @@ function_score: |
| 133 | 133 | boost_mode: "multiply" |
| 134 | 134 | functions: [] |
| 135 | 135 | |
| 136 | -# 重排配置(唯一实现:外部 BGE 重排服务,由请求参数 ai_search 控制是否执行) | |
| 137 | -# ai_search 且 from+size<=rerank_window 时:从 ES 取前 rerank_window 条、重排后再按 from/size 分页 | |
| 136 | +# 重排配置(唯一实现:外部 BGE 重排服务,由请求参数 enable_rerank 控制是否执行) | |
| 137 | +# enable_rerank 且 from+size<=rerank_window 时:从 ES 取前 rerank_window 条、重排后再按 from/size 分页 | |
| 138 | 138 | rerank: |
| 139 | 139 | rerank_window: 1000 |
| 140 | 140 | # service_url: "http://127.0.0.1:6007/rerank" # 可选,不填则用默认端口 6007 |
| 141 | 141 | timeout_sec: 15.0 # 文档多时重排耗时长,可按需调大 |
| 142 | 142 | weight_es: 0.4 |
| 143 | 143 | weight_ai: 0.6 |
| 144 | + # 模板:用于将搜索请求/文档字段组装成重排服务输入 | |
| 145 | + # - rerank_query_template:支持 {query} | |
| 146 | + # - rerank_doc_template:支持 {title} {brief} {vendor} {description} {category_path} | |
| 147 | + rerank_query_template: "{query}" | |
| 148 | + rerank_doc_template: "{title}" | |
| 144 | 149 | |
| 145 | 150 | # SPU配置(已启用,使用嵌套skus) |
| 146 | 151 | spu_config: | ... | ... |
config/config_loader.py
| ... | ... | @@ -88,14 +88,19 @@ class RankingConfig: |
| 88 | 88 | |
| 89 | 89 | @dataclass |
| 90 | 90 | class RerankConfig: |
| 91 | - """重排配置(唯一实现:调用外部 BGE 重排服务,由请求参数 ai_search 控制是否执行)""" | |
| 92 | - # 重排窗口:ai_search 且 from+size<=rerank_window 时,从 ES 取前 rerank_window 条重排后再分页 | |
| 91 | + """重排配置(唯一实现:调用外部 BGE 重排服务,由请求参数 enable_rerank 控制是否执行)""" | |
| 92 | + # 重排窗口:enable_rerank 且 from+size<=rerank_window 时,从 ES 取前 rerank_window 条重排后再分页 | |
| 93 | 93 | rerank_window: int = 1000 |
| 94 | 94 | # 可选:重排服务 URL,为空时使用 reranker 模块默认端口 6007 |
| 95 | 95 | service_url: Optional[str] = None |
| 96 | 96 | timeout_sec: float = 15.0 |
| 97 | 97 | weight_es: float = 0.4 |
| 98 | 98 | weight_ai: float = 0.6 |
| 99 | + # 模板:用于将搜索请求/文档字段组装成重排服务输入 | |
| 100 | + # - rerank_query_template:支持 {query} | |
| 101 | + # - rerank_doc_template:支持 {title} {brief} {vendor} {description} {category_path} | |
| 102 | + rerank_query_template: str = "{query}" | |
| 103 | + rerank_doc_template: str = "{title}" | |
| 99 | 104 | |
| 100 | 105 | |
| 101 | 106 | @dataclass |
| ... | ... | @@ -267,7 +272,7 @@ class ConfigLoader: |
| 267 | 272 | functions=fs_data.get("functions") or [] |
| 268 | 273 | ) |
| 269 | 274 | |
| 270 | - # Parse Rerank configuration(唯一实现:外部重排服务,由 ai_search 控制) | |
| 275 | + # Parse Rerank configuration(唯一实现:外部重排服务,由 enable_rerank 控制) | |
| 271 | 276 | rerank_data = config_data.get("rerank", {}) |
| 272 | 277 | rerank = RerankConfig( |
| 273 | 278 | rerank_window=int(rerank_data.get("rerank_window", 1000)), |
| ... | ... | @@ -275,6 +280,8 @@ class ConfigLoader: |
| 275 | 280 | timeout_sec=float(rerank_data.get("timeout_sec", 15.0)), |
| 276 | 281 | weight_es=float(rerank_data.get("weight_es", 0.4)), |
| 277 | 282 | weight_ai=float(rerank_data.get("weight_ai", 0.6)), |
| 283 | + rerank_query_template=str(rerank_data.get("rerank_query_template") or "{query}"), | |
| 284 | + rerank_doc_template=str(rerank_data.get("rerank_doc_template") or "{title}"), | |
| 278 | 285 | ) |
| 279 | 286 | |
| 280 | 287 | # Parse SPU config |
| ... | ... | @@ -410,6 +417,8 @@ class ConfigLoader: |
| 410 | 417 | "timeout_sec": config.rerank.timeout_sec, |
| 411 | 418 | "weight_es": config.rerank.weight_es, |
| 412 | 419 | "weight_ai": config.rerank.weight_ai, |
| 420 | + "rerank_query_template": config.rerank.rerank_query_template, | |
| 421 | + "rerank_doc_template": config.rerank.rerank_doc_template, | |
| 413 | 422 | }, |
| 414 | 423 | "spu_config": { |
| 415 | 424 | "enabled": config.spu_config.enabled, | ... | ... |
docs/搜索API对接指南.md
| ... | ... | @@ -167,7 +167,9 @@ curl -X POST "http://120.76.41.98:6002/search/" \ |
| 167 | 167 | "min_score": 0.0, |
| 168 | 168 | "sku_filter_dimension": ["string"], |
| 169 | 169 | "debug": false, |
| 170 | - "ai_search": false, | |
| 170 | + "enable_rerank": false, | |
| 171 | + "rerank_query_template": "{query}", | |
| 172 | + "rerank_doc_template": "{title}", | |
| 171 | 173 | "user_id": "string", |
| 172 | 174 | "session_id": "string" |
| 173 | 175 | } |
| ... | ... | @@ -189,7 +191,9 @@ curl -X POST "http://120.76.41.98:6002/search/" \ |
| 189 | 191 | | `min_score` | float | N | null | 最小相关性分数阈值 | |
| 190 | 192 | | `sku_filter_dimension` | array[string] | N | null | 子SKU筛选维度列表(见[SKU筛选维度](#35-sku筛选维度)) | |
| 191 | 193 | | `debug` | boolean | N | false | 是否返回调试信息 | |
| 192 | -| `ai_search` | boolean | N | false | 是否开启 AI 搜索(调用本地重排服务对 ES 结果进行二次排序) | | |
| 194 | +| `enable_rerank` | boolean | N | false | 是否开启重排(调用外部重排服务对 ES 结果进行二次排序)。开启后若 `from+size<=rerank_window` 才会触发重排 | | |
| 195 | +| `rerank_query_template` | string | N | null | 重排 query 模板(可选)。支持 `{query}` 占位符;不传则使用服务端配置 | | |
| 196 | +| `rerank_doc_template` | string | N | null | 重排 doc 模板(可选)。支持 `{title} {brief} {vendor} {description} {category_path}`;不传则使用服务端配置 | | |
| 193 | 197 | | `user_id` | string | N | null | 用户ID(用于个性化,预留) | |
| 194 | 198 | | `session_id` | string | N | null | 会话ID(用于分析,预留) | |
| 195 | 199 | ... | ... |
query/test_translation.py
| 1 | 1 | #!/usr/bin/env python3 |
| 2 | 2 | """ |
| 3 | -翻译功能测试脚本。 | |
| 3 | +Translation function test script. | |
| 4 | 4 | |
| 5 | -测试内容: | |
| 6 | -1. 翻译提示词配置加载 | |
| 7 | -2. 同步翻译(索引场景) | |
| 8 | -3. 异步翻译(查询场景) | |
| 9 | -4. 不同提示词的使用 | |
| 10 | -5. 缓存功能 | |
| 11 | -6. DeepL Context参数使用 | |
| 5 | +Test content: | |
| 6 | +1. Translation prompt configuration loading | |
| 7 | +2. Synchronous translation (indexing scenario) | |
| 8 | +3. Asynchronous translation (query scenario) | |
| 9 | +4. Usage of different prompts | |
| 10 | +5. Cache functionality | |
| 11 | +6. DeepL Context parameter usage | |
| 12 | 12 | """ |
| 13 | 13 | |
| 14 | 14 | import sys |
| ... | ... | @@ -31,37 +31,37 @@ logger = logging.getLogger(__name__) |
| 31 | 31 | |
| 32 | 32 | |
| 33 | 33 | def test_config_loading(): |
| 34 | - """测试配置加载""" | |
| 34 | + """Test configuration loading""" | |
| 35 | 35 | print("\n" + "="*60) |
| 36 | - print("测试1: 配置加载") | |
| 36 | + print("Test 1: Configuration loading") | |
| 37 | 37 | print("="*60) |
| 38 | 38 | |
| 39 | 39 | try: |
| 40 | 40 | config_loader = ConfigLoader() |
| 41 | 41 | config = config_loader.load_config() |
| 42 | 42 | |
| 43 | - print(f"✓ 配置加载成功") | |
| 44 | - print(f" 翻译服务: {config.query_config.translation_service}") | |
| 45 | - print(f" 翻译提示词配置:") | |
| 43 | + print(f"✓ Configuration loaded successfully") | |
| 44 | + print(f" Translation service: {config.query_config.translation_service}") | |
| 45 | + print(f" Translation prompt configuration:") | |
| 46 | 46 | for key, value in config.query_config.translation_prompts.items(): |
| 47 | 47 | print(f" {key}: {value[:60]}..." if len(value) > 60 else f" {key}: {value}") |
| 48 | 48 | |
| 49 | 49 | return config |
| 50 | 50 | except Exception as e: |
| 51 | - print(f"✗ 配置加载失败: {e}") | |
| 51 | + print(f"✗ Configuration loading failed: {e}") | |
| 52 | 52 | import traceback |
| 53 | 53 | traceback.print_exc() |
| 54 | 54 | return None |
| 55 | 55 | |
| 56 | 56 | |
| 57 | 57 | def test_translator_sync(config): |
| 58 | - """测试同步翻译(索引场景)""" | |
| 58 | + """Test synchronous translation (indexing scenario)""" | |
| 59 | 59 | print("\n" + "="*60) |
| 60 | - print("测试2: 同步翻译(索引场景)") | |
| 60 | + print("Test 2: Synchronous translation (indexing scenario)") | |
| 61 | 61 | print("="*60) |
| 62 | 62 | |
| 63 | 63 | if not config: |
| 64 | - print("✗ 跳过:配置未加载") | |
| 64 | + print("✗ Skipped: Configuration not loaded") | |
| 65 | 65 | return None |
| 66 | 66 | |
| 67 | 67 | try: |
| ... | ... | @@ -90,10 +90,10 @@ def test_translator_sync(config): |
| 90 | 90 | else: |
| 91 | 91 | prompt = config.query_config.translation_prompts.get('default_en') |
| 92 | 92 | |
| 93 | - print(f"\n翻译测试:") | |
| 94 | - print(f" 原文 ({source_lang}): {text}") | |
| 95 | - print(f" 目标语言: {target_lang}") | |
| 96 | - print(f" 提示词: {prompt[:50] if prompt else 'None'}...") | |
| 93 | + print(f"\nTranslation test:") | |
| 94 | + print(f" Original text ({source_lang}): {text}") | |
| 95 | + print(f" Target language: {target_lang}") | |
| 96 | + print(f" Prompt: {prompt[:50] if prompt else 'None'}...") | |
| 97 | 97 | |
| 98 | 98 | result = translator.translate( |
| 99 | 99 | text, |
| ... | ... | @@ -103,28 +103,28 @@ def test_translator_sync(config): |
| 103 | 103 | ) |
| 104 | 104 | |
| 105 | 105 | if result: |
| 106 | - print(f" 结果: {result}") | |
| 107 | - print(f" ✓ 翻译成功") | |
| 106 | + print(f" Result: {result}") | |
| 107 | + print(f" ✓ Translation successful") | |
| 108 | 108 | else: |
| 109 | - print(f" ⚠ 翻译返回None(可能是mock模式或无API key)") | |
| 109 | + print(f" ⚠ Translation returned None (possibly mock mode or no API key)") | |
| 110 | 110 | |
| 111 | 111 | return translator |
| 112 | 112 | |
| 113 | 113 | except Exception as e: |
| 114 | - print(f"✗ 同步翻译测试失败: {e}") | |
| 114 | + print(f"✗ Synchronous translation test failed: {e}") | |
| 115 | 115 | import traceback |
| 116 | 116 | traceback.print_exc() |
| 117 | 117 | return None |
| 118 | 118 | |
| 119 | 119 | |
| 120 | 120 | def test_translator_async(config, translator): |
| 121 | - """测试异步翻译(查询场景)""" | |
| 121 | + """Test asynchronous translation (query scenario)""" | |
| 122 | 122 | print("\n" + "="*60) |
| 123 | - print("测试3: 异步翻译(查询场景)") | |
| 123 | + print("Test 3: Asynchronous translation (query scenario)") | |
| 124 | 124 | print("="*60) |
| 125 | 125 | |
| 126 | 126 | if not config or not translator: |
| 127 | - print("✗ 跳过:配置或翻译器未初始化") | |
| 127 | + print("✗ Skipped: Configuration or translator not initialized") | |
| 128 | 128 | return |
| 129 | 129 | |
| 130 | 130 | try: |
| ... | ... | @@ -134,9 +134,9 @@ def test_translator_async(config, translator): |
| 134 | 134 | |
| 135 | 135 | query_prompt = config.query_config.translation_prompts.get('query_zh') |
| 136 | 136 | |
| 137 | - print(f"查询文本: {query_text}") | |
| 138 | - print(f"目标语言: {target_langs}") | |
| 139 | - print(f"提示词: {query_prompt}") | |
| 137 | + print(f"Query text: {query_text}") | |
| 138 | + print(f"Target languages: {target_langs}") | |
| 139 | + print(f"Prompt: {query_prompt}") | |
| 140 | 140 | |
| 141 | 141 | # 异步模式(立即返回,后台翻译) |
| 142 | 142 | results = translator.translate_multi( |
| ... | ... | @@ -148,15 +148,15 @@ def test_translator_async(config, translator): |
| 148 | 148 | prompt=query_prompt |
| 149 | 149 | ) |
| 150 | 150 | |
| 151 | - print(f"\n异步翻译结果:") | |
| 151 | + print(f"\nAsynchronous translation results:") | |
| 152 | 152 | for lang, translation in results.items(): |
| 153 | 153 | if translation: |
| 154 | - print(f" {lang}: {translation} (缓存命中)") | |
| 154 | + print(f" {lang}: {translation} (cache hit)") | |
| 155 | 155 | else: |
| 156 | - print(f" {lang}: None (后台翻译中...)") | |
| 156 | + print(f" {lang}: None (translating in background...)") | |
| 157 | 157 | |
| 158 | 158 | # 同步模式(等待完成) |
| 159 | - print(f"\n同步翻译(等待完成):") | |
| 159 | + print(f"\nSynchronous translation (waiting for completion):") | |
| 160 | 160 | results_sync = translator.translate_multi( |
| 161 | 161 | query_text, |
| 162 | 162 | target_langs, |
| ... | ... | @@ -170,7 +170,7 @@ def test_translator_async(config, translator): |
| 170 | 170 | print(f" {lang}: {translation}") |
| 171 | 171 | |
| 172 | 172 | except Exception as e: |
| 173 | - print(f"✗ 异步翻译测试失败: {e}") | |
| 173 | + print(f"✗ Asynchronous translation test failed: {e}") | |
| 174 | 174 | import traceback |
| 175 | 175 | traceback.print_exc() |
| 176 | 176 | |
| ... | ... | @@ -178,7 +178,7 @@ def test_translator_async(config, translator): |
| 178 | 178 | def test_cache(): |
| 179 | 179 | """测试缓存功能""" |
| 180 | 180 | print("\n" + "="*60) |
| 181 | - print("测试4: 缓存功能") | |
| 181 | + print("Test 4: Cache functionality") | |
| 182 | 182 | print("="*60) |
| 183 | 183 | |
| 184 | 184 | try: |
| ... | ... | @@ -195,29 +195,29 @@ def test_cache(): |
| 195 | 195 | source_lang = "zh" |
| 196 | 196 | prompt = config.query_config.translation_prompts.get('default_zh') |
| 197 | 197 | |
| 198 | - print(f"第一次翻译(应该调用API或返回mock):") | |
| 198 | + print(f"First translation (should call API or return mock):") | |
| 199 | 199 | result1 = translator.translate(test_text, target_lang, source_lang, prompt=prompt) |
| 200 | - print(f" 结果: {result1}") | |
| 201 | - | |
| 202 | - print(f"\n第二次翻译(应该使用缓存):") | |
| 200 | + print(f" Result: {result1}") | |
| 201 | + | |
| 202 | + print(f"\nSecond translation (should use cache):") | |
| 203 | 203 | result2 = translator.translate(test_text, target_lang, source_lang, prompt=prompt) |
| 204 | - print(f" 结果: {result2}") | |
| 205 | - | |
| 204 | + print(f" Result: {result2}") | |
| 205 | + | |
| 206 | 206 | if result1 == result2: |
| 207 | - print(f" ✓ 缓存功能正常") | |
| 207 | + print(f" ✓ Cache functionality working properly") | |
| 208 | 208 | else: |
| 209 | - print(f" ⚠ 缓存可能有问题") | |
| 209 | + print(f" ⚠ Cache might have issues") | |
| 210 | 210 | |
| 211 | 211 | except Exception as e: |
| 212 | - print(f"✗ 缓存测试失败: {e}") | |
| 212 | + print(f"✗ Cache test failed: {e}") | |
| 213 | 213 | import traceback |
| 214 | 214 | traceback.print_exc() |
| 215 | 215 | |
| 216 | 216 | |
| 217 | 217 | def test_context_parameter(): |
| 218 | - """测试DeepL Context参数使用""" | |
| 218 | + """Test DeepL Context parameter usage""" | |
| 219 | 219 | print("\n" + "="*60) |
| 220 | - print("测试5: DeepL Context参数") | |
| 220 | + print("Test 5: DeepL Context parameter") | |
| 221 | 221 | print("="*60) |
| 222 | 222 | |
| 223 | 223 | try: |
| ... | ... | @@ -233,8 +233,8 @@ def test_context_parameter(): |
| 233 | 233 | text = "手机" |
| 234 | 234 | prompt = config.query_config.translation_prompts.get('query_zh') |
| 235 | 235 | |
| 236 | - print(f"测试文本: {text}") | |
| 237 | - print(f"提示词(作为context): {prompt}") | |
| 236 | + print(f"Test text: {text}") | |
| 237 | + print(f"Prompt (as context): {prompt}") | |
| 238 | 238 | |
| 239 | 239 | # 带context的翻译 |
| 240 | 240 | result_with_context = translator.translate( |
| ... | ... | @@ -243,7 +243,7 @@ def test_context_parameter(): |
| 243 | 243 | source_lang='zh', |
| 244 | 244 | prompt=prompt |
| 245 | 245 | ) |
| 246 | - print(f"\n带context翻译结果: {result_with_context}") | |
| 246 | + print(f"\nTranslation result with context: {result_with_context}") | |
| 247 | 247 | |
| 248 | 248 | # 不带context的翻译 |
| 249 | 249 | result_without_context = translator.translate( |
| ... | ... | @@ -252,21 +252,21 @@ def test_context_parameter(): |
| 252 | 252 | source_lang='zh', |
| 253 | 253 | prompt=None |
| 254 | 254 | ) |
| 255 | - print(f"不带context翻译结果: {result_without_context}") | |
| 255 | + print(f"Translation result without context: {result_without_context}") | |
| 256 | 256 | |
| 257 | - print(f"\n✓ Context参数测试完成") | |
| 258 | - print(f" 注意:根据DeepL API,context参数影响翻译但不参与翻译本身") | |
| 257 | + print(f"\n✓ Context parameter test completed") | |
| 258 | + print(f" Note: According to DeepL API, context parameter affects translation but does not participate in translation itself") | |
| 259 | 259 | |
| 260 | 260 | except Exception as e: |
| 261 | - print(f"✗ Context参数测试失败: {e}") | |
| 261 | + print(f"✗ Context parameter test failed: {e}") | |
| 262 | 262 | import traceback |
| 263 | 263 | traceback.print_exc() |
| 264 | 264 | |
| 265 | 265 | |
| 266 | 266 | def main(): |
| 267 | - """主测试函数""" | |
| 267 | + """Main test function""" | |
| 268 | 268 | print("="*60) |
| 269 | - print("翻译功能测试") | |
| 269 | + print("Translation function test") | |
| 270 | 270 | print("="*60) |
| 271 | 271 | |
| 272 | 272 | # 测试1: 配置加载 |
| ... | ... | @@ -285,7 +285,7 @@ def main(): |
| 285 | 285 | test_context_parameter() |
| 286 | 286 | |
| 287 | 287 | print("\n" + "="*60) |
| 288 | - print("测试完成") | |
| 288 | + print("Test completed") | |
| 289 | 289 | print("="*60) |
| 290 | 290 | |
| 291 | 291 | ... | ... |
reranker/bge_reranker.py
| ... | ... | @@ -112,6 +112,17 @@ class BGEReranker: |
| 112 | 112 | total_docs = len(docs) |
| 113 | 113 | output_scores: List[float] = [0.0] * total_docs |
| 114 | 114 | |
| 115 | + # Log request summary (query + first 3 docs preview) | |
| 116 | + preview_docs: List[str] = [] | |
| 117 | + for d in docs[:3]: | |
| 118 | + preview_docs.append("" if d is None else str(d)) | |
| 119 | + logger.info( | |
| 120 | + "[BGE_RERANKER] Request | query=%r | docs=%d | docs_preview=%s", | |
| 121 | + query, | |
| 122 | + total_docs, | |
| 123 | + preview_docs, | |
| 124 | + ) | |
| 125 | + | |
| 115 | 126 | indexed_docs: List[Tuple[int, str]] = [] |
| 116 | 127 | for i, doc in enumerate(docs): |
| 117 | 128 | if doc is None: |
| ... | ... | @@ -158,6 +169,15 @@ class BGEReranker: |
| 158 | 169 | for (orig_idx, _text), unique_idx in zip(indexed_docs, position_to_unique): |
| 159 | 170 | output_scores[orig_idx] = float(unique_scores[unique_idx]) |
| 160 | 171 | |
| 172 | + # Log per-doc scores (aligned to original docs order) | |
| 173 | + try: | |
| 174 | + lines = [] | |
| 175 | + for i, d in enumerate(docs[:100]): | |
| 176 | + lines.append(f"{output_scores[i]},{'' if d is None else str(d)}") | |
| 177 | + logger.info("[BGE_RERANKER] query:%s Scores (score,doc):\n%s", query, "\n".join(lines)) | |
| 178 | + except Exception: | |
| 179 | + pass | |
| 180 | + | |
| 161 | 181 | elapsed_ms = (time.time() - start_ts) * 1000.0 |
| 162 | 182 | dedup_ratio = 0.0 |
| 163 | 183 | if indexed_docs: | ... | ... |
search/rerank_client.py
| ... | ... | @@ -22,12 +22,13 @@ DEFAULT_TIMEOUT_SEC = 15.0 |
| 22 | 22 | def build_docs_from_hits( |
| 23 | 23 | es_hits: List[Dict[str, Any]], |
| 24 | 24 | language: str = "zh", |
| 25 | + doc_template: str = "{title}", | |
| 25 | 26 | ) -> List[str]: |
| 26 | 27 | """ |
| 27 | 28 | 从 ES 命中结果构造重排服务所需的文档文本列表(与 hits 一一对应)。 |
| 28 | 29 | |
| 29 | - 文本由 title、brief、description、vendor、category_path 等多语言字段拼接, | |
| 30 | - 按 language 优先选取对应语言;若无内容则用 spu_id 兜底。 | |
| 30 | + 使用 doc_template 将文档字段组装为重排服务输入。 | |
| 31 | + 支持占位符:{title} {brief} {vendor} {description} {category_path} | |
| 31 | 32 | |
| 32 | 33 | Args: |
| 33 | 34 | es_hits: ES 返回的 hits 列表,每项含 _source |
| ... | ... | @@ -47,16 +48,29 @@ def build_docs_from_hits( |
| 47 | 48 | return str(obj.get(lang) or obj.get("zh") or obj.get("en") or "").strip() |
| 48 | 49 | return str(obj).strip() |
| 49 | 50 | |
| 51 | + class _SafeDict(dict): | |
| 52 | + def __missing__(self, key: str) -> str: | |
| 53 | + return "" | |
| 54 | + | |
| 50 | 55 | docs: List[str] = [] |
| 56 | + only_title = "{title}" == doc_template | |
| 57 | + need_brief = "{brief}" in doc_template | |
| 58 | + need_vendor = "{vendor}" in doc_template | |
| 59 | + need_description = "{description}" in doc_template | |
| 60 | + need_category_path = "{category_path}" in doc_template | |
| 51 | 61 | for hit in es_hits: |
| 52 | 62 | src = hit.get("_source") or {} |
| 53 | - parts: List[str] = [] | |
| 54 | - for key in ("title", "brief", "description", "vendor", "category_path"): | |
| 55 | - parts.append(pick_lang_text(src.get(key))) | |
| 56 | - text = " ".join(p for p in parts if p).strip() | |
| 57 | - if not text: | |
| 58 | - text = str(src.get("spu_id", "")) | |
| 59 | - docs.append(text) | |
| 63 | + if only_title: | |
| 64 | + docs.append(pick_lang_text(src.get("title"))) | |
| 65 | + else: | |
| 66 | + values = _SafeDict( | |
| 67 | + title=pick_lang_text(src.get("title")), | |
| 68 | + brief=pick_lang_text(src.get("brief")) if need_brief else "", | |
| 69 | + vendor=pick_lang_text(src.get("vendor")) if need_vendor else "", | |
| 70 | + description=pick_lang_text(src.get("description")) if need_description else "", | |
| 71 | + category_path=pick_lang_text(src.get("category_path")) if need_category_path else "", | |
| 72 | + ) | |
| 73 | + docs.append(str(doc_template).format_map(values)) | |
| 60 | 74 | return docs |
| 61 | 75 | |
| 62 | 76 | |
| ... | ... | @@ -188,6 +202,8 @@ def run_rerank( |
| 188 | 202 | timeout_sec: float = DEFAULT_TIMEOUT_SEC, |
| 189 | 203 | weight_es: float = DEFAULT_WEIGHT_ES, |
| 190 | 204 | weight_ai: float = DEFAULT_WEIGHT_AI, |
| 205 | + rerank_query_template: str = "{query}", | |
| 206 | + rerank_doc_template: str = "{title}", | |
| 191 | 207 | ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], List[Dict[str, Any]]]: |
| 192 | 208 | """ |
| 193 | 209 | 完整重排流程:从 es_response 取 hits -> 构造 docs -> 调服务 -> 融合分数并重排 -> 更新 max_score。 |
| ... | ... | @@ -222,8 +238,10 @@ def run_rerank( |
| 222 | 238 | if not hits: |
| 223 | 239 | return es_response, None, [] |
| 224 | 240 | |
| 225 | - docs = build_docs_from_hits(hits, language=language) | |
| 226 | - scores, meta = call_rerank_service(query, docs, url, timeout_sec=timeout_sec) | |
| 241 | + # Apply query template (supports {query}) | |
| 242 | + query_text = str(rerank_query_template).format_map({"query": query}) | |
| 243 | + docs = build_docs_from_hits(hits, language=language, doc_template=rerank_doc_template) | |
| 244 | + scores, meta = call_rerank_service(query_text, docs, url, timeout_sec=timeout_sec) | |
| 227 | 245 | |
| 228 | 246 | if scores is None or len(scores) != len(hits): |
| 229 | 247 | return es_response, None, [] | ... | ... |
search/searcher.py
| ... | ... | @@ -135,7 +135,9 @@ class Searcher: |
| 135 | 135 | debug: bool = False, |
| 136 | 136 | language: str = "en", |
| 137 | 137 | sku_filter_dimension: Optional[List[str]] = None, |
| 138 | - ai_search: bool = False, | |
| 138 | + enable_rerank: bool = False, | |
| 139 | + rerank_query_template: Optional[str] = None, | |
| 140 | + rerank_doc_template: Optional[str] = None, | |
| 139 | 141 | ) -> SearchResult: |
| 140 | 142 | """ |
| 141 | 143 | Execute search query (外部友好格式). |
| ... | ... | @@ -167,11 +169,11 @@ class Searcher: |
| 167 | 169 | index_langs = tenant_cfg.get("index_languages") or [] |
| 168 | 170 | enable_translation = len(index_langs) > 0 |
| 169 | 171 | enable_embedding = self.config.query_config.enable_text_embedding |
| 170 | - # 重排仅由请求参数 ai_search 控制,唯一实现为调用外部 BGE 重排服务 | |
| 171 | - enable_rerank = bool(ai_search) | |
| 172 | + # 重排仅由请求参数 enable_rerank 控制,唯一实现为调用外部 BGE 重排服务 | |
| 173 | + do_rerank = bool(enable_rerank) | |
| 172 | 174 | rerank_window = self.config.rerank.rerank_window or 1000 |
| 173 | 175 | # 若开启重排且请求范围在窗口内:从 ES 取前 rerank_window 条、重排后再按 from/size 分页;否则不重排,按原 from/size 查 ES |
| 174 | - in_rerank_window = enable_rerank and (from_ + size) <= rerank_window | |
| 176 | + in_rerank_window = do_rerank and (from_ + size) <= rerank_window | |
| 175 | 177 | es_fetch_from = 0 if in_rerank_window else from_ |
| 176 | 178 | es_fetch_size = rerank_window if in_rerank_window else size |
| 177 | 179 | |
| ... | ... | @@ -180,7 +182,7 @@ class Searcher: |
| 180 | 182 | |
| 181 | 183 | context.logger.info( |
| 182 | 184 | f"开始搜索请求 | 查询: '{query}' | 参数: size={size}, from_={from_}, " |
| 183 | - f"enable_rerank={enable_rerank}, in_rerank_window={in_rerank_window}, es_fetch=({es_fetch_from},{es_fetch_size}) | " | |
| 185 | + f"enable_rerank={do_rerank}, in_rerank_window={in_rerank_window}, es_fetch=({es_fetch_from},{es_fetch_size}) | " | |
| 184 | 186 | f"enable_translation={enable_translation}, enable_embedding={enable_embedding}, min_score={min_score}", |
| 185 | 187 | extra={'reqid': context.reqid, 'uid': context.uid} |
| 186 | 188 | ) |
| ... | ... | @@ -192,12 +194,14 @@ class Searcher: |
| 192 | 194 | 'es_fetch_from': es_fetch_from, |
| 193 | 195 | 'es_fetch_size': es_fetch_size, |
| 194 | 196 | 'in_rerank_window': in_rerank_window, |
| 197 | + 'rerank_query_template': rerank_query_template, | |
| 198 | + 'rerank_doc_template': rerank_doc_template, | |
| 195 | 199 | 'filters': filters, |
| 196 | 200 | 'range_filters': range_filters, |
| 197 | 201 | 'facets': facets, |
| 198 | 202 | 'enable_translation': enable_translation, |
| 199 | 203 | 'enable_embedding': enable_embedding, |
| 200 | - 'enable_rerank': enable_rerank, | |
| 204 | + 'enable_rerank': do_rerank, | |
| 201 | 205 | 'min_score': min_score, |
| 202 | 206 | 'sort_by': sort_by, |
| 203 | 207 | 'sort_order': sort_order |
| ... | ... | @@ -206,7 +210,7 @@ class Searcher: |
| 206 | 210 | context.metadata['feature_flags'] = { |
| 207 | 211 | 'translation_enabled': enable_translation, |
| 208 | 212 | 'embedding_enabled': enable_embedding, |
| 209 | - 'rerank_enabled': enable_rerank | |
| 213 | + 'rerank_enabled': do_rerank | |
| 210 | 214 | } |
| 211 | 215 | |
| 212 | 216 | # Step 1: Parse query |
| ... | ... | @@ -374,13 +378,15 @@ class Searcher: |
| 374 | 378 | context.end_stage(RequestContextStage.ELASTICSEARCH_SEARCH) |
| 375 | 379 | |
| 376 | 380 | # Optional Step 4.5: AI reranking(仅当请求范围在重排窗口内时执行) |
| 377 | - if enable_rerank and in_rerank_window: | |
| 381 | + if do_rerank and in_rerank_window: | |
| 378 | 382 | context.start_stage(RequestContextStage.RERANKING) |
| 379 | 383 | try: |
| 380 | 384 | from .rerank_client import run_rerank |
| 381 | 385 | |
| 382 | 386 | rerank_query = parsed_query.original_query if parsed_query else query |
| 383 | 387 | rc = self.config.rerank |
| 388 | + effective_query_template = rerank_query_template or rc.rerank_query_template | |
| 389 | + effective_doc_template = rerank_doc_template or rc.rerank_doc_template | |
| 384 | 390 | es_response, rerank_meta, fused_debug = run_rerank( |
| 385 | 391 | query=rerank_query, |
| 386 | 392 | es_response=es_response, |
| ... | ... | @@ -389,6 +395,8 @@ class Searcher: |
| 389 | 395 | timeout_sec=rc.timeout_sec, |
| 390 | 396 | weight_es=rc.weight_es, |
| 391 | 397 | weight_ai=rc.weight_ai, |
| 398 | + rerank_query_template=effective_query_template, | |
| 399 | + rerank_doc_template=effective_doc_template, | |
| 392 | 400 | ) |
| 393 | 401 | |
| 394 | 402 | if rerank_meta is not None: | ... | ... |