Added the SKU pre-selection step in search/searcher.py right before

ResultFormatter.format_search_results() runs. What changed: For each final paginated SPU hit, the searcher now scans skus[].option1_value against the query text set built from the original query, normalized query, rewritten query, and translations. If no option1_value matches textually, it falls back to embedding similarity and picks the SKU with the highest inner product against the query embedding. The matched SKU is promoted to the front of the SPU’s skus list. The SPU-level image_url is replaced with that matched SKU’s image_src. I left api/result_formatter.py unchanged because it already preserves the SKU order and reads image_url from _source; updating the page hits in searcher makes the formatter return the desired result automatically. Verification: ReadLints on the edited files: no errors Passed targeted tests: pytest tests/test_search_rerank_window.py -k "translated_query or no_direct_option_match"

Added the SKU pre-selection step in search/searcher.py right before
ResultFormatter.format_search_results() runs. What changed: For each final paginated SPU hit, the searcher now scans skus[].option1_value against the query text set built from the original query, normalized query, rewritten query, and translations. If no option1_value matches textually, it falls back to embedding similarity and picks the SKU with the highest inner product against the query embedding. The matched SKU is promoted to the front of the SPU’s skus list. The SPU-level image_url is replaced with that matched SKU’s image_src. I left api/result_formatter.py unchanged because it already preserves the SKU order and reads image_url from _source; updating the page hits in searcher makes the formatter return the desired result automatically. Verification: ReadLints on the edited files: no errors Passed targeted tests: pytest tests/test_search_rerank_window.py -k "translated_query or no_direct_option_match"
tangwang
1 parent e874eb50
Showing 6 changed files with 550 additions and 9 deletions Show diff stats
]
frontend/index.html
frontend/static/css/style.css
frontend/static/js/app.js
search/searcher.py
tests/test_search_rerank_window.py
@@ -0,0 +1,17 @@
+docs
+# Please enter the commit message for your changes. Lines starting
+# with '#' will be ignored, and an empty message aborts the commit.
+#
+# On branch master
+# Your branch is ahead of 'origin/master' by 5 commits.
+#   (use "git push" to publish your local commits)
+#
+# Changes to be committed:
+#	modified:   config/config.yaml
+#	modified:   docs/TODO.txt
+#	modified:   "docs/\346\220\234\347\264\242API\345\257\271\346\216\245\346\214\207\345\215\227-07-\345\276\256\346\234\215\345\212\241\346\216\245\345\217\243\357\274\210Embedding-Reranker-Translation\357\274\211.md"
+#	modified:   "docs/\347\233\270\345\205\263\346\200\247\346\243\200\347\264\242\344\274\230\345\214\226\350\257\264\346\230\216.md"
+#
+# Changes not staged for commit:
+#	modified:   third-party/clip-as-service (untracked content)
+#
@@ -199,7 +199,7 @@
     </footer>
  
     <script src="/static/js/tenant_facets_config.js?v=1.4"></script>
-    <script src="/static/js/app.js?v=1.0"></script>
+    <script src="/static/js/app.js?v=1.1"></script>
     <script>
         // 自动补全功能（使用后端 /search/suggestions 接口）
         const SUGGEST_API = API_BASE_URL + '/search/suggestions';
@@ -371,9 +371,31 @@ body {
     margin-bottom: 2px;
 }
  
+.product-debug-actions {
+    display: flex;
+    flex-wrap: wrap;
+    align-items: center;
+    gap: 10px;
+    margin-top: 8px;
+}
+
+.product-debug-btn-api-result {
+    font-family: inherit;
+    font-size: 12px;
+    padding: 4px 10px;
+    border: 1px solid #d35400;
+    border-radius: 4px;
+    background: #fff8f3;
+    color: #d35400;
+    cursor: pointer;
+}
+
+.product-debug-btn-api-result:hover {
+    background: #fdebd0;
+}
+
 .product-debug-link {
     display: inline-block;
-    margin-top: 6px;
     font-size: 12px;
     color: #e67e22;
     text-decoration: none;
@@ -383,6 +405,80 @@ body {
     text-decoration: underline;
 }
  
+.api-result-viewer-backdrop {
+    display: none;
+    position: fixed;
+    inset: 0;
+    z-index: 2000;
+    background: rgba(0, 0, 0, 0.45);
+    align-items: center;
+    justify-content: center;
+    padding: 24px;
+    box-sizing: border-box;
+}
+
+.api-result-viewer-panel {
+    background: #fff;
+    border-radius: 8px;
+    max-width: min(920px, 100%);
+    max-height: min(85vh, 100%);
+    display: flex;
+    flex-direction: column;
+    box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2);
+    overflow: hidden;
+}
+
+.api-result-viewer-header {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    gap: 12px;
+    padding: 12px 14px;
+    border-bottom: 1px solid #eee;
+    flex-shrink: 0;
+}
+
+.api-result-viewer-title {
+    font-size: 14px;
+    font-weight: 600;
+    color: #333;
+}
+
+.api-result-viewer-title code {
+    font-size: 12px;
+    font-weight: normal;
+    background: #f4f4f4;
+    padding: 1px 6px;
+    border-radius: 3px;
+}
+
+.api-result-viewer-close {
+    border: none;
+    background: transparent;
+    font-size: 22px;
+    line-height: 1;
+    color: #888;
+    cursor: pointer;
+    padding: 0 4px;
+}
+
+.api-result-viewer-close:hover {
+    color: #333;
+}
+
+.api-result-viewer-pre {
+    margin: 0;
+    padding: 14px;
+    overflow: auto;
+    flex: 1;
+    font-size: 12px;
+    line-height: 1.45;
+    background: #fafafa;
+    border: none;
+    font-family: Menlo, Consolas, "Courier New", monospace;
+    white-space: pre;
+}
+
 .product-card:hover {
     box-shadow: 0 4px 12px rgba(0,0,0,0.1);
     transform: translateY(-2px);
@@ -63,11 +63,81 @@ let state = {
     debug: true  // Always enable debug mode for test frontend
 };
  
+// 弹层：展示 /search/ 返回的 results[] 单条元素（非 ES 原始文档）
+function openApiResultViewer(item) {
+    let backdrop = document.getElementById('apiResultViewerBackdrop');
+    if (!backdrop) {
+        backdrop = document.createElement('div');
+        backdrop.id = 'apiResultViewerBackdrop';
+        backdrop.className = 'api-result-viewer-backdrop';
+        backdrop.innerHTML = `
+            <div class="api-result-viewer-panel" role="dialog" aria-modal="true" aria-labelledby="apiResultViewerTitle">
+                <div class="api-result-viewer-header">
+                    <span id="apiResultViewerTitle" class="api-result-viewer-title">搜索结果项（API <code>results[]</code>）</span>
+                    <button type="button" class="api-result-viewer-close" aria-label="关闭">&times;</button>
+                </div>
+                <pre class="api-result-viewer-pre"></pre>
+            </div>
+        `;
+        document.body.appendChild(backdrop);
+        backdrop.addEventListener('click', (e) => {
+            if (e.target === backdrop) {
+                closeApiResultViewer();
+            }
+        });
+        backdrop.querySelector('.api-result-viewer-close').addEventListener('click', closeApiResultViewer);
+        document.addEventListener('keydown', (e) => {
+            if (e.key === 'Escape') {
+                closeApiResultViewer();
+            }
+        });
+    }
+    const pre = backdrop.querySelector('.api-result-viewer-pre');
+    try {
+        pre.textContent = JSON.stringify(item, null, 2);
+    } catch (err) {
+        pre.textContent = String(item);
+    }
+    backdrop.style.display = 'flex';
+}
+
+function closeApiResultViewer() {
+    const backdrop = document.getElementById('apiResultViewerBackdrop');
+    if (backdrop) {
+        backdrop.style.display = 'none';
+    }
+}
+
+function initProductGridResultViewer() {
+    const grid = document.getElementById('productGrid');
+    if (!grid || grid.dataset.apiResultViewerBound === '1') {
+        return;
+    }
+    grid.dataset.apiResultViewerBound = '1';
+    grid.addEventListener('click', (e) => {
+        const btn = e.target.closest('.product-debug-btn-api-result');
+        if (!btn) {
+            return;
+        }
+        e.preventDefault();
+        const idx = parseInt(btn.getAttribute('data-result-index'), 10);
+        if (Number.isNaN(idx)) {
+            return;
+        }
+        const results = state.lastSearchData && state.lastSearchData.results;
+        if (!results || idx < 0 || idx >= results.length) {
+            return;
+        }
+        openApiResultViewer(results[idx]);
+    });
+}
+
 // Initialize
 function initializeApp() {
     // 初始化租户下拉框和分面面板
     console.log('Initializing app...');
     initTenantSelect();
+    initProductGridResultViewer();
     const searchInput = document.getElementById('searchInput');
     if (searchInput) {
         searchInput.focus();
@@ -350,7 +420,7 @@ function displayResults(data) {
  
     const tenantId = getTenantId();
  
-    data.results.forEach((result) => {
+    data.results.forEach((result, resultIndex) => {
         const product = result;
         const title = product.title || product.name || 'N/A';
         const price = product.min_price || product.price || 'N/A';
@@ -401,9 +471,14 @@ function displayResults(data) {
                     <div class="product-debug-line">Rerank score: ${rerankScore}</div>
                     <div class="product-debug-line">Fused score: ${fusedScore}</div>
                     ${titleLines}
-                    <a class="product-debug-link" href="${rawUrl}" target="_blank" rel="noopener noreferrer">
-                        查看 ES 原始文档
-                    </a>
+                    <div class="product-debug-actions">
+                        <button type="button" class="product-debug-btn-api-result" data-result-index="${resultIndex}">
+                            查看 API 结果项
+                        </button>
+                        <a class="product-debug-link" href="${rawUrl}" target="_blank" rel="noopener noreferrer">
+                            查看 ES 原始文档
+                        </a>
+                    </div>
                 </div>
             `;
         }
@@ -4,12 +4,13 @@ Main Searcher module - executes search queries against Elasticsearch.
 Handles query parsing, ranking, and result formatting.
 """
  
-from typing import Dict, Any, List, Optional, Union
+from typing import Dict, Any, List, Optional, Union, Tuple
 import os
 import time, json
 import logging
 import hashlib
 from string import Formatter
+import numpy as np
  
 from utils.es_client import ESClient
 from query import QueryParser, ParsedQuery
@@ -224,6 +225,231 @@ class Searcher:
             hits_by_id[str(hid)] = hit
         return hits_by_id, int(resp.get("took", 0) or 0)
  
+    @staticmethod
+    def _normalize_sku_match_text(value: Optional[str]) -> str:
+        """Normalize free text for lightweight SKU option matching."""
+        if value is None:
+            return ""
+        return " ".join(str(value).strip().casefold().split())
+
+    def _build_sku_query_texts(self, parsed_query: ParsedQuery) -> List[str]:
+        """Collect original and translated query texts for SKU option matching."""
+        candidates: List[str] = []
+        for text in (
+            getattr(parsed_query, "original_query", None),
+            getattr(parsed_query, "query_normalized", None),
+            getattr(parsed_query, "rewritten_query", None),
+        ):
+            normalized = self._normalize_sku_match_text(text)
+            if normalized:
+                candidates.append(normalized)
+
+        query_text_by_lang = getattr(parsed_query, "query_text_by_lang", {}) or {}
+        if isinstance(query_text_by_lang, dict):
+            for text in query_text_by_lang.values():
+                normalized = self._normalize_sku_match_text(text)
+                if normalized:
+                    candidates.append(normalized)
+
+        translations = getattr(parsed_query, "translations", {}) or {}
+        if isinstance(translations, dict):
+            for text in translations.values():
+                normalized = self._normalize_sku_match_text(text)
+                if normalized:
+                    candidates.append(normalized)
+
+        deduped: List[str] = []
+        seen = set()
+        for text in candidates:
+            if text in seen:
+                continue
+            seen.add(text)
+            deduped.append(text)
+        return deduped
+
+    def _find_query_matching_sku_index(
+        self,
+        skus: List[Dict[str, Any]],
+        query_texts: List[str],
+    ) -> Optional[int]:
+        """Return the first SKU whose option1_value appears in query texts."""
+        if not skus or not query_texts:
+            return None
+
+        for index, sku in enumerate(skus):
+            option1_value = self._normalize_sku_match_text(sku.get("option1_value"))
+            if not option1_value:
+                continue
+            if any(option1_value in query_text for query_text in query_texts):
+                return index
+        return None
+
+    def _encode_query_vector_for_sku_matching(
+        self,
+        parsed_query: ParsedQuery,
+        context: Optional[RequestContext] = None,
+    ) -> Optional[np.ndarray]:
+        """Best-effort fallback query embedding for final-page SKU matching."""
+        query_text = (
+            getattr(parsed_query, "rewritten_query", None)
+            or getattr(parsed_query, "query_normalized", None)
+            or getattr(parsed_query, "original_query", None)
+        )
+        if not query_text:
+            return None
+
+        text_encoder = getattr(self.query_parser, "text_encoder", None)
+        if text_encoder is None:
+            return None
+
+        try:
+            vectors = text_encoder.encode([query_text], priority=1)
+        except Exception as exc:
+            logger.warning("Failed to encode query vector for SKU matching: %s", exc, exc_info=True)
+            if context is not None:
+                context.add_warning(f"SKU query embedding failed: {exc}")
+            return None
+
+        if vectors is None or len(vectors) == 0:
+            return None
+
+        vector = vectors[0]
+        if vector is None:
+            return None
+        return np.asarray(vector, dtype=np.float32)
+
+    def _select_sku_by_embedding(
+        self,
+        skus: List[Dict[str, Any]],
+        option1_vectors: Dict[str, np.ndarray],
+        query_vector: np.ndarray,
+    ) -> Tuple[Optional[int], Optional[float]]:
+        """Select the SKU whose option1_value is most similar to the query."""
+        best_index: Optional[int] = None
+        best_score: Optional[float] = None
+
+        for index, sku in enumerate(skus):
+            option1_value_raw = sku.get("option1_value")
+            if option1_value_raw is None:
+                continue
+            option1_value = str(option1_value_raw).strip()
+            if not option1_value:
+                continue
+            option_vector = option1_vectors.get(option1_value)
+            if option_vector is None:
+                continue
+            score = float(np.inner(query_vector, option_vector))
+            if best_score is None or score > best_score:
+                best_index = index
+                best_score = score
+
+        return best_index, best_score
+
+    @staticmethod
+    def _promote_matching_sku(source: Dict[str, Any], match_index: int) -> Optional[Dict[str, Any]]:
+        """Move the matched SKU to the front and swap the SPU image."""
+        skus = source.get("skus")
+        if not isinstance(skus, list) or match_index < 0 or match_index >= len(skus):
+            return None
+
+        matched_sku = skus.pop(match_index)
+        skus.insert(0, matched_sku)
+
+        image_src = matched_sku.get("image_src") or matched_sku.get("imageSrc")
+        if image_src:
+            source["image_url"] = image_src
+        return matched_sku
+
+    def _apply_sku_sorting_for_page_hits(
+        self,
+        es_hits: List[Dict[str, Any]],
+        parsed_query: ParsedQuery,
+        context: Optional[RequestContext] = None,
+    ) -> None:
+        """Sort each page hit's SKUs so the best-matching SKU is first."""
+        if not es_hits:
+            return
+
+        query_texts = self._build_sku_query_texts(parsed_query)
+        unmatched_hits: List[Dict[str, Any]] = []
+        option1_values_to_encode: List[str] = []
+        seen_option1_values = set()
+        text_matched = 0
+        embedding_matched = 0
+
+        for hit in es_hits:
+            source = hit.get("_source")
+            if not isinstance(source, dict):
+                continue
+            skus = source.get("skus")
+            if not isinstance(skus, list) or not skus:
+                continue
+
+            match_index = self._find_query_matching_sku_index(skus, query_texts)
+            if match_index is not None:
+                self._promote_matching_sku(source, match_index)
+                text_matched += 1
+                continue
+
+            unmatched_hits.append(hit)
+            for sku in skus:
+                option1_value_raw = sku.get("option1_value")
+                if option1_value_raw is None:
+                    continue
+                option1_value = str(option1_value_raw).strip()
+                if not option1_value or option1_value in seen_option1_values:
+                    continue
+                seen_option1_values.add(option1_value)
+                option1_values_to_encode.append(option1_value)
+
+        if not unmatched_hits or not option1_values_to_encode:
+            return
+
+        query_vector = getattr(parsed_query, "query_vector", None)
+        if query_vector is None:
+            query_vector = self._encode_query_vector_for_sku_matching(parsed_query, context=context)
+        if query_vector is None:
+            return
+
+        text_encoder = getattr(self.query_parser, "text_encoder", None)
+        if text_encoder is None:
+            return
+
+        try:
+            encoded_option_vectors = text_encoder.encode(option1_values_to_encode, priority=1)
+        except Exception as exc:
+            logger.warning("Failed to encode SKU option1 values for final-page sorting: %s", exc, exc_info=True)
+            if context is not None:
+                context.add_warning(f"SKU option embedding failed: {exc}")
+            return
+
+        option1_vectors: Dict[str, np.ndarray] = {}
+        for option1_value, vector in zip(option1_values_to_encode, encoded_option_vectors):
+            if vector is None:
+                continue
+            option1_vectors[option1_value] = np.asarray(vector, dtype=np.float32)
+
+        query_vector_array = np.asarray(query_vector, dtype=np.float32)
+        for hit in unmatched_hits:
+            source = hit.get("_source")
+            if not isinstance(source, dict):
+                continue
+            skus = source.get("skus")
+            if not isinstance(skus, list) or not skus:
+                continue
+            match_index, _ = self._select_sku_by_embedding(skus, option1_vectors, query_vector_array)
+            if match_index is None:
+                continue
+            self._promote_matching_sku(source, match_index)
+            embedding_matched += 1
+
+        if text_matched or embedding_matched:
+            logger.info(
+                "Final-page SKU sorting completed | text_matched=%s | embedding_matched=%s",
+                text_matched,
+                embedding_matched,
+            )
+
     def search(
         self,
         query: str,
@@ -622,6 +848,8 @@ class Searcher:
                         continue
                     rerank_debug_by_doc[str(doc_id)] = item
  
+            self._apply_sku_sorting_for_page_hits(es_hits, parsed_query, context=context)
+
             # Format results using ResultFormatter
             formatted_results = ResultFormatter.format_search_results(
                 es_hits,
@@ -5,6 +5,7 @@ from pathlib import Path
 from types import SimpleNamespace
 from typing import Any, Dict, List
  
+import numpy as np
 import yaml
  
 from config import (
@@ -157,9 +158,7 @@ def _build_search_config(*, rerank_enabled: bool = True, rerank_window: int = 38
         rerank=RerankConfig(enabled=rerank_enabled, rerank_window=rerank_window),
         spu_config=SPUConfig(enabled=False),
         es_index_name="test_products",
-        tenant_config={},
         es_settings={},
-        services={},
     )
  
  
@@ -173,6 +172,19 @@ def _build_searcher(config: SearchConfig, es_client: _FakeESClient) -&gt; Searcher:
     return searcher
  
  
+class _FakeTextEncoder:
+    def __init__(self, vectors: Dict[str, List[float]]):
+        self.vectors = {
+            key: np.array(value, dtype=np.float32)
+            for key, value in vectors.items()
+        }
+
+    def encode(self, sentences, priority: int = 0, **kwargs):
+        if isinstance(sentences, str):
+            sentences = [sentences]
+        return np.array([self.vectors[text] for text in sentences], dtype=object)
+
+
 def test_config_loader_rerank_enabled_defaults_true(tmp_path: Path):
     config_data = {
         "es_index_name": "test_products",
@@ -327,3 +339,116 @@ def test_searcher_skips_rerank_when_page_exceeds_window(monkeypatch):
     assert es_client.calls[0]["size"] == 10
     assert es_client.calls[0]["include_named_queries_score"] is False
     assert len(es_client.calls) == 1
+
+
+def test_searcher_promotes_sku_when_option1_matches_translated_query(monkeypatch):
+    es_client = _FakeESClient(total_hits=1)
+    searcher = _build_searcher(_build_search_config(rerank_enabled=False), es_client)
+    context = create_request_context(reqid="sku-text", uid="u-sku-text")
+
+    monkeypatch.setattr(
+        "search.searcher.get_tenant_config_loader",
+        lambda: SimpleNamespace(get_tenant_config=lambda tenant_id: {"index_languages": ["en", "zh"]}),
+    )
+
+    class _TranslatedQueryParser:
+        text_encoder = None
+
+        def parse(self, query: str, tenant_id: str, generate_vector: bool, context: Any):
+            return _FakeParsedQuery(
+                original_query=query,
+                query_normalized=query,
+                rewritten_query=query,
+                translations={"en": "black dress"},
+            )
+
+    searcher.query_parser = _TranslatedQueryParser()
+
+    def _full_source_with_skus(doc_id: str) -> Dict[str, Any]:
+        return {
+            "spu_id": doc_id,
+            "title": {"en": f"product-{doc_id}"},
+            "brief": {"en": f"brief-{doc_id}"},
+            "vendor": {"en": f"vendor-{doc_id}"},
+            "image_url": "https://img/default.jpg",
+            "skus": [
+                {"sku_id": "sku-red", "option1_value": "Red", "image_src": "https://img/red.jpg"},
+                {"sku_id": "sku-black", "option1_value": "Black", "image_src": "https://img/black.jpg"},
+            ],
+        }
+
+    monkeypatch.setattr(_FakeESClient, "_full_source", staticmethod(_full_source_with_skus))
+
+    result = searcher.search(
+        query="黑色 连衣裙",
+        tenant_id="162",
+        from_=0,
+        size=1,
+        context=context,
+        enable_rerank=False,
+    )
+
+    assert len(result.results) == 1
+    assert result.results[0].skus[0].sku_id == "sku-black"
+    assert result.results[0].image_url == "https://img/black.jpg"
+
+
+def test_searcher_promotes_sku_by_embedding_when_query_has_no_direct_option_match(monkeypatch):
+    es_client = _FakeESClient(total_hits=1)
+    searcher = _build_searcher(_build_search_config(rerank_enabled=False), es_client)
+    context = create_request_context(reqid="sku-embed", uid="u-sku-embed")
+
+    monkeypatch.setattr(
+        "search.searcher.get_tenant_config_loader",
+        lambda: SimpleNamespace(get_tenant_config=lambda tenant_id: {"index_languages": ["en"]}),
+    )
+
+    encoder = _FakeTextEncoder(
+        {
+            "linen summer dress": [0.8, 0.2],
+            "Red": [1.0, 0.0],
+            "Blue": [0.0, 1.0],
+        }
+    )
+
+    class _EmbeddingQueryParser:
+        text_encoder = encoder
+
+        def parse(self, query: str, tenant_id: str, generate_vector: bool, context: Any):
+            return _FakeParsedQuery(
+                original_query=query,
+                query_normalized=query,
+                rewritten_query=query,
+                translations={},
+                query_vector=np.array([0.0, 1.0], dtype=np.float32),
+            )
+
+    searcher.query_parser = _EmbeddingQueryParser()
+
+    def _full_source_with_skus(doc_id: str) -> Dict[str, Any]:
+        return {
+            "spu_id": doc_id,
+            "title": {"en": f"product-{doc_id}"},
+            "brief": {"en": f"brief-{doc_id}"},
+            "vendor": {"en": f"vendor-{doc_id}"},
+            "image_url": "https://img/default.jpg",
+            "skus": [
+                {"sku_id": "sku-red", "option1_value": "Red", "image_src": "https://img/red.jpg"},
+                {"sku_id": "sku-blue", "option1_value": "Blue", "image_src": "https://img/blue.jpg"},
+            ],
+        }
+
+    monkeypatch.setattr(_FakeESClient, "_full_source", staticmethod(_full_source_with_skus))
+
+    result = searcher.search(
+        query="linen summer dress",
+        tenant_id="162",
+        from_=0,
+        size=1,
+        context=context,
+        enable_rerank=False,
+    )
+
+    assert len(result.results) == 1
+    assert result.results[0].skus[0].sku_id == "sku-blue"
+    assert result.results[0].image_url == "https://img/blue.jpg"